-
Notifications
You must be signed in to change notification settings - Fork 2
/
cron_rsync_generator_package_from_eos.sh
executable file
·347 lines (313 loc) · 15.6 KB
/
cron_rsync_generator_package_from_eos.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
#!/bin/bash
# versiono 0.1.7
version=0.1.7
source $HOME/cron_install_cmssw.config # notifytowhom
updated_list=/cvmfs/cms.cern.ch/cvmfs-cms.cern.ch-updates
#
# Use the one that is created by create_host_proxy_download_siteconf.sh
# If timeleft is not sufficient, create one
#
#x509cert=$HOME/CERTS/cvmfs.ihepa.ufl.edu/hostcert.pem
#x509certkey=$HOME/CERTS/cvmfs.ihepa.ufl.edu/hostkey.pem
#x509proxyvalid="168:30"
#export X509_USER_PROXY=$HOME/.cvmfs.host.proxy
export X509_USER_PROXY=$HOME/.florida.t2.proxy
# initial values:
export EOS_MGM_URL="root://eoscms.cern.ch"
#TOTAL_RSYNC_SIZE_LIMIT=100 # in Gigabytes
TOTAL_RSYNC_SIZE_LIMIT=150 # in Gigabytes 29APR2015
INDIVIDUAL_RSYNC_SIZE_LIMIT=5 # in Gigabytes
#$(ls -al /afs/cern.ch/project/eos/installation/cms | awk '{print $NF}')
#EOS_CLIENT_VERSION=${EOS_CLIENT_VERSION:-$(ls -al /afs/cern.ch/project/eos/installation/cms 2>/dev/null | awk '{print $NF}')}
#EOS_CLIENT_VERSION=${EOS_CLIENT_VERSION:-0.3.4}
EOS_CLIENT_VERSION=${EOS_CLIENT_VERSION:-0.3.15}
#EOS_CLIENT_VERSION=${EOS_CLIENT_VERSION:-0.3.35}
#alias eos='/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select'
#alias eoscms='eos'
#alias eosforceumount='killall eosfsd 2>/dev/null; killall -9 eosfsd 2>/dev/null; fusermount -u '
#alias eosmount='/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select -b fuse mount'
#alias eosumount='/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select -b fuse umount'
export EOSSYS=/home/cvcms/eos_installation/${EOS_CLIENT_VERSION}
#export EOSSYS=/afs/cern.ch/project/eos/installation/${EOS_CLIENT_VERSION}
function eos () {
$EOSSYS/bin/eos.select
return $?
}
function eoscms () {
$EOSSYS/bin/eos.select
return $?
}
function eosforceumount () {
killall eosfsd 2>/dev/null
killall -9 eosfsd 2>/dev/null
fusermount -u $1
return $?
}
#function eosmount () {
# $EOSSYS/bin/eos.select -b fuse mount $1
# return $?
#}
#function eosumount () {
# $EOSSYS/bin/eos.select -b fuse umount $1
# return $?
#}
function cic_sed_del_line () { # func description: It deletes a line
if [ $# -lt 2 ] ; then
echo ERROR cic_sed_del_line string file
return 1
fi
string=$1
infile=$2
sed -i "/$(echo $string | sed 's^/^\\\/^g')/ d" $infile
}
function cvmfs_server_transaction_check () {
status=$1
what="$2"
itry=0
while [ $itry -lt 10 ] ; do
if [ $status -eq 0 ] ; then
return 0
else
printf "$what cvmfs_server transaction Failing\n" | mail -s "ERROR cvmfs_server transaction Failed" $notifytowhom
echo INFO retrying $itry
cvmfs_server abort -f
cvmfs_server transaction
status=$?
[ $status -eq 0 ] && return 0
fi
itry=$(expr $itry + 1)
done
return 1
}
:
: ######### $0 ###################################################
:
# On DEC 16, 2014, I realized I need the grid proxy to mount eos personally
echo INFO X509_USER_PROXY $X509_USER_PROXY
echo INFO executing voms-proxy-info -timeleft
voms-proxy-info -timeleft 2>&1
#/usr/bin/lcg-cp -b -n 1 --vo cms -D srmv2 -T srmv2 -v srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/t2/operations/.cmsphedex.proxy file://$X509_USER_PROXY.copy
#if [ $? -eq 0 ] ; then
# cp $X509_USER_PROXY.copy $X509_USER_PROXY
# voms-proxy-info -all
#else
# printf "$(basename $0) ERROR failed to download $X509_USER_PROXY\n$(/usr/bin/lcg-cp -b -n 1 --vo cms -D srmv2 -T srmv2 -v srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/t2/operations/.cmsphedex.proxy file://${X509_USER_PROXY}.copy 2>&1 | sed 's#%#%%#g')n" | mail -s "$(basename $0) ERROR proxy download failed" $notifytowhom
#fi
#/usr/bin/lcg-cp -b -n 1 --vo cms -D srmv2 -T srmv2 -v srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/t2/operations/.cmsphedex.proxy file://$X509_USER_PROXY.copy
/usr/bin/lcg-cp -b -n 1 --vo cms -D srmv2 -T srmv2 -v gsiftp://cmsio6.rc.ufl.edu/cms/t2/operations/.cmsphedex.proxy file://$X509_USER_PROXY.copy
if [ $? -eq 0 ] ; then
cp $X509_USER_PROXY.copy $X509_USER_PROXY
voms-proxy-info -all
else
printf "$(basename $0) ERROR failed to download $X509_USER_PROXY\n$(/usr/bin/lcg-cp -b -n 1 --vo cms -D srmv2 -T srmv2 -v gsiftp://cmsio6.rc.ufl.edu/cms/t2/operations/.cmsphedex.proxy file://${X509_USER_PROXY}.copy 2>&1 | sed 's#%#%%#g')n" | mail -s "$(basename $0) ERROR proxy download failed" $notifytowhom
fi
timeleft=$(voms-proxy-info -timeleft 2>/dev/null)
if [ $timeleft -lt 1900 ] ; then # 1800 + 100
#/usr/bin/lcg-cp -b -n 1 --vo cms -D srmv2 -T srmv2 -v srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/t2/operations/.cmsphedex.proxy file://$X509_USER_PROXY
/usr/bin/lcg-cp -b -n 1 --vo cms -D srmv2 -T srmv2 -v gsiftp://cmsio6.rc.ufl.edu/cms/t2/operations/.cmsphedex.proxy file://$X509_USER_PROXY.copy
[ $? -eq 0 ] && cp $X509_USER_PROXY.copy $X509_USER_PROXY
if [ ] ; then
echo INFO creating the grid proxy
voms-proxy-init -cert $x509cert -key $x509certkey -out $X509_USER_PROXY -valid ${x509proxyvalid} 2>&1
if [ $? -ne 0 ] ; then
printf "$(basename $0) ERROR voms-proxy-init failed\n" | mail -s "$(basename $0) ERROR voms-proxy-init failed" $notifytowhom
exit 1
fi
fi
timeleft=$(voms-proxy-info -timeleft 2>/dev/null)
fi
echo INFO proxy timeleft $timeleft
#
# Limit frequency of the execution
#
# Execute only during the first half of the even hours
# Execute only every five hours
#[ $(expr $(date +%H) % 5 ) -eq 0 ] || { echo INFO Execute only every five hours ; exit 0 ; } ;
#[ $(date +%M) -lt 30 ] || { echo INFO Execute only the first half of the even hours ; exit 0 ; } ;
rsync_source="$HOME/eos2/cms/store/group/phys_generator/cvmfs/gridpacks"
rsync_name="/cvmfs/cms.cern.ch/phys_generator/gridpacks"
echo INFO Checking mounting eos
ls $HOME/eos2 | grep "Bad address" | grep -q "cannot access"
if [ $? -eq 0 ] ; then
echo Warning $HOME/eos2 is not properly mounted
ls $HOME/eos2
echo Warning eosforceumount $HOME/eos2
eosforceumount $HOME/eos2
fi
df -h | grep -q $(echo $EOS_MGM_URL | cut -d/ -f3 | cut -d: -f1)
if [ $? -eq 0 ] ; then
echo INFO $HOME/eos2 is already mounted
else
$EOSSYS/bin/eos.select -b fuse mount $HOME/eos2
fi
if [ ! -d $rsync_source ] ; then
echo ERROR rsync_source not found eosmount error
printf "$(basename $0) $rsync_source not found \n Issue with: $EOSSYS/bin/eos.select -b fuse mount $HOME/eos2 did not work\nls $HOME/eos2/cms follows\n$(ls $HOME/eos2/cms)\ntail -10 eos log\n$(tail -10 /tmp/eos*)" | mail -s "$(basename $0) ERROR $EOSSYS/bin/eos.select -b fuse mount $HOME/eos2 failed " $notifytowhom
$EOSSYS/bin/eos.select -b fuse umount $HOME/eos2
ps auxwww | grep -v grep | grep -q eosfsd
if [ $? -eq 0 ] ; then
echo Warning eosforceumount $HOME/eos2
eosforceumount $HOME/eos2
fi
ls $HOME/eos2
exit 1
fi
echo INFO looks good $rsync_source exists
TOTAL_RSYNC_SIZE=$(/usr/bin/du -s $rsync_source | awk '{print $1}')
TOTAL_RSYNC_SIZE=$(echo "scale=2 ; $TOTAL_RSYNC_SIZE / 1024 / 1024" | bc | cut -d. -f1)
[ "x$TOTAL_RSYNC_SIZE" == "x" ] && TOTAL_RSYNC_SIZE=0
if [ $TOTAL_RSYNC_SIZE -gt $TOTAL_RSYNC_SIZE_LIMIT ] ; then
echo Warning TOTAL RSYNC SIZE is too large
#printf "$(basename $0) Warning TOTAL_RSYNC_SIZE > TOTAL_RSYNC_SIZE_LIMIT : $TOTAL_RSYNC_SIZE > $TOTAL_RSYNC_SIZE_LIMIT \n Will not rsync $rsync_source" | mail -s "$(basename $0) Warning ERROR TOTAL_RSYNC_SIZE > TOTAL_RSYNC_SIZE_LIMIT" $notifytowhom
fi
echo INFO looks good TOTAL_RSYNC_SIZE "<" TOTAL_RSYNC_SIZE_LIMIT
# Check Point 1
#echo rsync -arzuvp --delete $rsync_source $(dirname $rsync_name)
#exit 0
cvmfs_server list | grep stratum0 | grep -q transaction
if [ $? -eq 0 ] ; then
#if [ ! -f $lock ] ; then
# echo INFO $lock does not exist
#need_to_fix_mount_issue=2
#printf "$(basename $0) cvmfs mount issue\n$lock does not exist\ncvmfs_server list\n$(cvmfs_server list)\n" | mail -s "$(basename $0) needs to fix the mount issue" $notifytowhom
echo ERROR cvmfs server already in transaction
exit 1
#fi
fi
echo INFO Doing cvmfs_server transaction
cvmfs_server transaction
status=$?
what="$(basename $0)"
cvmfs_server_transaction_check $status $what
if [ $? -eq 0 ] ; then
echo INFO transaction OK for $what
else
echo ERROR transaction check FAILED
printf "cvmfs_server_transaction_check Failed for $what\n" | mail -s "ERROR: cvmfs_server_transaction_check Failed" $notifytowhom
$EOSSYS/bin/eos.select -b fuse umount $HOME/eos2
ps auxwww | grep -v grep | grep -q eosfsd
if [ $? -eq 0 ] ; then
echo Warning eosforceumount $HOME/eos2
eosforceumount $HOME/eos2
fi
ls $HOME/eos2
exit 1
fi
echo DEBUG check point will execute
#echo rsync -arzuvp $rsync_source $(dirname $rsync_name)
#rm -f $HOME/rsync+generator+package+from+eos.log
#rsync -arzuvp $rsync_source $(dirname $rsync_name) 2>&1 | tee $HOME/rsync+generator+package+from+eos.log
echo rsync -arzuvp --delete $rsync_source $(dirname $rsync_name)
#echo rsync -arzuvp $rsync_source $(dirname $rsync_name)
thelog=$HOME/logs/rsync+generator+package+from+eos.log
rm -f $thelog
rsync -arzuvp --delete $rsync_source $(dirname $rsync_name) > $thelog 2>&1
#rsync -arzuvp $rsync_source $(dirname $rsync_name) > $thelog 2>&1
status=$?
cat $thelog
if [ $status -eq 0 ] ; then
publish_needed=0
files_with_strange_permission=""
i=0
for f in $(grep ^$(basename $rsync_source) $thelog 2>/dev/null) ; do
i=$(expr $i + 1)
[ -f "$(dirname $rsync_name)/$f" ] || { echo "[ $i ] " $(dirname $rsync_name)/$f is not a file $publish_needed ; continue ; } ;
publish_needed=1
echo "[ $i ] " $(dirname $rsync_name)/$f is a file $publish_needed
INDIVIDUAL_RSYNC_SIZE=$(/usr/bin/du -s $(dirname $rsync_name)/$f | awk '{print $1}')
INDIVIDUAL_RSYNC_SIZE=$(echo "scale=2 ; $INDIVIDUAL_RSYNC_SIZE / 1024 / 1024" | bc | cut -d. -f1)
[ "x$INDIVIDUAL_RSYNC_SIZE" == "x" ] && INDIVIDUAL_RSYNC_SIZE=0
if [ $INDIVIDUAL_RSYNC_SIZE -gt $INDIVIDUAL_RSYNC_SIZE_LIMIT ] ; then
echo ERROR INDIVIDUAL_RSYNC_SIZE -gt INDIVIDUAL_RSYNC_SIZE_LIMIT $INDIVIDUAL_RSYNC_SIZE -gt $INDIVIDUAL_RSYNC_SIZE_LIMIT
printf "$(basename $0) ERROR INDIVIDUAL_RSYNC_SIZE > INDIVIDUAL_RSYNC_SIZE_LIMIT : $INDIVIDUAL_RSYNC_SIZE > $INDIVIDUAL_RSYNC_SIZE_LIMIT $(dirname $rsync_name)/$f \n Will not publish the rsync result" | mail -s "$(basename $0) ERROR ERROR INDIVIDUAL_RSYNC_SIZE > INDIVIDUAL_RSYNC_SIZE_LIMIT" $notifytowhom
publish_needed=0
break
fi
themode=$(/usr/bin/stat -c %a $(dirname $rsync_name)/$f)
original_file=$(echo $(dirname $rsync_name)/$f | sed "s#$rsync_name#$rsync_source#")
original_mode=$(/usr/bin/stat -c %a $original_file)
original_user=$(/usr/bin/stat -c %U $original_file)
if [ $themode -lt 400 ] ; then
theuser=$(/usr/bin/stat -c %U $(dirname $rsync_name)/$f)
files_with_strange_permission="$files_with_strange_permission ${original_mode}+${original_user}+${themode}+${theuser}+$(dirname $rsync_name)/${f}"
fi
if [ $(echo $themode | cut -c2-) -lt 40 ] ; then
theuser=$(/usr/bin/stat -c %U $(dirname $rsync_name)/$f)
echo "$files_with_strange_permission" | grep -q "+$(dirname $rsync_name)/$f" || files_with_strange_permission="$files_with_strange_permission ${original_mode}+${original_user}+${themode}+${theuser}+$(dirname $rsync_name)/$f"
fi
if [ $(echo $themode | cut -c3-) -lt 4 ] ; then
theuser=$(/usr/bin/stat -c %U $(dirname $rsync_name)/$f)
#files_with_strange_permission="$files_with_strange_permission ${original_mode}+${original_user}+${themode}+${theuser}+$(dirname $rsync_name)/$f"
echo "$files_with_strange_permission" | grep -q "+$(dirname $rsync_name)/$f" || files_with_strange_permission="$files_with_strange_permission ${original_mode}+${original_user}+${themode}+${theuser}+$(dirname $rsync_name)/$f"
fi
done
if [ "x$files_with_strange_permission" != "x" ] ; then
printout=$(printf "$(basename $0) Found files with strange permsion\n$(for f in $files_with_strange_permission ; do echo $f ; done)\n")
for f in $files_with_strange_permission ; do
thefile=$(echo $f | sed 's#+# #g' | awk '{print $NF}')
chmod 644 $thefile
done
printf "$printout\nStrange files after changing the perm\n$(for f in $files_with_strange_permission ; do ls -al $f ; done)\n" | mail -s "$(basename $0) Warning Found files with strange permsion" $notifytowhom
fi
echo INFO check point publish_needed $publish_needed
if [ $publish_needed -eq 0 ] ; then
echo INFO publish was not needed, So ending the transaction
( cd ; cvmfs_server abort -f ; ) ;
else
echo INFO publish necessary
echo INFO updating $updated_list
# db updated_list
date_s_now=$(echo $(/bin/date +%s) $(/bin/date -u))
grep -q "gridpacks $(echo $f | cut -d/ -f2) $(echo $date_s_now | awk '{print $1}')" $updated_list
if [ $? -eq 0 ] ; then
echo Warning "gridpacks $(echo $f | cut -d/ -f2) $(echo $date_s_now | awk '{print $1}')" is already in the $updated_list
else
echo INFO adding "gridpacks $(echo $f | cut -d/ -f2) $(echo $date_s_now | awk '{print $1}')" to $updated_list
echo "gridpacks $(echo $f | cut -d/ -f2) $date_s_now" >> $updated_list
fi
thestring="gridpacks $(echo $f | cut -d/ -f2) $(echo $date_s_now | awk '{print $1}')"
echo INFO adding 'phys_generator/gridpacks/slc*/*/*' to /cvmfs/cms.cern.ch/.cvmfsdirtab
# nested stuff
grep -q /phys_generator/gridpacks/slc /cvmfs/cms.cern.ch/.cvmfsdirtab
if [ $? -ne 0 ] ; then
echo '/phys_generator/gridpacks/slc*/*/*' >> /cvmfs/cms.cern.ch/.cvmfsdirtab
fi
echo INFO publishing $rsync_name
currdir=$(pwd)
cd
time cvmfs_server publish 2>&1 | tee $HOME/logs/cvmfs_server+publish+rsync+generator+package+from+eos.log
status=$?
cd $currdir
if [ $status -eq 0 ] ; then
#printf "$(basename $0) cvmfs_server_publish OK \n$(cat $HOME/cvmfs_server+publish+rsync+generator+package+from+eos.log | sed 's#%#%%#g')\n" | mail -s "$(basename $0) cvmfs_server publish for $package OK" $notifytowhom
printf "$(basename $0) cvmfs_server_publish OK \n$(cat $HOME/logs/cvmfs_server+publish+rsync+generator+package+from+eos.log | sed 's#%#%%#g')\n"
else
( cd ; echo Warning deleting "$thestring" from $updated_list ; cic_del_line "$thestring" $updated_list ; ) ;
echo ERROR failed cvmfs_server publish
printf "$(basename $0) cvmfs_server publish failed\n$(cat $HOME/logs/cvmfs_server+publish+rsync+generator+package+from+eos.log | sed 's#%#%%#g')\n" | mail -s "$(basename $0) cvmfs_server publish failed" $notifytowhom
( cd ; cvmfs_server abort -f ; ) ; # cvmfs_server abort -f
fi
fi
else
echo ERROR failed : rsync -arzuvp $rsync_source $(dirname $rsync_name)
printf "$(basename $0) ERROR FAILED: rsync -arzuvp $rsync_source $(dirname $rsync_name)\n" | mail -s "$(basename $0) ERROR FAILED rsync" $notifytowhom
( cd ; cvmfs_server abort -f ; ) ; # cvmfs_server abort -f
fi
echo INFO eosumount $HOME/eos2
$EOSSYS/bin/eos.select -b fuse umount $HOME/eos2
ps auxwww | grep -v grep | grep -q eosfsd
if [ $? -eq 0 ] ; then
echo Warning eosforceumount $HOME/eos2
eosforceumount $HOME/eos2
fi
echo INFO checking with ls $HOME/eos2
ls $HOME/eos2
echo script $0 Done
log=$(basename $0 | sed 's#\.sh#\.log#g')
eos_fuse_logs=
for f in /tmp/eos-fuse.*.log ; do
[ -f "$f" ] && { eos_fuse_logs="$eos_fuse_logs $f" ; rm -f $f ; } ;
done
printf "$(basename $0) Done\nEOS Client Version=$EOS_CLIENT_VERSION\nRemoved $eos_fuse_logs\n$(ls -al /tmp)\n$(cat $log 2>&1 | sed 's#%#%%#g')\n"
#printf "$(basename $0) Done\nEOS Client Version=$EOS_CLIENT_VERSION\nRemoved $eos_fuse_logs\n$(ls -al /tmp)\n$(cat $log 2>&1 | sed 's#%#%%#g')\n" | mail -s "$(basename $0) Done" $notifytowhom
exit 0