forked from ESGF/esgf-installer
-
Notifications
You must be signed in to change notification settings - Fork 0
/
esg-node
executable file
·7208 lines (6226 loc) · 312 KB
/
esg-node
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
#####
# esg-node: ESGF Node Application Stack
# chkconfig: 345 98 02
# description: Installer for the ESGF Node application stack
#
#****************************************************************************
#* *
#* Organization: Earth System Grid Federation *
#* Project: Earth Systems Grid Fed (ESGF) Node Software Stack *
#* First Author: Gavin M. Bell ([email protected]) *
#* *
#****************************************************************************
#* *
#* Copyright (c) 2009, Lawrence Livermore National Security, LLC. *
#* Produced at the Lawrence Livermore National Laboratory *
#* Written by: Gavin M. Bell ([email protected]) *
#* LLNL-CODE-420962 *
#* *
#* All rights reserved. This file is part of the: *
#* Earth System Grid Fed (ESGF) Node Software Stack, Version 1.0 *
#* *
#* For details, see http://esgf.llnl.gov *
#* Please also read this link *
#* http://esgf.llnl.gov/LICENSE *
#* *
#* * Redistribution and use in source and binary forms, with or *
#* without modification, are permitted provided that the following *
#* conditions are met: *
#* *
#* * Redistributions of source code must retain the above copyright *
#* notice, this list of conditions and the disclaimer below. *
#* *
#* * Redistributions in binary form must reproduce the above copyright *
#* notice, this list of conditions and the disclaimer (as noted below) *
#* in the documentation and/or other materials provided with the *
#* distribution. *
#* *
#* Neither the name of the LLNS/LLNL nor the names of its contributors *
#* may be used to endorse or promote products derived from this *
#* software without specific prior written permission. *
#* *
#* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS *
#* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT *
#* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS *
#* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE *
#* LIVERMORE NATIONAL SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR *
#* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, *
#* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT *
#* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF *
#* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND *
#* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, *
#* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT *
#* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF *
#* SUCH DAMAGE. *
#* *
#****************************************************************************
#####
#uses: perl, awk, ifconfig, tar, wget, curl, su, useradd, groupadd,
# id, chmod, chown, chgrp, cut, svn, mkdir, killall, java, egrep,
# lsof, unlink, ln, pax, keytool, openssl, getent
#note: usage of readlink not macosx friendly :-( usage of useradd /
# groupadd is RedHat/CentOS dependent :-(
#note on getting bash version...
#bash --version | head -n +1 | sed -n -e 's/.*version[ ]*\([^(-]*\).*/\1/p'
export LANG=POSIX
umask 022
DEBUG=${DEBUG:-0}
VERBOSE=${VERBOSE:-0}
devel=${devel:-0}
recommended=1
custom=0
use_local_files=0
progname="esg-node"
script_version="v2.0-RC5.4.0-devel"
script_maj_version="2.0"
script_release="Centaur"
envfile="/etc/esg.env"
#--------------
#User Defined / Settable (public)
#--------------
install_prefix=${install_prefix:-${ESGF_INSTALL_PREFIX:-"/usr/local"}}
#--------------
#--------------
#Script vars - do not edit
#--------------
esg_functions_file="${install_prefix}/bin/esg-functions"
esg_init_file="${install_prefix}/bin/esg-init"
#--------------
esgf_nm_funcs_file="${install_prefix}/bin/esgf-nm-func"
export UVCDAT_ANONYMOUS_LOG=False
[ -e "${envfile}" ] && source ${envfile} && ((VERBOSE)) && printf "sourcing environment from: ${envfile} \n"
init_structure() {
#--------------
#Prepare necessary support filesystem structure and files
#--------------
(($DEBUG)) && echo "init_structure: esg_dist_url = ${esg_dist_url}"
#--------------
#Let's go down the line and make sure that we have what we need structurally on the filesystem
local config_check=8
if [ ! -e ${scripts_dir} ]; then mkdir -p ${scripts_dir} && ((config_check--)); else ((config_check--)); fi
if [ ! -e ${esg_backup_dir} ]; then mkdir -p ${esg_backup_dir} && ((config_check--)); else ((config_check--)); fi
if [ ! -e ${esg_tools_dir} ]; then mkdir -p ${esg_tools_dir} && ((config_check--)); else ((config_check--)); fi
if [ ! -e ${esg_log_dir} ]; then mkdir -p ${esg_log_dir} && ((config_check--)); else ((config_check--)); fi
if [ ! -e ${esg_config_dir} ]; then mkdir -p ${esg_config_dir} && ((config_check--)); else ((config_check--)); fi
if [ ! -e ${esg_etc_dir} ]; then mkdir -p ${esg_etc_dir} && ((config_check--)); else ((config_check--)); fi
if [ ! -e ${tomcat_conf_dir} ]; then mkdir -p ${tomcat_conf_dir} && ((config_check--)); else ((config_check--)); fi
if [ ! -e ${config_file} ]; then touch ${config_file} && ((config_check--)); else ((config_check--)); fi
debug_print ${config_check}
((config_check != 0 )) && echo "ERROR: checklist incomplete $([FAIL])" && checked_done 1 || verbose_print "checklist $([OK])"
#--------------
chmod 777 ${esg_etc_dir} 2> /dev/null
[ -w "${envfile}" ] && write_paths
#--------------
#Setup variables....
#--------------
check_for_my_ip
esgf_host=${esgf_host}
[ -z "${esgf_host}" ] && get_property esgf_host
esgf_default_peer=${esgf_default_peer}
[ -z "${esgf_default_peer}" ] && get_property esgf_default_peer
esgf_idp_peer_name=${esgf_idp_peer_name}
[ -z "${esgf_idp_peer_name}" ] && get_property esgf_idp_peer_name
esgf_idp_peer=${esgf_idp_peer}
[ -z "${esgf_idp_peer}" ] && get_property esgf_idp_peer
myproxy_endpoint=${esgf_idp_peer%%/*}
[ -z "${myproxy_port}" ] && get_property myproxy_port
myproxy_port=${myproxy_port:-7512}
esg_root_id=${esg_root_id}
[ -z "${esg_root_id}" ] && get_property esg_root_id
node_peer_group=${node_peer_group}
[ -z "${node_peer_group}" ] && get_property node_peer_group
[ -z "${node_short_name}" ] && get_property node_short_name
#NOTE: Calls to get_property must be made AFTER we touch the file ${config_file} to make sure it exists
#this is actually an issue with dedup_properties that gets called in the get_property function
#Get the distinguished name from environment... if not, then esgf.properties... and finally this can be overwritten by the --dname option
#Here node_dn is written in the /XX=yy/AAA=bb (macro->micro) scheme.
#We transform it to dname which is written in the java style AAA=bb, XX=yy (micro->macro) scheme using "standard2java_dn" function
dname=${dname}
[ -z "${dname}" ] && get_property node_dn && dname=$(standard2java_dn ${node_dn})
gridftp_config=${gridftp_config}
[ -z "${gridftp_config}" ] && get_property gridftp_config "bdm end-user"
publisher_config=${publisher_config}
[ -z "${publisher_config}" ] && get_property publisher_config "esg.ini"
publisher_home=${publisher_home}
[ -z "${publisher_home}" ] && get_property publisher_home ${esg_config_dir}/esgcet
# Sites can override default keystore_alias in esgf.properties (keystore.alias=)
get_property keystore_alias ${keystore_alias}
export ESGINI=${publisher_home}/${publisher_config}
((DEBUG)) && echo "ESGINI = ${ESGINI}"
return 0
}
write_paths() {
((show_summary_latch++))
echo "export ESGF_HOME=${esg_root_dir}" >> ${envfile}
echo "export ESG_USER_HOME=${installer_home}" >> ${envfile}
echo "export ESGF_INSTALL_WORKDIR=${workdir}" >> ${envfile}
echo "export ESGF_INSTALL_PREFIX=${install_prefix}" >> ${envfile}
echo "export PATH=$myPATH:\$PATH" >> ${envfile}
echo "export LD_LIBRARY_PATH=$myLD_LIBRARY_PATH:\$LD_LIBRARY_PATH" >> ${envfile}
dedup ${envfile} && source ${envfile}
}
#checking for what we expect to be on the system a-priori
#that we are not going to install or be responsible for
check_prerequisites() {
printf "
\033[01;31m
EEEEEEEEEEEEEEEEEEEEEE SSSSSSSSSSSSSSS GGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFF
E::::::::::::::::::::E SS:::::::::::::::S GGG::::::::::::GF::::::::::::::::::::F
E::::::::::::::::::::ES:::::SSSSSS::::::S GG:::::::::::::::GF::::::::::::::::::::F
EE::::::EEEEEEEEE::::ES:::::S SSSSSSS G:::::GGGGGGGG::::GFF::::::FFFFFFFFF::::F
E:::::E EEEEEES:::::S G:::::G GGGGGG F:::::F FFFFFF\033[0m
\033[01;33m E:::::E S:::::S G:::::G F:::::F
E::::::EEEEEEEEEE S::::SSSS G:::::G F::::::FFFFFFFFFF
E:::::::::::::::E SS::::::SSSSS G:::::G GGGGGGGGGG F:::::::::::::::F
E:::::::::::::::E SSS::::::::SS G:::::G G::::::::G F:::::::::::::::F
E::::::EEEEEEEEEE SSSSSS::::S G:::::G GGGGG::::G F::::::FFFFFFFFFF\033[0m
\033[01;32m E:::::E S:::::SG:::::G G::::G F:::::F
E:::::E EEEEEE S:::::S G:::::G G::::G F:::::F
EE::::::EEEEEEEE:::::ESSSSSSS S:::::S G:::::GGGGGGGG::::GFF:::::::FF
E::::::::::::::::::::ES::::::SSSSSS:::::S GG:::::::::::::::GF::::::::FF
E::::::::::::::::::::ES:::::::::::::::SS GGG::::::GGG:::GF::::::::FF
EEEEEEEEEEEEEEEEEEEEEE SSSSSSSSSSSSSSS GGGGGG GGGGFFFFFFFFFFF.llnl.gov
\033[0m
"
printf "Checking that you have root privs on $(hostname)... "
id | grep root >& /dev/null
[ $? != 0 ] && printf "$([FAIL]) \n\tMust run this program with root's effective UID\n\n" && return 1
[OK]
#----------------------------------------
echo "Checking requisites... "
# checking for OS, architecture, distribution and version
OS=`uname -s`
MACH=`uname -m`
if [ "${OS}" = "Linux" ] && [ "${MACH}" = "x86_64" ] && [ -f /etc/redhat-release ] ; then
MAJREV=`cat /etc/redhat-release | sed s/.*release\ // | sed s/\ .*// | sed s/[.].*//`
fi
if [ "${MAJREV}" = 6 ] ; then
DIST=`cat /etc/redhat-release | sed s/release.*//`
DISTRIB=`echo $DIST`
fi
if [[ -z $DISTRIB ]] ; then echo "ESGF can only be installed on versions 6 of Red Hat, CentOS or Scientific Linux x86_64 systems" && exit 1 ; fi
#----------------------------------------
echo
return 0
}
#####
# Java
#####
setup_java() {
echo -n "Checking for java >= ${java_min_version} and valid JAVA_HOME... "
[ -e ${java_install_dir} ] && check_version $java_install_dir/bin/java ${java_min_version}
[ $? == 0 ] && (( ! force_install )) && [OK] && return 0
echo
echo "*******************************"
echo "Setting up Java... ${java_version}"
echo "*******************************"
echo
local last_java_truststore_file
local default="Y"
((force_install)) && default="N"
local dosetup
if [ -x ${java_install_dir}/bin/java ]; then
echo "Detected an existing java installation..."
read -e -p "Do you want to continue with Java installation and setup? $([ "$default" = "N" ] && echo "[y/N]" || echo "[Y/n]") " dosetup
[ -z ${dosetup} ] && dosetup=${default}
if [ "${dosetup}" != "Y" ] && [ "${dosetup}" != "y" ]; then
echo "Skipping Java installation and setup - will assume Java is setup properly"
return 0
fi
last_java_truststore_file=$(readlink -f ${truststore_file})
echo
fi
mkdir -p ${workdir}
[ $? != 0 ] && checked_done 1
pushd ${workdir} #>& /dev/null
local java_dist_file=${java_dist_url##*/}
#strip off -(32|64).tar.gz at the end
java_dist_dir=$(echo ${java_dist_file} | awk 'gsub(/-(32|64)('$compress_extensions')/,"")')
#Check to see if we have an Java distribution directory
if [ ! -e ${java_install_dir%/*}/${java_dist_dir} ]; then
echo "Don't see java distribution dir ${java_install_dir%/*}/${java_dist_dir}"
if [ ! -e ${java_dist_file} ]; then
echo "Don't see java distribution file $(pwd)/${java_dist_file} either"
echo "Downloading Java from ${java_dist_url}"
checked_get ${java_dist_file} ${java_dist_url} $((force_install))
[ $? != 0 ] && echo " ERROR: Could not download Java" && popd && checked_done 1
echo "unpacking ${java_dist_file}..."
tar xzf ${java_dist_file} -C ${java_install_dir%/*} # i.e. /usr/local
[ $? != 0 ] && echo " ERROR: Could not extract Java" && popd && checked_done 1
fi
fi
#If you don't see the directory but see the tar.gz distribution
#then expand it
if [ -e ${java_dist_file} ] && [ ! -e ${java_install_dir%/*}/${java_dist_dir} ]; then
echo "unpacking ${java_dist_file}..."
tar xzf ${java_dist_file} -C ${java_install_dir%/*} # i.e. /usr/local
[ $? != 0 ] && echo " ERROR: Could not extract Java..." && popd && checked_done 1
fi
if [ ! -e ${java_install_dir} ]; then
ln -s ${java_install_dir%/*}/${java_dist_dir} ${java_install_dir}
[ $? != 0 ] && \
echo " ERROR: Could not create sym link ${java_install_dir%/*}/${java_dist_dir} -> ${java_install_dir}" && popd && checked_done 1
else
unlink ${java_install_dir}
[ $? != 0 ] && mv ${java_install_dir} ${java_install_dir}.$(date ${date_format}).bak
ln -s ${java_install_dir%/*}/${java_dist_dir} ${java_install_dir}
[ $? != 0 ] && \
echo " ERROR*: Could not create sym link ${java_install_dir%/*}/${java_dist_dir} -> ${java_install_dir}" && popd && checked_done 1
fi
debug_print "chown -R ${installer_uid}:${installer_gid} ${java_install_dir}"
chown ${installer_uid}:${installer_gid} ${java_install_dir}
chown -R ${installer_uid}:${installer_gid} $(readlink -f ${java_install_dir})
popd >& /dev/null
${java_install_dir}/bin/java -version
[ $? != 0 ] && echo "ERROR cannot run ${java_install_dir}/bin/java" && checked_done 1
write_java_env
write_java_install_log
#-----------------------------
#In the situation where this function is called under update
#semantics i.e. there is already a previous installation of java
#and installation of tomcat with tomcat setup with a properly
#generated/configured jssecacerts file and there is a valid
#ESGF_IDP_PEER being pointed to. We should copy over that
#jssecacerts into this newly installed VM to satisfy SSL.
if [ -n "${last_java_truststore_file}" ] && [ -e "${last_java_truststore_file}" ]; then
mkdir -p ${java_install_dir}/conf
cp -v ${last_java_truststore_file} ${java_install_dir}/conf
chmod 644 ${java_install_dir}/conf/${last_java_truststore_file##*/}
fi
#-----------------------------
checked_done 0
}
write_java_env() {
((show_summary_latch++))
echo "export JAVA_HOME=${java_install_dir}" >> ${envfile}
prefix_to_path PATH ${java_install_dir}/bin >> ${envfile}
dedup ${envfile} && source ${envfile}
return 0
}
write_java_install_log() {
echo "$(date ${date_format}) java=${java_version} ${java_install_dir%/*}/${java_dist_dir}" >> ${install_manifest}
dedup ${install_manifest}
return 0
}
#####
# Ant
#####
setup_ant() {
echo -n "Checking for ant >= ${ant_min_version} "
[ -e ${ant_install_dir} ] && check_version ${ant_install_dir}/bin/ant ${ant_min_version}
[ $? == 0 ] && (( ! force_install )) && [OK] && return 0
echo
echo "*******************************"
echo "Setting up Ant... ${ant_version}"
echo "*******************************"
echo
local default="Y"
((force_install)) && default="N"
local dosetup
if [ -x ${ant_install_dir}/bin/ant ]; then
echo "Detected an existing ant installation..."
read -e -p "Do you want to continue with Ant installation and setup? $([ "$default" = "N" ] && echo "[y/N]" || echo "[Y/n]") " dosetup
[ -z ${dosetup} ] && dosetup=${default}
if [ "${dosetup}" != "Y" ] && [ "${dosetup}" != "y" ]; then
echo "Skipping Ant installation and setup - will assume ant is setup properly"
return 0
fi
echo
fi
mkdir -p ${workdir}
[ $? != 0 ] && checked_done 1
pushd ${workdir} >& /dev/null
local ant_dist_file=${ant_dist_url##*/}
#strip off -bin.tar.gz at the end
ant_dist_dir=${ant_dist_file/-bin.tar.gz}
#There is this pesky case of having a zero sized dist file...
if [ -e ${ant_dist_file} ]; then
ls -l ${ant_dist_file}
local size=$(stat -c%s ${ant_dist_file})
(( size == 0 )) && rm -v ${ant_dist_file}
fi
#Check to see if we have an Ant distribution directory
if [ ! -e ${ant_install_dir%/*}/${ant_dist_dir} ]; then
echo "Don't see ant distribution dir ${ant_install_dir%/*}/${ant_dist_dir}"
if [ ! -e ${ant_dist_file} ]; then
echo "Don't see ant distribution file $(pwd)/${ant_dist_file} either"
echo "Downloading Ant from ${ant_dist_url}"
wget -O ${ant_dist_file} ${ant_dist_url}
[ $? != 0 ] && echo " ERROR: Could not download Ant" && popd && checked_done 1
echo "unpacking ${ant_dist_file}..."
tar xzf ${ant_dist_file} -C ${ant_install_dir%/*} # i.e. /usr/local
[ $? != 0 ] && echo " ERROR: Could not extract Ant" && popd && checked_done 1
fi
fi
#If you don't see the directory but see the tar.gz distribution
#then expand it
if [ -e ${ant_dist_file} ] && [ ! -e ${ant_install_dir%/*}/${ant_dist_dir} ]; then
echo "unpacking ${ant_dist_file}..."
tar xzf ${ant_dist_file} -C ${ant_install_dir%/*} # i.e. /usr/local
[ $? != 0 ] && echo " ERROR: Could not extract Ant..." && popd && checked_done 1
fi
if [ ! -e ${ant_install_dir} ]; then
ln -s ${ant_install_dir%/*}/${ant_dist_dir} ${ant_install_dir}
[ $? != 0 ] && \
echo " ERROR: Could not create sym link ${ant_install_dir%/*}/${ant_dist_dir} -> ${ant_install_dir}" && popd && checked_done 1
else
unlink ${ant_install_dir}
[ $? != 0 ] && mv ${ant_install_dir} ${ant_install_dir}.$(date ${date_format}).bak
ln -s ${ant_install_dir%/*}/${ant_dist_dir} ${ant_install_dir}
[ $? != 0 ] && \
echo " ERROR: Could not create sym link ${ant_install_dir%/*}/${ant_dist_dir} -> ${ant_install_dir}" && popd && checked_done 1
fi
${ant_install_dir}/bin/ant -version
[ $? != 0 ] && echo "ERROR cannot run ${ant_install_dir}/bin/ant" && checked_done 1
write_ant_env
write_ant_install_log
checked_done 0
}
write_ant_env() {
((show_summary_latch++))
echo "export ANT_HOME=${ant_install_dir}" >> ${envfile}
prefix_to_path PATH ${ant_install_dir}/bin >> ${envfile}
dedup ${envfile} && source ${envfile}
return 0
}
write_ant_install_log() {
echo "$(date ${date_format}) ant=${ant_version} $(readlink -f ${ant_install_dir})" >> ${install_manifest}
dedup ${install_manifest}
return 0
}
#####
# PostgreSQL
#####
setup_postgress() {
debug_print "DEBUG: entering setup_postgress()"
_is_managed_db && return 0
echo -n "Checking for postgresql >= ${postgress_min_version} "
check_version ${postgress_bin_dir}/postgres ${postgress_min_version}
local ret=$?
if [ $ret == 0 ] ; then
(( ! force_install )) && [OK] && return 0
fi
local upgrade=$(( (ret==1) ? 1 : 0 )) #see check_version() function comments for meaning of return values
#---------------------------------------
#Setup PostgreSQL RPM repository
#---------------------------------------
echo
echo "*****************************"
echo "Setting PostgreSQL Server RPM"
echo "*****************************"
echo
yum -y install postgresql postgresql-server postgresql-devel
[ $? != 0 ] && printf "$([FAIL]) \n\tCould not install or update postgresql\n\n" && return 1
local default="Y"
((force_install)) && default="N"
local dosetup
if [ -x ${postgress_install_dir}/bin/psql ]; then
echo "Detected an existing postgress installation... (will not re-install)"
echo "To force a re-install you must manually remove the database completely - remove ${postgress_install_dir}/bin/psql"
echo 'Only if you know what you are doing!!!!!'
echo
read -e -p "Do you want to backup the curent database? $([ "$default" = "N" ] && echo "[y/N]" || echo "[Y/n]") " dosetup
dosetup=$(echo "${dosetup}" | tr [a-z] [A-Z])
[ -z ${dosetup} ] && dosetup=${default}
if [ "${dosetup}" = "Y" ] || [ "${dosetup}" = "YES" ]; then
backup_db
fi
echo
return 0
fi
if ((upgrade)); then
echo "UPGRADE of postgress completed"
echo "It is recommended to restart the entire node as to not leave"
echo "orphaned db connections that have been created by the rest of"
echo "the application stack"
echo "Restarting Database..."
stop_postgress
start_postgress
checked_done $?
fi
########
#Create the system account for postgress to run as.
########
local pg_sys_acct_homedir="/var/lib/pgsql"
id $pg_sys_acct
if [ $? != 0 ]; then
echo " Hmmm...: There is no postgres system account user \"$pg_sys_acct\" present on system, making one..."
#NOTE: "useradd/groupadd" are a RedHat/CentOS thing... to make this cross distro compatible clean this up.
if [ ! $(getent group ${pg_sys_acct_group}) ]; then
/usr/sbin/groupadd -r ${pg_sys_acct_group}
[ $? != 0 ] && [ $? != 9 ] && echo "ERROR: Could not add postgres system group: ${pg_sys_acct_group}" && popd && checked_done 1
fi
if [ -z "${pg_sys_acct_passwd}" ]; then
#set the password for the system user...
while [ 1 ]; do
local input
read -e -s -p "Create password for postgress system account: " input
[ -n "${input}" ] && pg_sys_acct_passwd=${input} && unset input && break
done
fi
echo "Creating account..."
/usr/sbin/useradd -r -c"PostgreSQL Service ESGF" -d $pg_sys_acct_homedir -g $pg_sys_acct_group -p $pg_sys_acct_passwd -s /bin/bash $pg_sys_acct
[ $? != 0 ] && [ $? != 9 ] && echo "ERROR: Could not add postgres system account user" && popd && checked_done 1
echo "${pg_sys_acct_passwd}" > ${pg_secret_file}
else
#check that this existing postgres account has a shell (required)
local postgress_user_shell="$(sed -n 's#\('${pg_sys_acct}'.*:\)\(.*\)$#\2#p' /etc/passwd)"
if [ "${postgress_user_shell}" != "/bin/bash" ]; then
echo "Noticed that the existing postgres user [${pg_sys_acct}] does not have the bash shell... Hmmm... making it so "
sed -i 's#\('${pg_sys_acct}'.*:\)\(.*\)$#\1/\bin/\bash#' /etc/passwd
echo "grep '${pg_sys_acct}': /etc/passwd"
[ "$(sed -n 's#\('${pg_sys_acct}'.*:\)\(.*\)$#\2#p' /etc/passwd)" = "/bin/bash" ] && [OK] || [FAIL]
fi
fi
[ -e "${pg_secret_file}" ] && chmod 640 ${pg_secret_file} && chown ${installer_uid}:${tomcat_group} ${pg_secret_file}
########
sleep 3
#double check that the account is really there!
echo
id $pg_sys_acct >& /dev/null
[ $? != 0 ] && grep $pg_sys_acct /etc/passwd && echo " ERROR: Problem with $pg_sys_acct creation!!!" && checked_done 1
chown -R $pg_sys_acct $postgress_install_dir
chgrp -R $pg_sys_acct_group $postgress_install_dir
#Create the database:
mkdir -p ${postgress_install_dir}/data
chown -R ${pg_sys_acct} ${postgress_install_dir}/data
[ $? != 0 ] && " ERROR: Could not change ownership of postgres' data to \"$pg_sys_acct\" user" && popd && checked_done 1
chmod 700 $postgress_install_dir/data
su $pg_sys_acct -c "$postgress_bin_dir/initdb -D $postgress_install_dir/data"
mkdir $postgress_install_dir/log
chown -R $pg_sys_acct $postgress_install_dir/log
[ $? != 0 ] && " ERROR: Could not change ownership of postgres' log to \"$pg_sys_acct\" user" && popd && checked_done 1
#Start the database
start_postgress
#Check to see if there is a ${postgress_user} already on the system if not, make one
if [ ! -x ${postgress_bin_dir}/psql ] ; then
echo " ERROR: psql not found after install!" && checked_done 1
fi
if (( $(PGPASSWORD=${pg_sys_acct_passwd:=${security_admin_password}} psql -U postgres -c "select count(*) from pg_roles where rolname='${postgress_user}'" postgres | tail -n +3 | head -n 1) > 0 )); then
echo "${postgress_user} exists!! :-)";
else
while [ 1 ]; do
while [ 1 ]; do
echo "Enter password for postgres user $postgress_user: ";
read -s p1;
echo "Re-enter password for postgres user $postgress_user: ";
read -s p2;
if [ "$p1" != "$p2" ]; then
echo "The passwords did not tally. Enter same password twice";
continue;
else
break;
fi
done
sudo -u $pg_sys_acct $postgress_bin_dir/psql -c "create user $postgress_user with superuser password '$p1';"
if [ $? -eq 0 ]; then popd && checked_done 0; break; fi
done
fi
#stop_postgress && return 1 #See trap in 'main'... that is who calls this.
local fetch_file
cd $postgress_install_dir/data
#Get files
fetch_file=pg_hba.conf
checked_get ./${fetch_file} ${esg_dist_url}/externals/bootstrap/${fetch_file} $((force_install))
(( $? > 1 )) && popd && checked_done 1
chmod 600 ${fetch_file}
#Get File...
fetch_file=postgresql.conf
checked_get ./${fetch_file} ${esg_dist_url}/externals/bootstrap/${fetch_file} $((force_install))
(( $? > 1 )) && popd && checked_done 1
chmod 600 ${fetch_file}
#-----
#NOTE: This database is an internal database to this esg
#application stack... I don't think it would even be prudent to
#offer then opportunity for someone to bind to the public
#interface. If they choose to do so after the fact, then they are
#making that conscious decision, but I won't make it a part of
#this process.
#@@postgress_host@@ #Token in file...
#local input
#read -e -p "Please Enter the IP address or name of this host [${postgress_host}]:> " input
#[ ! -z "${input}" ] && postgress_host=${input}
#printf "\nUsing IP: ${postgress_host}\n"
#eval "perl -p -i -e 's/\\@\\@postgress_host\\@\\@/${postgress_host}/g' ${fetch_file}"
#-----
#@@postgress_port@@ #Token in file...
unset input
read -e -p "Please Enter PostgreSQL port number [${postgress_port}]:> " input
[ ! -z "${input}" ] && postgress_port=${input}
printf "\nSetting Postgress Port: ${postgress_port} "
eval "perl -p -i -e 's/\\@\\@postgress_port\\@\\@/${postgress_port}/g' ${fetch_file}"
[ $? == 0 ] && [OK] || [FAIL]
printf "Setting Postgress Log Dir: ${postgress_install_dir} "
eval "perl -p -i -e 's#\\@\\@postgress_install_dir\\@\\@#${postgress_install_dir}#g' ${fetch_file}"
[ $? == 0 ] && [OK] || [FAIL]
chown -R $pg_sys_acct $postgress_install_dir
chgrp -R $pg_sys_acct_group $postgress_install_dir
popd >& /dev/null
echo
check_shmmax
echo
write_postgress_env
write_postgress_install_log
checked_done 0
}
write_postgress_env() {
((show_summary_latch++))
echo "export PGHOME=$PGHOME" >> ${envfile}
echo "export PGUSER=$PGUSER" >> ${envfile}
echo "export PGHOST=$PGHOST" >> ${envfile}
echo "export PGPORT=$PGPORT" >> ${envfile}
echo "export PGBINDIR=$PGBINDIR" >> ${envfile}
echo "export PGLIBDIR=$PGLIBDIR" >> ${envfile}
prefix_to_path PATH ${postgress_bin_dir} >> ${envfile}
prefix_to_path LD_LIBRARY_PATH ${postgress_lib_dir} >> ${envfile}
dedup ${envfile} && source ${envfile}
return 0
}
write_postgress_install_log() {
echo "$(date ${date_format}) postgres=${postgress_version} ${postgress_install_dir}" >> ${install_manifest}
dedup ${install_manifest}
return 0
}
#returns 1 if it is already running (if check_postgress_process returns 0 - true)
start_postgress() {
_is_managed_db && echo "Please be sure external database is running at this point..." && return 0
check_postgress_process && return 1
echo "Starting Postgress..."
/etc/init.d/postgresql start
check_shmmax
sleep 3
/bin/ps -elf | grep postgres | grep -v grep
checked_done 0
}
stop_postgress() {
_is_managed_db && echo "Please be sure external database is NOT running at this point..." && return 0
check_postgress_process
[ $? != 0 ] && return 1
/etc/init.d/postgresql stop
check_shmmax
/bin/ps -elf | grep postgres | grep -v grep
return 0
}
test_postgress() {
echo
echo "----------------------------"
echo "Postgress Test... "
echo "----------------------------"
echo
${postgress_bin_dir}/psql --version
[ $? != 0 ] && echo" ERROR: Could NOT successfully locate postgres installation!!" && popd >& /dev/null && checked_done 1
start_postgress
local ret=$(PGPASSWORD=${PGPASSWORD:-${pg_sys_acct_passwd}} psql -qt -c "select table_name from information_schema.tables;" postgres ${postgress_user} | grep -v ^$ | wc -l)
((ret == 0)) && echo " ERROR: Could not verify database installation! (perhaps \"pg_sys_acct_passwd\" was not set correctly?)" && checked_done 1
[OK]
checked_done 0
}
#Needed for UV-CDAT build...
setup_cmake() {
echo -n "Checking for CMake >= ${cmake_min_version} "
check_version_with cmake "cmake --version | awk '{print \$3}' | sed -re 's/([^-]*)-.*/\1/'" ${cmake_min_version} ${cmake_max_version}
[ $? == 0 ] && (( ! force_install )) && [OK] && return 0
echo
echo "*******************************"
echo "Setting up CMake ${cmake_version}"
echo "*******************************"
echo
local default="Y"
((force_install)) && default="N"
local dosetup
if [ -x ${cmake_install_dir}/bin/cmake ]; then
echo "Detected an existing CMAKE installation..."
read -e -p "Do you want to continue with CMAKE installation and setup? $([ "$default" = "N" ] && echo "[y/N]" || echo "[Y/n]") " dosetup
[ -z ${dosetup} ] && dosetup=${default}
if [ "${dosetup}" != "Y" ] && [ "${dosetup}" != "y" ]; then
echo "Skipping CMAKE installation and setup - will assume CMAKE is setup properly"
return 0
fi
echo
fi
#make top level directory for cmake repo clone
mkdir -p ${cmake_workdir%/*}
chmod a+rw ${cmake_workdir%/*}
if [ ! -d "${cmake_workdir}" ]; then
echo "Cloning CMake repository ${cmake_repo}..."
git clone ${cmake_repo} ${cmake_workdir}
[ $? != 0 ] && echo "ERROR: Could not clone CMake repository" && checked_done 1
fi
(
unset LD_LIBRARY_PATH
unset CFLAGS
unset LDFLAGS
((DEBUG)) && printf "\n-----\n cd ${cmake_workdir} \n-----\n"
cd ${cmake_workdir}
((DEBUG)) && printf "\n-----\n git checkout v${cmake_version} \n-----\n"
git checkout v${cmake_version}
[ $? != 0 ] && echo "ERROR: Could not checkout CMake @ v${cmake_version}" && checked_done 2
((DEBUG)) && printf "\n-----\n ./configure --parallel=${num_cpus} --prefix=${cmake_install_dir} \n-----\n"
./configure --parallel=${num_cpus} --prefix=${cmake_install_dir}
[ $? != 0 ] && echo "ERROR: Could not configure CMake successfully" && checked_done 3
((DEBUG)) && printf "\n-----\n make -j ${num_cpus} \n-----\n"
make -j ${num_cpus}
[ $? != 0 ] && echo "ERROR: Could not make CMake successfully" && checked_done 4
((DEBUG)) && printf "\n-----\n make install \n-----\n"
make install
[ $? != 0 ] && echo "ERROR: Could not install CMake successfully" && checked_done 5
)
echo "returning from build subshell with code: [$?]"
(( $? > 1 )) && echo "ERROR: Could not setup CMake successfully aborting... " && checked_done 1
cmake_version=$(${cmake_install_dir}/bin/cmake --version | awk '{print $3}' | sed -re 's/([^-]*)-.*/\1/')
printf "\ninstalled CMake version = ${cmake_version}\n\n"
write_cmake_env
write_cmake_install_log
checked_done 0
}
write_cmake_env() {
((show_summary_latch++))
echo "export CMAKE_HOME=${cmake_install_dir}" >> ${envfile}
prefix_to_path PATH ${cmake_install_dir}/bin >> ${envfile}
dedup ${envfile} && source ${envfile}
return 0
}
write_cmake_install_log() {
echo "$(date ${date_format}) cmake=${cmake_version} ${cmake_install_dir}" >> ${install_manifest}
dedup ${install_manifest}
return 0
}
#####
# CDAT = Python+CDMS
#####
setup_cdat() {
echo -n "Checking for *UV* CDAT (Python+CDMS) ${cdat_version} "
#-----------------------------------------------------
cdat_home=$(perl -pe 's/(?<!uv)cdat/uvcdat/' <<<${cdat_home})
CDAT_HOME=${cdat_home}
#-----------------------------------------------------
check_version_with "cdat" " source $cdat_home/bin/activate esgf-pub && python -c \"import cdat_info; print cdat_info.Version\" && source deactivate" ${cdat_version} 2> /dev/null
local ret=$?
((ret == 0)) && (( ! force_install )) && [OK] && return 0
echo
echo "*******************************"
echo "Setting up CDAT - (Python + CDMS)... ${cdat_version}"
echo "*******************************"
echo
local dosetup="N"
# TODO - remove old installation
# upgrate existing installation
if [ -x ${cdat_home}/bin/python ]; then
echo "Detected an existing CDAT installation..."
read -e -p "Do you want to continue with CDAT installation and setup? [y/N] " dosetup
if [ "${dosetup}" != "Y" ] && [ "${dosetup}" != "y" ]; then
echo "Skipping CDAT installation and setup - will assume CDAT is setup properly"
return 0
fi
echo
fi
mkdir -p ${workdir}
[ $? != 0 ] && checked_done 1
pushd ${workdir} >& /dev/null
#---------
# Under FORCE (--force) semantics we clean everything out: the cloned repo and build dir
#((force_install)) && [ -d ${workdir}/uvcdat ] && \rm -rf ${workdir}/uvcdat && \rm -rf ${workdir}/uvcdat_build
#---------
# TODO check for conda
if [ ! -f Miniconda2-latest-Linux-x86_64.sh ] ; then
wget --no-check-certificate https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh
[ $? != 0 ] && printf "$([FAIL]) \n\tCould not fetch miniconda setup\n\n" && checked_done 1
fi
bash Miniconda2-latest-Linux-x86_64.sh -b -p $cdat_home
[ $? != 0 ] && printf "$([FAIL]) \n\tError in executing Miniconda setup\n\n" && checked_done 1
export PATH=${cdat_home}/bin:$PATH
if [ $ESGF_INSECURE > 0 ] ; then
conda config --set ssl_verify False
# binstar config --set verify_ssl False
fi
# create a default environment for the publisher with cduitl (and cdms2)
if [ ! -d ${cdat_home}/envs/esgf-pub ] ; then
unset LD_LIBRARY_PATH
conda create -y -n esgf-pub -c conda-forge -c uvcdat cdutil
[ $? != 0 ] && printf "$([FAIL]) \n\tCould not install or update uvcdat via conda\n\n" && checked_done 1
fi
source activate esgf-pub
python -c "import cdms2" 2>/dev/null
[ $? != 0 ] && echo " ERROR: Could not load CDMS (cdms2) module" && popd && checked_done 1
source deactivate
popd >& /dev/null
echo
#NOTE: order is important in next three calls...
# _housekeeping_cdat_to_uvcdat
write_cdat_env
#----------------
# write_cdat_install_log
checked_done 0
}
write_cdat_env() {
((show_summary_latch++))
echo "export CDAT_HOME=${cdat_home}" >> ${envfile}
echo "export UVCDAT_ANONYMOUS_LOG=False" >> ${envfile}
prefix_to_path PATH ${cdat_home}/bin >> ${envfile}
prefix_to_path LD_LIBRARY_PATH ${cdat_home}/Externals/lib >> ${envfile}
dedup ${envfile} && source ${envfile}
return 0
}
write_cdat_install_log() {
echo "$(date ${date_format}) uvcdat=${cdat_version} ${cdat_home}" >> ${install_manifest}
#Parse the cdat installation config.log file and entries to the install log
local build_log=${workdir}/uvcdat_build/build_info.txt
if [ -e "${build_log}" ]; then
awk '{print "'"$(date ${date_format})"' uvcdat->"$1"="$2" '"${cdat_home}"'"}' ${build_log} | sed '$d' >> ${install_manifest}
else
echo " WARNING: Could not find cdat build logfile [${build_log}], installation log entries could not be generated!"
fi
dedup ${install_manifest}
return 0
}
test_cdat() {
echo -n "Checking for cdms2 module... "
python -c "import cdms2" >& /dev/null && [OK] || [FAIL]
}
#------------------------------------------------------------
#Special function for the upgrade scenario (cdat -> uvcdat)
#------------------------------------------------------------#
_housekeeping_cdat_to_uvcdat() {
#Determine the install directory for cdat from the install_manifest
#(NOTE: Whatever the resulting path is, it must contain the string "cdat")
local cdat_installed_dir=$(sed -nre 's/.*[^v]cdat=([^ ]*)([^ ]*.*cdat)/\2/p' ${install_manifest} | tr -d " ")
[[ ${cdat_installed_dir} ]] && backup ${cdat_installed_dir} && rm -rf ${cdat_installed_dir}
#Remove all cdat install manifest entries from install manifest (NOTE: Not touching new uvcdat entries)
sed -ire '/[^v]cdat/d' ${install_manifest}
#reset library path to point to new uvcdat location
myLD_LIBRARY_PATH=$(perl -pe 's/(?<!uv)cdat/uvcdat/' <<<${myLD_LIBRARY_PATH})
export LD_LIBRARY_PATH=$(_path_unique $myLD_LIBRARY_PATH:$LD_LIBRARY_PATH)
dedup ${envfile} && source ${envfile}
}
#####
# ESGCET Package
#####
setup_esgcet() {
echo -n "Checking for esgcet (publisher) ${esgcet_version} "
check_module_version esgcet ${esgcet_version}
local ret=$?
#-------
if [ -e "${ESGINI}" ]; then
local urls_mis_match=1
local esgini_dburl=$(sed -n 's@^[^#]*[ ]*dburl[ ]*=[ ]*\(.*\)$@\1@p' ${ESGINI} | head -n1 | sed 's@\r@@')
local db_connect_str="postgresql://${publisher_db_user}:${publisher_db_user_passwd}@$([ -n "${db_host}" ] && ([ "${db_host}" = "${esgf_host}" ] || [ "${db_host}" = "localhost" ]) && echo "localhost" || echo ${db_host}):${db_port}/${db_database}"
debug_print "[${esgini_dburl}] =? [${db_connect_str}]"
if [ "${esgini_dburl}" != "${db_connect_str}" ]; then
printf "\n [ERROR]: publisher database connection string mis-match... (inspecting ${ESGINI}'s \"dburl\" value) \n"
printf " Attempting to address the issue... "
if [ "$(sed -e 's#\(postgresql://.*@\)\([^:]*\)\(.*\)#\1localhost\3#' <<< ${esgini_dburl})" != "${db_connect_str}" ]; then
trap - INT TERM
printf " $(echo_fail "Sorry, [FAILED]")\n"
exit 1
fi
printf "Editing esg.ini accordingly... "
sed -i 's#^[ ]*\(dburl[ ]*=[ ]*\).*$#\1'${db_connect_str}'#' ${ESGINI} && echo -n ":-) " && urls_mis_match=0 || echo -n " 8-( "
else
urls_mis_match=0
fi
((urls_mis_match)) && exit 1