#! /bin/bash

# This script installs lammps automatically in a directory /share/apps/lammps-16Mar18_gnu
# Execute as su

### In jaws (CentOS Linux release 7.5.1804 ), it is found that when the following requirements are fulfilled, lammps-16Mar18 can be compiled properly with gpu. These include:

#### Mandatory requirements: 

##1. /share/apps/openmpi-2.1.1/gnu/lib/libmpi.so, /share/apps/openmpi-2.1.1/gnu/include/mpi.h

##2. libfftw3.a   libfftw3_mpi.a   libfftw3_omp.a   libfftw3_threads.a in /share/apps/fftw-3.3.6/gnu/lib/ 

##3. libatlas.a  libcblas.a  libf77blas.a  liblapack.a  libptcblas.a  libptf77blas.a in /share/apps/atlas-3-10-2/gnu/lib/

##4. /usr/local/cuda-9.2

### It was found that openmpi-3.1.2 fails to produce a gpu-enabled lammps-16Mar18 installation using all-gnu compilers


### begin installation 

mkdir /share
mkdir /share/apps
mkdir /share/apps/configrepo

rootdir=lammps-16Mar18_gnu
src_lammps=lammps-16Mar18

#2. download stable.zip
if [ -e /share/apps/configrepo/$src_lammps/stable.zip ]; then
        filesize=$(ls -la /share/apps/configrepo/$src_lammps/stable.zip | awk '{print $5}')
        if [ "$filesize" -eq 98837930 ]; then
                echo 'found stable.zip with the right size =' $filesize'. Not to wget it from the cloud'
else
                echo 'found stable.zip with the wrong size $filesize = ' $filesize'. Will remove existing (but wrong size) stable.zip and wget it from the cloud'
               	rm -rf /share/apps/configrepo/$src_lammps/stable.zip
                echo 'initiate download:'
				cd /share/apps/configrepo/$src_lammps
               	wget https://www.dropbox.com/s/5vpk14vj7qg0ie8/stable.zip?dl=0 -O stable.zip
        fi
else
	echo 'stable.zip not found'
	echo 'to initiate download'
	mkdir /share/apps/configrepo/$src_lammps
	cd /share/apps/configrepo/$src_lammps
	wget https://www.dropbox.com/s/5vpk14vj7qg0ie8/stable.zip?dl=0 -O stable.zip
fi #end if


#3. 
cd /share/apps
unzip /share/apps/configrepo/$src_lammps/stable.zip
mv lammps-stable $rootdir
cd $rootdir

#4. 
cd lib/atc
## !!!!!!!!!!!!!!!!!!!! to modify Makefile.mpic++ ###
	filetomod=Makefile.mpic++
	make -f $filetomod clean
	keyword=mpicxx
	# backup orig $filetomod copy
	cp $filetomod $filetomod'.orig'
	### target line number to be replaced
	#lineno=$(cat $filetomod | awk -v keyword=$keyword '/keyword/ {print NR}')
	lineno=$(cat $filetomod | awk -v keyword=$keyword '$0 ~ keyword {print NR}')
	#echo $lineno
	### end of target line number to be replaced
	#### original target string 
	os=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print}')
	#echo $os 
	#### end of original target string 
	#### modified string 
	s2a='CC = /share/apps/openmpi-2.1.1/gnu/bin/mpicxx'
	ns=$s2a
	#### end of modified string 
	echo To replace 
	echo $os 
	echo 'in line' "$lineno" 'in' $filetomod
	echo by 
	echo $ns 
	cat $filetomod | awk -v lineno=$lineno -v ns="$ns" 'NR==lineno {$0=ns} {print}' > output.txt
	mv output.txt $filetomod
## !!!!!!!!!!!!!!!!!!!! to modify Makefile.mpic++ ###


##!!!! To modify Makefile.lammps.installed in atc so that it is pointing to tunned atlast library library !!!!##

cp Makefile.lammps.installed Makefile.lammps.installed.orig

################# create outfile.txt #################
line1='user-atc_SYSINC = -I/share/apps/atlas-3-10-2/gnu/include/atlas -I/share/apps/atlas-3-10-2/gnu/include'

line2='user-atc_SYSLIB = -L/share/apps/atlas-3-10-2/gnu/lib -lf77blas  -lptcblas -lcblas  -llapack -lptf77blas -latlas'

line3='user-atc_SYSPATH ='
cat <<EOF >outfile.txt
$line1
$line2
$line3
EOF
################# create outfile.txt #################
mv outfile.txt Makefile.lammps.installed

echo 'In atc, I need to generate Makefile.lammps with the content of  Makefile.lammps.installed via  Makefile.lammps.linalg'

cp  Makefile.lammps.linalg  Makefile.lammps.linalg.orig
cp Makefile.lammps.installed Makefile.lammps.linalg

echo 'Makefile.lammps in' $PWD 'should be now modified to point to tunned atlas libraries'

make -f $filetomod



#5. 
cd ../awpmd

## !!!!!!!!!!!!!!!!!!!! to modify Makefile.mpicc ###
	filetomod=Makefile.mpicc
	make -f $filetomod clean
	keyword=mpicxx
	# backup orig $filetomod copy
	cp $filetomod $filetomod'.orig'
	### target line number to be replaced
	lineno=$(cat $filetomod | awk -v keyword=$keyword '$0 ~ keyword {print NR}')
	#echo $lineno
	### end of target line number to be replaced
	#### original target string 
	os=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print}')
	#echo $os 
	#### end of original target string 
	#### modified string 
	s2a='CC = /share/apps/openmpi-2.1.1/gnu/bin/mpicxx'
	ns=$s2a
	#### end of modified string 
	echo To replace 
	echo $os 
	echo 'in line' "$lineno" 'in' $filetomod
	echo by 
	echo $ns 
	cat $filetomod | awk -v lineno=$lineno -v ns="$ns" 'NR==lineno {$0=ns} {print}' > output.txt
	mv output.txt $filetomod
## !!!!!!!!!!!!!!!!!!!! to modify Makefile.mpicc ###


##!!!! To modify Makefile.lammps.installed in awpmd so that it is pointing to tunned atlast library !!!!##

cp Makefile.lammps.installed Makefile.lammps.installed.orig

#################
line1='user-awpmd_SYSINC = -I/share/apps/atlas-3-10-2/gnu/include/atlas -I/share/apps/atlas-3-10-2/gnu/include'

line2='user-awpmd_SYSLIB = -L/share/apps/atlas-3-10-2/gnu/lib -lf77blas  -lptcblas -lcblas  -llapack -lptf77blas -latlas'

line3='user-awpmd_SYSPATH ='

cat <<EOF >outfile.txt
$line1
$line2
$line3
EOF
#################
#cat <<EOF >outfile.txt

mv outfile.txt Makefile.lammps.installed
echo 'Makefile.lammps.installed in' $PWD modified to point to tunned atlas library

##!!!! To modify Makefile.lammps.installed in awpmd so that it is pointing to tunned atlast library !!!!##

make -f $filetomod



#6. linalg #
cd ../linalg
	filetomod=Makefile.mpi
	make -f $filetomod clean
	make -f Makefile.gfortran


#7. meam #
cd ../meam
filetomod=Makefile.gfortran
make -f $filetomod clean
make -f $filetomod



#8. poems #
cd ../poems

## !!!!!!!!!!!!!!!!!!!! to modify Makefile.mpi ###
	filetomod=Makefile.mpi
	make -f $filetomod clean
	cp $filetomod $filetomod'.orig'
	keyword='mpicxx'
	s2a='/share/apps/openmpi-2.1.1/gnu/bin/mpicxx'
	sed -i "s|$keyword|$s2a|g" $filetomod
####
	make -f $filetomod



#9. reax #
cd ../reax
filetomod=Makefile.gfortran
make -f $filetomod clean
make -f $filetomod


# 10. qmmm  #### has to be linked against QE. Modify $filetomod so that $QETOPDIR is properly linked
	cd ../qmmm
	filetomod=Makefile.mpi
	cp $filetomod $filetomod'.orig'
	keyword='mpicxx'
	s2a='/share/apps/openmpi-2.1.1/gnu/bin/mpicxx'
	sed -i "s|$keyword|$s2a|g" $filetomod

	keyword='QETOPDIR=$(HOME)/compile/espresso'
	s2a='QETOPDIR=/share/apps/qe-6.3'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	make -f $filetomod


#11. colvars #
cd ../colvars
	filetomod=Makefile.mpi
	cp $filetomod $filetomod'.orig'
	keyword='mpicxx'
	s2a='/share/apps/openmpi-2.1.1/gnu/bin/mpicxx'
	sed -i "s|$keyword|$s2a|g" $filetomod
	make -f $filetomod


#12. src #
cd ../../src

make yes-all
make no-kokkos
make no-user-netcdf
make no-user-quip
make no-voronoi
make no-user-h5md
make no-kim
make no-latte
make no-mscg
make no-user-smd
make no-user-vtk
make no-gpu


#13. make gpu #
### Must tweak -arch=sm_xx so that it suits your GPU architecture
### At times certain cuda version works but not for others. May have to do some trial-and-error which cuda version will work properly with which version of lammps.
### Assumes cuda has been intalled by default /usr/local/cuda and the links are properly sourced in .bashrc
### For jaws, GTX 570 Ti is a Maxwell, hence sm_50
### For TITAN X, Titan Xp, GTX 1060, computability is 6.1, hence sm_61
# Refer to https://lammps.sandia.gov/doc/Build_extras.html, or better still,
# https://en.wikipedia.org/wiki/CUDA#GPUs_supported
# to set the value for -arch=sm_xx
#    sm_12 or sm_13 for GT200 (supported by CUDA 3.2 until CUDA 6.5)
#    sm_20 or sm_21 for Fermi (supported by CUDA 3.2 until CUDA 7.5)
#    sm_30 or sm_35 or sm_37 for Kepler (supported since CUDA 5)
#    sm_50 or sm_52 for Maxwell (supported since CUDA 6)
#    sm_60 or sm_61 for Pascal (supported since CUDA 8)
#    sm_70 for Volta (supported since CUDA 9)
#    sm_75 for Turing (supported since CUDA 10)

# Successful gpu-enabled installation:
# 1. For the GeForce 940M gpu (Maxwell architecture) in my laptop, lib/gpu is compiled using 
# mixed precsion, CUDA_PRECISION = -D_SINGLE_DOUBLE, CUDA_ARCH = -arch=sm_50, lammps version of 5May2018, compiled fully using intel compiler (parallel studio 2017). cuda-8.0 was used by setting CUDA_HOME = /usr/local/cuda-8.0 in Makefile.linux.
# 2. For the GTX 1060 gpu (Pascal architecture) in my server, lib/gpu is compiled using mixed precsion, CUDA_PRECISION = -D_SINGLE_DOUBLE, CUDA_ARCH = -arch=sm_60, lammps version of 5May2018, compiled fully using intel compiler (parallel studio 2017). cuda-9.2 was used (cuda-9.0 did not work) by setting CUDA_HOME = /usr/local/cuda-9.2 in Makefile.linux.

### check if nvidia-smi is working. If yes, make yes-gpu; else make no-gpu

filetomod=Makefile.linux
command=nvidia-smi
$command > /dev/null 2>&1
if [ $? -eq 0 ]; then
    echo $command found. CUDA driver and toolkit properly installed
    echo To install gpu library in lammps
    cd ../lib/gpu
    cp $filetomod $filetomod'.orig'
    keyword='mpicxx'
	s2a='/share/apps/openmpi-2.1.1/gnu/bin/mpicxx'
	#s2a='/share/apps/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpicxx'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	keyword='/usr/local/cuda'
	s2a='/usr/local/cuda-9.2'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	ca=$(/usr/local/cuda/samples/bin/x86_64/linux/release/deviceQuery | grep  'CUDA Capability' | awk 'END {print $6}' | awk -F  "." '{print $1$2}')
	echo compute capability given by deviceQuery is $ca
	echo 'CUDA_ARCH = -arch=sm_'$ca >> $filetomod
    make -f $filetomod clean
	make -f $filetomod
	if [ -e libgpu.a ]; then
		echo 'libgpu.a found. To make yes-gpu in ../../src'
		cd ../../src
		make no-gpu
		make yes-gpu
	else
		echo libgpu.a not found after make -f Makefile.linux. Not to install gpu in ../../src
		cd ../../src
		make no-gpu
	fi 
else
    echo no $command found. CUDA driver or toolkit not properly installed
    echo Not to install gpu library in lammps
fi
### check if nvidia-smi is working. If yes, make yes-gpu; else make no-gpu


#14. To modify Makefile.xx in MAKE/OPTIONS and copy it to MAKE

	filetomod=Makefile.gpu
	cd MAKE
	cp OPTIONS/$filetomod .
	
	cp $filetomod $filetomod'.orig'
	keyword='mpicxx'
	s2a='/share/apps/openmpi-2.1.1/gnu/bin/mpicxx'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	keyword='MPI_INC =       -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1'
	s2a='MPI_INC = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1  -I/share/apps/openmpi-2.1.1/gnu/include'
	sed -i "s|$keyword|$s2a|g" $filetomod

	keyword='MPI_PATH ='
	s2a='MPI_PATH = -L/share/apps/openmpi-2.1.1/gnu/lib'
	sed -i "s|$keyword|$s2a|g" $filetomod

	keyword='MPI_LIB ='
	s2a='MPI_LIB = -lmpi'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	keyword='FFT_INC ='
	s2a='FFT_INC = -I/share/apps/fftw-3.3.6/gnu/include'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	keyword='FFT_LIB ='
	s2a='FFT_LIB = -L/share/apps/fftw-3.3.6/gnu/lib -lfftw3 -lfftw3_mpi -lfftw3_omp -lfftw3_threads'
	sed -i "s|$keyword|$s2a|g" $filetomod

## end To modify Makefile.xx in MAKE/OPTIONS and copy it to MAKE

#15. 
cd ../
fn=$(echo $filetomod | awk -F  "." '{print $2}')
cp MAKE/Makefile.$fn MAKE/Makefile.ggpu    ## ggpu stands for gpu-enabled lammps, compiled using gnu compilers

make ggpu
ln -s $PWD/lmp_ggpu /share/apps/local/bin