#! /bin/bash

# This script installs lammps automatically in a directory /share/apps/$rootdir
# where $rootdir=lammps_16Mar18_intel
# Execute as su

##1.
#### Mandatory requirements: 
#### /share/apps/configrepo exists
#### intel parallel_studio_xe_2017 has been installed in /share/apps/intel.
#### libfftw3xf_intel.a has been installed in $MKLROOT/lib/intel64
#### libz-dev is installed. Else install it using either apt-get or yum install libz-dev
#### intel mpi are found in 
#### /share/apps/intel/compilers_and_libraries/linux/mpi/include64
#### /share/apps/intel/compilers_and_libraries/linux/mpi/lib64
#### /share/apps/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpiicpc

#### definition of variables used ################
	mpiincpath=/share/apps/intel/compilers_and_libraries/linux/mpi/include64
	mpilibpath=/share/apps/intel/compilers_and_libraries/linux/mpi/lib64
	rootdir=lammps-16Mar18_intel

##2.
###### automatically sourcing intel parallel studio ##########################
######  assures that intel library is found and properly sourced before proceeding with the installation

source /share/apps/intel/compilers_and_libraries/linux/bin/compilervars.sh intel64
source /share/apps/intel/compilers_and_libraries/linux/mkl/bin/mklvars.sh intel64
source /share/apps/intel/compilers_and_libraries/linux/mpi/bin64/mpivars.sh intel64
mklroot=$(echo $MKLROOT)

if [ -z "$mklroot" ]; then
        echo '$MKLROOT empty. Installtion will abort. Please assure that the PATH to $MKLROOT is define before retrying.'
        exit 1
else
        echo '$MKLROOT = '$MKLROOT non-empty. Installtion will proceed.
fi #end if


#4. 
cd /share/apps/configrepo
src_lammps=lammps-16Mar18
mkdir $src_lammps
cd $src_lammps
if [ -e stable.zip ]; then
        filesize=$(ls -la stable.zip | awk '{print $5}')
        if [ "$filesize" -eq 98837930 ]; then
                echo 'found stable.zip with the right size =' $filesize'. Not to wget it from the cloud'
		else
                echo 'found stable.zip with the wrong size $filesize = ' $filesize'. Will remove existing (but wrong size) stable.zip and wget it from the cloud'
               	rm -rf stable.zip
                echo 'initiate download:'
               	wget https://www.dropbox.com/s/5vpk14vj7qg0ie8/stable.zip?dl=0 -O stable.zip
        fi
else
	echo 'stable.zip not found'
	wget https://www.dropbox.com/s/5vpk14vj7qg0ie8/stable.zip?dl=0 -O stable.zip
	echo 'to initiate download'
fi #end if


#5. 
cd /share/apps
unzip configrepo/$src_lammps/stable.zip
mv lammps-stable $rootdir
cd $rootdir

##%%%% #6. 
##%%%% cd lib/atc
##%%%% ## !!!!!!!!!!!!!!!!!!!! to modify Makefile.icc
##%%%%         filetomod=Makefile.icc
##%%%%         make -f $filetomod clean
##%%%%  cp $filetomod $filetomod'.orig'
##%%%%  keyword='icc'
##%%%% s2a='/share/apps/intel/compilers_and_libraries/linux/bin/intel64/icc'
##%%%% sed -i "s|$keyword|$s2a|g" $filetomod
##%%%% ## end of !!!!!!!!!!!!!!!!!!!! to modify Makefile.icc
##%%%% 
##%%%% ## To modify Makefile.lammps.installed so that it is pointing to mkl library
##%%%% cp Makefile.lammps.installed Makefile.lammps.installed.orig
##%%%% 
##%%%% line1='user-atc_SYSINC = -I/share/apps/intel/mkl/include'
##%%%% line2='user-atc_SYSLIB = -L/share/apps/intel/mkl/lib/intel64 #-lblas -llapack'
##%%%% line3='user-atc_SYSPATH = -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core ## -lmkl_lapack95_lp64'
##%%%% cat <<EOF >outfile.txt
##%%%% $line1
##%%%% $line2
##%%%% $line3
##%%%% EOF
##%%%% #################
##%%%% mv outfile.txt Makefile.lammps.installed
##%%%% echo 'Makefile.lammps.installed in' $PWD modified to point to mkl library
##%%%% ## end of To modify Makefile.lammps.installed so that it is pointing to mkl library
##%%%% make -f $filetomod clean
##%%%% make -f $filetomod
##%%%% 
##%%%% 


#6. 
cd lib/atc
## !!!!!!!!!!!!!!!!!!!! to modify  Makefile.mpic++
        filetomod=Makefile.mpic++
        make -f $filetomod clean
        cp $filetomod $filetomod'.orig'
        keyword='mpicxx'
        s2a='/share/apps/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpicxx'
        sed -i "s|$keyword|$s2a|g" $filetomod
## end of !!!!!!!!!!!!!!!!!!!! to modify Makefile.icc
make -f $filetomod


#7. 
cd ../awpmd
## To modify Makefile.lammps.installed so that it is pointing to mkl library
cp Makefile.lammps.installed Makefile.lammps.installed.orig

#################
line1='user-awpmd_SYSINC = -I$(MKLROOT)/include'
line2='user-awpmd_SYSPATH = -L$(MKLROOT)/lib/intel64'
line3='user-awpmd_SYSLIB = -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core ## -lmkl_lapack95_lp64'
cat <<EOF >outfile.txt
$line1
$line2
$line3
EOF
#################

mv outfile.txt Makefile.lammps.installed
echo 'Makefile.lammps.installed in' $PWD modified to point to mkl library
## end of To modify Makefile.lammps.installed so that it is pointing to mkl library

make -f Makefile.mpicc clean
make -f Makefile.mpicc


#8. linalg #
cd ../linalg
## To modify Makefile.mpi so that it is using mpiifort
	filetomod=Makefile.mpi
	keyword=mpifort
## backup orig $filetomod copy
	cp $filetomod $filetomod'.orig'

### target line number to be replaced
	lineno=$(cat $filetomod | awk -v keyword=$keyword '$0 ~ keyword {print NR}')

#### original target string 
	os=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print}')
#echo $os 
#### end of original target string 

#### modified string 
	s2a='FC = mpiifort'
	ns=$s2a
#### end of modified string 
	echo To replace 
	echo $os 
	echo 'in line' "$lineno" 'in' $filetomod
	echo by 
	echo $ns 
	cat $filetomod | awk -v lineno=$lineno -v ns="$ns" 'NR==lineno {$0=ns} {print}' > output.txt
	mv output.txt $filetomod
## end of To modify Makefile.mpi so that it is using mpiifort 
make -f Makefile.mpi clean
make -f Makefile.mpi


#9. meam #
cd ../meam
## To modify Makefile.lammps.ifort so that it is pointing to the right mkl library
targetf=Makefile.lammps.ifort
cp $targetf $targetf'.orig'

#################
line1='meam_SYSINC = -I$(MKLROOT)/include'
line2='meam_SYSLIB = -lifcore -lsvml -liompstubs5 -limf -lintlc #-lompstub'
line3='meam_SYSPATH = -L$(MKLROOT)/lib/intel64 -L/share/apps/intel/lib/intel64'
cat <<EOF >outfile.txt
$line1
$line2
$line3
EOF
#################
mv outfile.txt $targetf
echo $targetf 'in' $PWD is modified to point to mkl library
## end of To modify Makefile.lammps.ifort so that it is pointing to the right mkl library
make -f Makefile.ifort clean
make -f Makefile.ifort


#10. poems #
cd ../poems
make -f Makefile.icc clean
make -f Makefile.icc


#11. reax #
cd ../reax
make -f Makefile.ifort clean
make -f Makefile.ifort 

# 12. qmmm  #### has to be linked against QE. Modify Makefile.ifort so that $QETOPDIR is properly linked
cd ../qmmm
make -f Makefile.ifort clean
make -f Makefile.ifort


#13. colvars #
cd ../colvars
make -f Makefile.mpi clean
make -f Makefile.mpi

#14. src #
cd ../../src

make yes-all
make no-kokkos
make no-user-netcdf
make no-user-quip
make no-voronoi
make no-user-h5md
make no-kim
make no-latte
make no-mscg
make no-user-smd
make no-user-vtk
make no-gpu


#15. make gpu #
### Must tweak -arch=sm_xx so that it suits your GPU architecture
### At times certain cuda version works but not for others. May have to do some trial-and-error which cuda version will work properly with which version of lammps.
### Assumes cuda has been intalled by default /usr/local/cuda and the links are properly sourced in .bashrc
### For jaws, GTX 570 Ti is a Maxwell, hence sm_50
### For TITAN X, Titan Xp, GTX 1060, computability is 6.1, hence sm_61
# Refer to https://lammps.sandia.gov/doc/Build_extras.html, or better still,
# https://en.wikipedia.org/wiki/CUDA#GPUs_supported
# to set the value for -arch=sm_xx
#    sm_12 or sm_13 for GT200 (supported by CUDA 3.2 until CUDA 6.5)
#    sm_20 or sm_21 for Fermi (supported by CUDA 3.2 until CUDA 7.5)
#    sm_30 or sm_35 or sm_37 for Kepler (supported since CUDA 5)
#    sm_50 or sm_52 for Maxwell (supported since CUDA 6)
#    sm_60 or sm_61 for Pascal (supported since CUDA 8)
#    sm_70 for Volta (supported since CUDA 9)
#    sm_75 for Turing (supported since CUDA 10)

# Successful gpu-enabled installation:
# 1. For the GeForce 940M gpu (Maxwell architecture) in my laptop, lib/gpu is compiled using 
# mixed precsion, CUDA_PRECISION = -D_SINGLE_DOUBLE, CUDA_ARCH = -arch=sm_50, lammps version of 5May2018, compiled fully using intel compiler (parallel studio 2017). cuda-8.0 was used by setting CUDA_HOME = /usr/local/cuda-8.0 in Makefile.linux.
# 2. For the GTX 1060 gpu (Pascal architecture) in my server, lib/gpu is compiled using mixed precsion, CUDA_PRECISION = -D_SINGLE_DOUBLE, CUDA_ARCH = -arch=sm_60, lammps version of 5May2018, compiled fully using intel compiler (parallel studio 2017). cuda-9.2 was used (cuda-9.0 did not work) by setting CUDA_HOME = /usr/local/cuda-9.2 in Makefile.linux.

### check if nvidia-smi is working. If yes, make yes-gpu; else make no-gpu

filetomod=Makefile.linux
command=nvidia-smi
$command > /dev/null 2>&1
if [ $? -eq 0 ]; then
    echo $command found. CUDA driver and toolkit properly installed
    echo To install gpu library in lammps
    cd ../lib/gpu
    cp $filetomod $filetomod'.orig'
    keyword='mpicxx'
	#s2a='/share/apps/openmpi-3.1.2/gnu/bin/mpicxx'
	s2a='/share/apps/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpicxx'
	sed -i "s|$keyword|$s2a|g" $filetomod
	ca=$(/usr/local/cuda/samples/bin/x86_64/linux/release/deviceQuery | grep  'CUDA Capability' | awk 'END {print $6}' | awk -F  "." '{print $1$2}')
	echo compute capability given by deviceQuery is $ca
	echo 'CUDA_ARCH = -arch=sm_'$ca >> $filetomod
    make -f $filetomod clean
	make -f $filetomod
	if [ -e libgpu.a ]; then
		echo 'libgpu.a found. To make yes-gpu in ../../src'
		cd ../../src
		make no-gpu
		make yes-gpu
	else
		echo libgpu.a not found after make -f Makefile.linux. Not to install gpu in ../../src
		cd ../../src
		make no-gpu
	fi 
else
    echo no $command found. CUDA driver or toolkit not properly installed
    echo Not to install gpu library in lammps
fi
### check if nvidia-smi is working. If yes, make yes-gpu; else make no-gpu


#16. To modify Makefile.xx in MAKE/OPTIONS and copy it to MAKE

	filetomod=Makefile.intel_cpu_intelmpi
	cd MAKE
	cp OPTIONS/$filetomod .

## 0
        keyword='mpiicpc'
        s2a='/share/apps/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpiicpc'
        sed -i "s|$keyword|$s2a|g" $filetomod

## 1
	keyword='-DMPICH_SKIP_MPICXX'
##      backup orig $filetomod copy
	cp $filetomod $filetomod'.orig'

### target line number to be replaced
	lineno=$(cat $filetomod | awk -v keyword=$keyword '$0 ~ keyword {print NR}')
#	echo $lineno
#### original target string 
	os=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print}')
	#echo $os 
#### end of original target string 

#### modified string 
	s2a='MPI_INC = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1 -I'$mpiincpath
	ns=$s2a
#### end of modified string 
	echo To replace 
	echo $os 
	echo 'in line' "$lineno" 'in' $filetomod
	echo by 
	echo $ns 
	cat $filetomod | awk -v lineno=$lineno -v ns="$ns" 'NR==lineno {$0=ns} {print}' > output.txt
	mv output.txt $filetomod
	echo '1 is done'
	echo ' ' 

### 2 
### target line number to be replaced
	keyword='MPI_PATH ='
	lineno=$(cat $filetomod | awk -v keyword='MPI_PATH =' '$0 ~ keyword {print NR}')
	echo $lineno
#### original target string 
	os=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print}')
	echo $os 
#### end of original target string 

#### modified string 
	s2a='MPI_PATH = -L'$mpilibpath
	ns=$s2a
#### end of modified string 
	echo To replace 
	echo $os 
	echo 'in line' "$lineno" 'in' $filetomod
	echo by 
	echo $ns 
	cat $filetomod | awk -v lineno=$lineno -v ns="$ns" 'NR==lineno {$0=ns} {print}' > output.txt
	mv output.txt $filetomod
	echo '2 is done'
	echo ' ' 

### 3
### target line number to be replaced
	keyword='MPI_LIB ='
	lineno=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print NR}')
	echo $lineno
#### original target string 
	os=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print}')
	echo $os 
#### end of original target string 

#### modified string 
	s2a='MPI_LIB = -lmpi'
	ns=$s2a
#### end of modified string 
	echo To replace 
	echo $os 
	echo 'in line' "$lineno" 'in' $filetomod
	echo by 
	echo $ns 
	cat $filetomod | awk -v lineno=$lineno -v ns="$ns" 'NR==lineno {$0=ns} {print}' > output.txt
	mv output.txt $filetomod
	echo '3 is done'
	echo ' ' 

### 4
### target line number to be replaced
	keyword='FFT_PATH = '
	lineno=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print NR}')
	echo $lineno
#### original target string 
	os=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print}')
	echo $os 
#### end of original target string 

#### modified string 
	s2a='FFT_PATH = -L$(MKLROOT)/lib/intel64'
	ns=$s2a
#### end of modified string 
	echo To replace 
	echo $os 
	echo 'in line' "$lineno" 'in' $filetomod
	echo by 
	echo $ns 
	cat $filetomod | awk -v lineno=$lineno -v ns="$ns" 'NR==lineno {$0=ns} {print}' > output.txt
	mv output.txt $filetomod
	echo '4 is done'
	echo ' ' 

### 5
### target line number to be replaced
	keyword='FFT_LIB ='
	lineno=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print NR}')
	echo $lineno
#### original target string 
	os=$(cat $filetomod | awk -v keyword="$keyword" '$0 ~ keyword {print}')
	echo $os 
#### end of original target string 

#### modified string 
	s2a='FFT_LIB = -lfftw3xf_intel #-lmkl_intel_ilp64 -lmkl_sequential -lmkl_core	'
	ns=$s2a
#### end of modified string 
	echo To replace 
	echo $os 
	echo 'in line' "$lineno" 'in' $filetomod
	echo by 
	echo $ns 
	cat $filetomod | awk -v lineno=$lineno -v ns="$ns" 'NR==lineno {$0=ns} {print}' > output.txt
	mv output.txt $filetomod
	echo '5 is done'
	echo ' ' 

### 6
### to delete the specific line 
	ERR=$(( $lineno + 1 ))
	line2del=$(cat $filetomod | awk -v ERR=$ERR 'NR==ERR {print}')
	echo $line2del 'in line' "$ERR" 'in' $filetomod 'is deleted'
	awk -v no="$ERR" 'NR!=no' $filetomod > output.txt
	mv output.txt $filetomod
	echo '6 is done'
	echo ' '
## end To modify Makefile.xx in MAKE/OPTIONS and copy it to MAKE

cd ../
fn=$(echo $filetomod | awk -F  "." '{print $2}')
cp MAKE/Makefile.$fn MAKE/Makefile.igpu    ## igpu stands for gpu-enabled lammps, compiled using fill intel compiler
# make $fn
make igpu
ln -s /share/apps/$rootdir/src/lmp_igpu /share/apps/local/bin/