#! /bin/bash

## installation of LAMMPS version 18 Jun 2019 in ubuntu 16 in WSL

## preliminary requirements: gnu version openmpi, g++, cmake3

## sudo apt install libfftw3-dev libfftw3-mpi3 
## sudo apt-get install --reinstall zlib1g 
## sudo apt-get zlib1g-dev zlibc libghc-zlib-dev libadios-dev
## In Makefile.ggpu, keyword='FFT_LIB =' the libraries assigned to 'FFT_LIB' item have to specify full path, i.e.,
## FFT_LIB = /usr/lib/x86_64-linux-gnu/libfftw3.a /usr/lib/x86_64-linux-gnu/libfftw3_mpi.so.3 /usr/lib/x86_64-linux-gnu/libfftw3_omp.a /usr/lib/x86_64-linux-gnu/libfftw3_threads.a 

## install cmake3, see https://peshmerge.io/how-to-install-cmake-3-11-0-on-ubuntu-16-04/
sudo apt purge cmake
wget https://github.com/Kitware/CMake/releases/download/v3.15.0-rc4/cmake-3.15.0-rc4-Linux-x86_64.tar.gz
tar -zxvf cmake-3.15.0-rc4-Linux-x86_64.tar.gz
cd cmake-3.15.0-rc4-Linux-x86_64
sudo cp -r bin /usr/
sudo cp -r share /usr/
sudo cp -r doc /usr/share/
sudo cp -r man /usr/share/

## install adios2, see https://adios2.readthedocs.io/en/latest/setting_up/setting_up.html#. Install adios2 needs cmake3
echo 'alias g++="g++ --std=c++0x"' >> ~/.bashrc
## add -std=c++0x to CCFLAGS and LINKFLAGS in MAKE/Makefile_ggpu

# sudo apt install libghc-bool-extras-dev

cd src

make yes-all
make no-kokkos
make no-user-netcdf
make no-user-quip
make no-voronoi
make no-user-h5md
make no-kim
make no-latte
make no-mscg
make no-user-smd
make no-user-vtk
make no-gpu
make no-user-plumed
no-user-scafacos
make no-user-smd
make no-message
make no-user-adios


#13. make gpu #
### Must tweak -arch=sm_xx so that it suits your GPU architecture
### At times certain cuda version works but not for others. May have to do some trial-and-error which cuda version will work properly with which version of lammps.
### Assumes cuda has been intalled by default /usr/local/cuda and the links are properly sourced in .bashrc
### For jaws, GTX 570 Ti is a Maxwell, hence sm_50
### For TITAN X, Titan Xp, GTX 1060, computability is 6.1, hence sm_61
# Refer to https://lammps.sandia.gov/doc/Build_extras.html, or better still,
# https://en.wikipedia.org/wiki/CUDA#GPUs_supported
# to set the value for -arch=sm_xx
#    sm_12 or sm_13 for GT200 (supported by CUDA 3.2 until CUDA 6.5)
#    sm_20 or sm_21 for Fermi (supported by CUDA 3.2 until CUDA 7.5)
#    sm_30 or sm_35 or sm_37 for Kepler (supported since CUDA 5)
#    sm_50 or sm_52 for Maxwell (supported since CUDA 6)
#    sm_60 or sm_61 for Pascal (supported since CUDA 8)
#    sm_70 for Volta (supported since CUDA 9)
#    sm_75 for Turing (supported since CUDA 10)

# Successful gpu-enabled installation:
# 1. For the GeForce 940M gpu (Maxwell architecture) in my laptop, lib/gpu is compiled using 
# mixed precsion, CUDA_PRECISION = -D_SINGLE_DOUBLE, CUDA_ARCH = -arch=sm_50, lammps version of 5May2018, compiled fully using intel compiler (parallel studio 2017). cuda-8.0 was used by setting CUDA_HOME = /usr/local/cuda-8.0 in Makefile.linux.
# 2. For the GTX 1060 gpu (Pascal architecture) in my server, lib/gpu is compiled using mixed precsion, CUDA_PRECISION = -D_SINGLE_DOUBLE, CUDA_ARCH = -arch=sm_60, lammps version of 5May2018, compiled fully using intel compiler (parallel studio 2017). cuda-9.2 was used (cuda-9.0 did not work) by setting CUDA_HOME = /usr/local/cuda-9.2 in Makefile.linux.

### check if nvidia-smi is working. If yes, make yes-gpu; else make no-gpu

filetomod=Makefile.linux
command=nvidia-smi
$command > /dev/null 2>&1
if [ $? -eq 0 ]; then
    echo $command found. CUDA driver and toolkit properly installed
    echo To install gpu library in lammps
    cd ../lib/gpu
    cp $filetomod $filetomod'.orig'
    keyword='mpicxx'
	s2a='/share/apps/openmpi-2.1.1/gnu/bin/mpicxx'
	#s2a='/share/apps/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpicxx'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	keyword='/usr/local/cuda'
	s2a='/usr/local/cuda-9.2'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	ca=$(/usr/local/cuda/samples/bin/x86_64/linux/release/deviceQuery | grep  'CUDA Capability' | awk 'END {print $6}' | awk -F  "." '{print $1$2}')
	echo compute capability given by deviceQuery is $ca
	echo 'CUDA_ARCH = -arch=sm_'$ca >> $filetomod
    make -f $filetomod clean
	make -f $filetomod
	if [ -e libgpu.a ]; then
		echo 'libgpu.a found. To make yes-gpu in ../../src'
		cd ../../src
		make no-gpu
		make yes-gpu
	else
		echo libgpu.a not found after make -f Makefile.linux. Not to install gpu in ../../src
		cd ../../src
		make no-gpu
	fi 
else
    echo no $command found. CUDA driver or toolkit not properly installed
    echo Not to install gpu library in lammps
fi
### check if nvidia-smi is working. If yes, make yes-gpu; else make no-gpu


#14. To modify Makefile.xx in MAKE/OPTIONS and copy it to MAKE

	filetomod=Makefile.gpu
	cd MAKE
	cp OPTIONS/$filetomod .
	
	cp $filetomod $filetomod'.orig'
	keyword='mpicxx'
	s2a='mpicxx'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	keyword='MPI_INC =       -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1'
	s2a='MPI_INC = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1  -I/usr/include/openmpi'
	sed -i "s|$keyword|$s2a|g" $filetomod

	keyword='MPI_PATH ='
	s2a='MPI_PATH = -L/usr/lib/openmpi/lib'
	sed -i "s|$keyword|$s2a|g" $filetomod

	keyword='MPI_LIB ='
	s2a='MPI_LIB = -lmpi'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	keyword='FFT_INC ='
	s2a='FFT_INC = -I/usr/include'
	sed -i "s|$keyword|$s2a|g" $filetomod
	
	keyword='FFT_LIB ='
	s2a='FFT_LIB = -L/usr/lib/x86_64-linux-gnu -lfftw3 -lfftw3_mpi -lfftw3_omp -lfftw3_threads'
	sed -i "s|$keyword|$s2a|g" $filetomod

## end To modify Makefile.xx in MAKE/OPTIONS and copy it to MAKE

#15. 
cd ../
fn=$(echo $filetomod | awk -F  "." '{print $2}')
cp MAKE/Makefile.$fn MAKE/Makefile.ggpu    ## ggpu stands for gpu-enabled lammps, compiled using gnu compilers

mkdir ~/local
mkdir ~/local/bin

make ggpu
ln -s $PWD/lmp_ggpu ~/local/bin