# For a verbose description of these fields please refer to the original 
# Quantum ESPRESSO make.inc file

TOPDIR = $(dir $(abspath $(filter %make.inc,$(MAKEFILE_LIST))))

# See install/README.DFLAGS for a list of options and their meaning
DFLAGS         = -DUSE_CUDA -D__FFTW -D__MPI -D__MKL -DUSE_GPU_MPI -DUSE_IPC
FDFLAGS        = $(DFLAGS) $(MANUAL_DFLAGS)

IFLAGS         = -I$(TOPDIR)/include -I../include/ -I$(TOPDIR)/EIGENSOLVER_GPU/lib_eigsolve -I${MKLROOT}/include

MOD_FLAG      = -I

MPIF90         = /share/apps/pgi/linux86-64/2018/mpi/openmpi/bin/mpif90 #mpif90
MPICC          = /share/apps/pgi/linux86-64/2018/mpi/openmpi/bin/mpicc #mpicc
CC             = pgcc
F77            = pgf77

# GPU architecture (Kepler: 35, Pascal: 60, Volta: 70 )
GPU_ARCH=60

# CUDA runtime (Pascal: 8.0, Volta: 9.0)
CUDA_RUNTIME=8.0

CFLAGS         = -O3 -Mpreprocess $(DFLAGS) $(IFLAGS)
F90FLAGS       = -O3 -Mpreprocess -mp -Mcuda=cc$(GPU_ARCH),cuda$(CUDA_RUNTIME) -Mlarge_arrays $(FDFLAGS) $(IFLAGS) $(MODFLAGS)
FFLAGS         = -O3 -Mpreprocess -mp


LD             = /share/apps/pgi/linux86-64/2018/mpi/openmpi/bin/mpif90
LDFLAGS        = -pgf90libs  -mp -Mcuda=cc$(GPU_ARCH),cuda$(CUDA_RUNTIME) -Mlarge_arrays
LD_LIBS        = -Mcudalib=cufft,cublas,cusolver $(TOPDIR)/EIGENSOLVER_GPU/lib_eigsolve/lib_eigsolve.a

MATH_LIBS      = -L${MKLROOT}/lib/intel64 -lmkl_intel_lp64 -pgf90libs -lmkl_core -lmkl_pgi_thread

EIGENSOLVER_VERSION	= 0.3


# Do not edit below this line
-include install/make.check-cuda

AR             = ar
ARFLAGS        = ruv
RANLIB         = ranlib
WGET           = wget -O

.SUFFIXES :
.SUFFIXES : .o .c .f .f90 .F90

.F90.o:
	$(MPIF90) $(F90FLAGS) -c $< -o $(*)_cpu.o ; \
	$(MPIF90) $(F90FLAGS) -c -DUSE_GPU $< -o $(*)_gpu.o ; \
	ld -r $(*)_cpu.o $(*)_gpu.o -o $(*).o ; \
	rm $(*)_cpu.o $(*)_gpu.o

.f90.o:
	$(MPIF90) $(F90FLAGS) -c $(*).f90 -o $(*).o

.f.o:
	$(F77) $(FFLAGS) -c $<

.c.o:
	$(CC) $(CFLAGS)  -c $<

LIBOBJS        = $(TOPDIR)/clib/clib.a $(TOPDIR)/iotk/src/libiotk.a
LIBS           = $(MATH_LIBS) $(LD_LIBS)