WS15

FFTlib/Makefile

@@ -44,7 +44,7 @@ $(LIB)  : $(ARCH)
 mkdirs:
 	-mkdir -p $(ROOT)/lib
 	-mkdir -p $(ROOT)/include
-	-cp -rp genfft.h ../include/genfft.h
+	-cp -rp genfft.h $(ROOT)/include/genfft.h
 
 bins: 
 	cd test ; $(MAKE)

Make_include_swan

+# Makefile for general rules
+
+# To Change the compile environment to your current system you should set:
+#   -1- ROOT variable to the directory where you found this file
+#   -2- if needed use a different compiler (CC) if gcc is not available
+#   -3- on Solaris system use RANLIB=ranlib which is defined below
+
+# the current directory (in vi ":r!pwd")
+ROOT=/lus/scratch/${USER}/OpenSource
+
+#############################################################################
+# Some convenient abbreviations
+B = $(ROOT)/bin
+I = $(ROOT)/include
+L = $(ROOT)/lib
+
+########################################################################
+# C compiler; change this only if you are using a different C-compiler
+
+#GNU 
+#CC = gcc-mp-5 
+#CC = gcc
+#FC = gfortran
+# Linux gcc version 4.x
+#OPTC = -O3 -ffast-math
+#to include parallelisation with OpenMP
+#OPTC += -fopenmp
+
+# for better performing code you can try:
+#OPTC = -O3 -fno-trapping-math -ffast-math -funroll-all-loops -mavx -fomit-frame-pointer -mfpmath=sse -ftree-vectorizer-verbose=1
+# Linux gcc version 3.x
+#OPTC = -O3 -ffast-math -funroll-all-loops -mfpmath=sse
+
+#for double precision use FFTlib and emmod
+#OPTC += -DDOUBLE
+#OPTF += -fdefault-double-8 -fdefault-real-8
+
+#Cray
+#CC=cc
+#FC=ftn
+#OPTC = -O2
+#OPTF = -O2
+
+#Intel 
+CC = icc
+FC = ifort
+### Linux
+##OPTC = -O3 -no-prec-div -qopt-report-phase=vec,openmp
+##OPTF = -O3 -no-prec-div -qopt-report-phase=vec,openmp
+OPTC = -O3 -no-prec-div -xSANDYBRIDGE 
+#OPTF = -O3 -no-prec-div -xCORE-AVX2
+##to include parallelisation with OpenMP
+OPTC += -qopenmp
+
+# Apple OSX intel 11.1.076 snow leopard 10.6.2
+#OPTC = -O3 -msse3 -no-prec-div -vec-report2 -fno-builtin-__sprintf_chk 
+
+#PGI 
+#CC = pgcc
+#FC = pgf90
+#OPTC = -fast  -Minfo=vect -Mvect=simd:256 -Msafeptr
+#OPTC = -fast  -Minfo=vect -Mvect=simd:256 -Msafeptr -Mprof=lines
+#OPTF = -fast
+#LDFLAGS = -fast -Minfo=vect -Mvect=simd:256 -Msafeptr 
+
+#Pathscale 
+#CC = cc
+#FC = ftn
+#OPTC = -Ofast -OPT:Ofast -fno-math-errno
+#OPTF = -Ofast -OPT:Ofast -fno-math-errno
+
+#Apple OSX clang/gcc (10.9) llvm
+#CC = clang
+#OPTC = -Ofast
+#LDFLAGS = -Ofast
+
+#AMD Open64
+#CC = opencc
+#FC = openf95
+#OPTC = -Ofast
+#OPTF = -Ofast
+#LDFLAGS = -static -Ofast
+
+#############################################################################
+# BLAS and LAPACK libraries 
+MKLLIB=${MKLROOT}/lib
+#for GNU compilers
+#you might need to add intel64 to : ${MKLROOT}/lib/intel64
+#BLAS = -L${MKLROOT}/lib/ -lmkl_gf_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
+#for GNU on OSX
+#BLAS = -L${MKLROOT}/lib/ -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
+#on linux you want to use groups and MKL is in lib/intel64
+#MKLLIB=${MKLROOT}/lib/intel64
+#BLAS = -Wl,-rpath ${MKLLIB} -Wl,--start-group -L${MKLLIB} -lmkl_gf_lp64 -lmkl_sequential -lmkl_core -Wl,--end-group -lpthread -lm -ldl
+#for intel compilers
+BLAS = -mkl
+
+#############################################################################
+# FOR FFT LIBRARIES
+#AMD ACML 4.4.0
+#AMDROOT = /home/thorbcke/amdsdk/v1.0/acml/open64_64
+#OPTC += -DACML440 -I$(AMDROOT)/include
+#BLAS = -L$(AMDROOT)/lib -lacml -lfortran -lffio -lrt -lm
+#Intel MKL
+#MKLROOT=/opt/intel/mkl/
+MKLLIB=${MKLROOT}/lib
+OPTC += -DMKL -I$(MKLROOT)/include
+#for GNU compilers
+#FFT = -Wl,-rpath ${MKLLIB} -L${MKLLIB} -lmkl_gf_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
+#for GNU on OSX
+#FFT = -L${MKLROOT}/lib/ -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
+#on linux you want to use groups and MKL is in lib/intel64
+#MKLLIB=${MKLROOT}/lib/intel64
+#FFT = -Wl,-rpath ${MKLLIB} -Wl,--start-group -L${MKLLIB} -lmkl_gf_lp64 -lmkl_sequential -lmkl_core -Wl,--end-group -lpthread -lm -ldl
+#for Intel compilers
+FFT = -Wl,-rpath ${MKLLIB} -L${MKLLIB} -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
+
+#LIBARIES
+LIBS= -L$L -lgenfft $(FFT) $(BLAS)
+
+
+########################################################################
+# standard CFLAGS
+CFLAGS = -I$I -I.
+
+#############################################################################
+# the archiver
+AR = ar
+
+#############################################################################
+# ar FLAGS
+ARFLAGS = rv
+
+#############################################################################
+# ranlib definition
+RANLIB = ar -s
+# on Sun SOLARIS use:
+#RANLIB = ranlib
+
+
+.SUFFIXES : .o .c .cc .f .a .F90
+.c.o	:
+	$(CC) -c $(CFLAGS) $(OPTC) $<
+.c.a	:
+	$(CC) -c $(CFLAGS) $(OPTC) $<
+	$(AR) $(ARFLAGS) $@ $*.o
+	rm -f $*.o
+.o.a	:
+	$(AR) $(ARFLAGS) $@ $*.o
+	rm -f $*.o
+.f.o	:
+	$(FC) -c $(FFLAGS) $(OPTF) $<
+.F90.o	:
+	$(FC) -c $(FFLAGS) $(OPTF) $<
+.f.a	:
+	$(FC) -c $(FFLAGS) -I$I $<
+	$(AR) $(ARFLAGS) $@ $*.o
+	rm -f $*.o
+.cc.a	:
+	$(C++) -c $(C++FLAGS) -I$I $<
+	$(AR) $(ARFLAGS) $@ $*.o
+	rm -f $*.o
+.cc.o	:
+	$(C++) -c $(C++FLAGS) $<
+

Makefile

@@ -4,7 +4,7 @@ all: mkdirs
 	cd FFTlib		; $(MAKE)
 	cd fdelmodc		; $(MAKE) install
 	cd utils		; $(MAKE) install
-	cd marchenko	; $(MAKE) install
+	cd marchenko		; $(MAKE) install
 	cd corrvir		; $(MAKE) install
 	cd raytime		; $(MAKE) install
 	cd MDD			; $(MAKE) install
@@ -18,7 +18,7 @@ clean:
 	cd FFTlib 		; $(MAKE) $@
 	cd fdelmodc		; $(MAKE) $@
 	cd utils		; $(MAKE) $@
-	cd marchenko	; $(MAKE) $@
+	cd marchenko		; $(MAKE) $@
 	cd corrvir		; $(MAKE) $@
 	cd raytime		; $(MAKE) $@
 	cd MDD			; $(MAKE) $@
@@ -27,7 +27,7 @@ realclean:
 	cd FFTlib 		; $(MAKE) $@
 	cd fdelmodc		; $(MAKE) $@
 	cd utils		; $(MAKE) $@
-	cd marchenko	; $(MAKE) $@
+	cd marchenko		; $(MAKE) $@
 	cd corrvir		; $(MAKE) $@
 	cd raytime		; $(MAKE) $@
 	cd MDD			; $(MAKE) $@

marchenko/demo/WS15/README.1

 
 *1* Source the setup.sh script that you can find on 
 
-source /home/users/jan/WS15/setup.sh
+source /home/users/jan/WS15setup.sh
 
 The setup.sh script completes the following tasks:
 
@@ -25,7 +25,8 @@ cp Make_include_swan Make_include
 make clean
 make 
 
-After succesfull complication the code can be used running the exercises and demo's
+After succesfull complication the code can be used for running the exercises and demo's
+
 
 *3* Running jobs 
 
@@ -35,9 +36,23 @@ An example script can be found on:
 /home/users/jan/WS15/job.pbs
 
 in job.pbs leave the number of nodes set to 1, the code is not MPI parallel and cannot use more than 1 node. The number of OMP_NUM_THREADS can be changed and is currently set to 40.
-Add the ...scr command as the last line in the aprun command line and submit the job to the queue with 'qsub job.pbs'
+Add the ...scr command (that can be found in the demo directories) as the last line in the aprun command line and submit the job to the queue with 'qsub job.pbs'
+
+*4* Display of results:
+
+suximage < result.su perc=99 &
+
+if X11 display 1s too slow:
+
+supsimage < result.su perc=99 > results.eps
+convert results.eps ~/results.png
+
+copy png file to local machine
+
+scp trxx@swan.cray.com:~/*png . 
+
 
-*4* Usefull job monitor commands specific for Cray
+*5* Usefull job monitor commands specific for Cray
 
 apstat : status of applications running
 xtnodestat : placement of application on system

marchenko/demo/WS15/README.2

 
-Marchenko demo/twoD: reproduce the figures from the paper, but woth a 2D model.
+Marchenko demo/twoD: reproduce the figures from the paper, but with a 2D model.
 
 cd /lus/scratch/$USER/OpenSource/marchenko/demo/twoD
 
-cp ~jan/WS15/job.pbs .
 # Copy pre-computed shots
 cp -rp /lus/scratch/jan/OpenSource/marchenko/demo/twoD/shots .
+cp /lus/scratch/$USER/OpenSource/marchenko/demo/WS15/job.pbs .
+
+Adapt job.pbs to 
+
+Compute the intial focusing operator placed at a depth of 1100 m by running 
+
+initialFocus.scr
+
+and run the marchenko program
+ 
+marchenko.scr
+
+
 

marchenko/demo/WS15/job.pbs

@@ -12,6 +12,7 @@ set -x
 cd $PBS_O_WORKDIR
 ulimit -s unlimited
 
+export KMP_AFFINITY=disabled
 export OMP_NUM_THREADS=40
 
 starttime=`date +%s%N`

marchenko/demo/invisible/model.scr

@@ -25,7 +25,7 @@ makemod sizex=6000 sizez=1500 dx=$dx dz=$dx cp0=1000  ro0=1000 \
 #define wavelet for modeling R
 makewave w=fw fmin=0 flef=5 frig=80 fmax=100 dt=$dt file_out=wavefw.su nt=4096 t0=0.3 scale=0 scfft=1
 
-export OMP_NUM_THREADS=8
+export OMP_NUM_THREADS=12
 
 #Model shot record in middle of model
 fdelmodc \

marchenko/demo/twoD/initialFocus.scr

+#!/bin/bash
+
+export PATH=$HOME/src/OpenSource/bin:$PATH:
+
+dx=2.5
+dt=0.0005
+
+makemod sizex=6000 sizez=2000 dx=$dx dz=$dx cp0=1900  ro0=1200 \
+        orig=-3000,0 file_base=synclDown.su verbose=2 \
+        intt=def x=-3000,500,3000 z=195,195,195 poly=1 cp=1950 ro=3700 \
+        intt=def x=-3000,3000 z=600,600 poly=0 cp=2050 ro=1750 \
+        intt=def x=-3000,3000 z=680,680 poly=0 cp=2150 ro=2220 \
+        intt=def x=-3000,3000 z=780,780 poly=0 cp=2230 ro=1700 \
+        intt=def x=-3000,-2200,-1500,0,1300,2100,3000 z=520,580,680,840,680,600,500 poly=2 cp=2400 ro=2800 \
+
+makewave fp=20 dt=$dt file_out=wave.su nt=4096 t0=0.1 scale=1
+
+dxshot=10
+ishot=300
+nshots=301
+
+export KMP_AFFINITY=disabled
+export OMP_NUM_THREADS=16
+mkdir -p shots
+mkdir -p jobs
+
+while (( ishot < nshots ))
+do
+
+	(( xsrc = -3000 + ${ishot}*${dxshot} ))
+	echo xsrc=$xsrc
+	file_rcv=shots/iniFocus_z1100_x${xsrc}.su
+	
+fdelmodc \
+	file_cp=synclDown_cp.su ischeme=1 iorder=4 \
+   	file_den=synclDown_ro.su \
+   	file_src=wave.su \
+   	file_rcv=$file_rcv \
+   	src_type=1 \
+	src_orient=1 \
+	src_injectionrate=1 \
+   	rec_type_vz=0 \
+   	rec_type_p=1 \
+   	rec_int_vz=2 \
+	rec_delay=0.1 \
+   	dtrcv=0.004 \
+   	verbose=2 \
+   	tmod=2.100 \
+   	dxrcv=10.0 \
+   	xrcv1=-3000 xrcv2=3000 \
+   	zrcv1=0 zrcv2=0 \
+   	xsrc=$xsrc zsrc=1100 \
+   	ntaper=200 \
+   	left=2 right=2 top=2 bottom=2
+
+	(( ishot = $ishot + 1))
+done
+
+
+

marchenko/demo/twoD/initialFocus_pbs.scr

@@ -33,17 +33,18 @@ do
 cat << EOF > jobs/pbs_$ishot.job 
 #!/bin/bash
 #
-#PBS -q medium
+#PBS -q bw44-sm
 #PBS -N mod_${xsrc}
 #PBS -j eo 
-#PBS -m n 
-#PBS -l nodes=1
+#PBS -l place=scatter,select=1
+#PBS -l walltime=00:10:00
 #PBS -V
 
 export PATH=\$HOME/src/OpenSource/bin:\$PATH:
 cd \$PBS_O_WORKDIR
 
-export OMP_NUM_THREADS=4
+export KMP_AFFINITY=disabled
+export OMP_NUM_THREADS=16
 	
 fdelmodc \
 	file_cp=synclDown_cp.su ischeme=1 iorder=4 \