From e2f1eefcbddbad434173a8c36edf249f8f03a2fd Mon Sep 17 00:00:00 2001
From: JanThorbecke <janth@xs4all.nl>
Date: Thu, 23 May 2019 07:55:59 -0500
Subject: [PATCH] WS15
---
FFTlib/Makefile | 2 +-
Make_include_swan | 165 +++++++++++++++++++++++
Makefile | 6 +-
marchenko/demo/WS15/README.1 | 23 +++-
marchenko/demo/WS15/README.2 | 16 ++-
marchenko/demo/WS15/job.pbs | 1 +
marchenko/demo/invisible/model.scr | 2 +-
marchenko/demo/twoD/initialFocus.scr | 60 +++++++++
marchenko/demo/twoD/initialFocus_pbs.scr | 9 +-
9 files changed, 269 insertions(+), 15 deletions(-)
create mode 100644 Make_include_swan
create mode 100755 marchenko/demo/twoD/initialFocus.scr
diff --git a/FFTlib/Makefile b/FFTlib/Makefile
index b6e4faf..bf53a5a 100644
--- a/FFTlib/Makefile
+++ b/FFTlib/Makefile
@@ -44,7 +44,7 @@ $(LIB) : $(ARCH)
mkdirs:
-mkdir -p $(ROOT)/lib
-mkdir -p $(ROOT)/include
- -cp -rp genfft.h ../include/genfft.h
+ -cp -rp genfft.h $(ROOT)/include/genfft.h
bins:
cd test ; $(MAKE)
diff --git a/Make_include_swan b/Make_include_swan
new file mode 100644
index 0000000..b37cffb
--- /dev/null
+++ b/Make_include_swan
@@ -0,0 +1,165 @@
+# Makefile for general rules
+
+# To Change the compile environment to your current system you should set:
+# -1- ROOT variable to the directory where you found this file
+# -2- if needed use a different compiler (CC) if gcc is not available
+# -3- on Solaris system use RANLIB=ranlib which is defined below
+
+# the current directory (in vi ":r!pwd")
+ROOT=/lus/scratch/${USER}/OpenSource
+
+#############################################################################
+# Some convenient abbreviations
+B = $(ROOT)/bin
+I = $(ROOT)/include
+L = $(ROOT)/lib
+
+########################################################################
+# C compiler; change this only if you are using a different C-compiler
+
+#GNU
+#CC = gcc-mp-5
+#CC = gcc
+#FC = gfortran
+# Linux gcc version 4.x
+#OPTC = -O3 -ffast-math
+#to include parallelisation with OpenMP
+#OPTC += -fopenmp
+
+# for better performing code you can try:
+#OPTC = -O3 -fno-trapping-math -ffast-math -funroll-all-loops -mavx -fomit-frame-pointer -mfpmath=sse -ftree-vectorizer-verbose=1
+# Linux gcc version 3.x
+#OPTC = -O3 -ffast-math -funroll-all-loops -mfpmath=sse
+
+#for double precision use FFTlib and emmod
+#OPTC += -DDOUBLE
+#OPTF += -fdefault-double-8 -fdefault-real-8
+
+#Cray
+#CC=cc
+#FC=ftn
+#OPTC = -O2
+#OPTF = -O2
+
+#Intel
+CC = icc
+FC = ifort
+### Linux
+##OPTC = -O3 -no-prec-div -qopt-report-phase=vec,openmp
+##OPTF = -O3 -no-prec-div -qopt-report-phase=vec,openmp
+OPTC = -O3 -no-prec-div -xSANDYBRIDGE
+#OPTF = -O3 -no-prec-div -xCORE-AVX2
+##to include parallelisation with OpenMP
+OPTC += -qopenmp
+
+# Apple OSX intel 11.1.076 snow leopard 10.6.2
+#OPTC = -O3 -msse3 -no-prec-div -vec-report2 -fno-builtin-__sprintf_chk
+
+#PGI
+#CC = pgcc
+#FC = pgf90
+#OPTC = -fast -Minfo=vect -Mvect=simd:256 -Msafeptr
+#OPTC = -fast -Minfo=vect -Mvect=simd:256 -Msafeptr -Mprof=lines
+#OPTF = -fast
+#LDFLAGS = -fast -Minfo=vect -Mvect=simd:256 -Msafeptr
+
+#Pathscale
+#CC = cc
+#FC = ftn
+#OPTC = -Ofast -OPT:Ofast -fno-math-errno
+#OPTF = -Ofast -OPT:Ofast -fno-math-errno
+
+#Apple OSX clang/gcc (10.9) llvm
+#CC = clang
+#OPTC = -Ofast
+#LDFLAGS = -Ofast
+
+#AMD Open64
+#CC = opencc
+#FC = openf95
+#OPTC = -Ofast
+#OPTF = -Ofast
+#LDFLAGS = -static -Ofast
+
+#############################################################################
+# BLAS and LAPACK libraries
+MKLLIB=${MKLROOT}/lib
+#for GNU compilers
+#you might need to add intel64 to : ${MKLROOT}/lib/intel64
+#BLAS = -L${MKLROOT}/lib/ -lmkl_gf_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
+#for GNU on OSX
+#BLAS = -L${MKLROOT}/lib/ -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
+#on linux you want to use groups and MKL is in lib/intel64
+#MKLLIB=${MKLROOT}/lib/intel64
+#BLAS = -Wl,-rpath ${MKLLIB} -Wl,--start-group -L${MKLLIB} -lmkl_gf_lp64 -lmkl_sequential -lmkl_core -Wl,--end-group -lpthread -lm -ldl
+#for intel compilers
+BLAS = -mkl
+
+#############################################################################
+# FOR FFT LIBRARIES
+#AMD ACML 4.4.0
+#AMDROOT = /home/thorbcke/amdsdk/v1.0/acml/open64_64
+#OPTC += -DACML440 -I$(AMDROOT)/include
+#BLAS = -L$(AMDROOT)/lib -lacml -lfortran -lffio -lrt -lm
+#Intel MKL
+#MKLROOT=/opt/intel/mkl/
+MKLLIB=${MKLROOT}/lib
+OPTC += -DMKL -I$(MKLROOT)/include
+#for GNU compilers
+#FFT = -Wl,-rpath ${MKLLIB} -L${MKLLIB} -lmkl_gf_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
+#for GNU on OSX
+#FFT = -L${MKLROOT}/lib/ -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
+#on linux you want to use groups and MKL is in lib/intel64
+#MKLLIB=${MKLROOT}/lib/intel64
+#FFT = -Wl,-rpath ${MKLLIB} -Wl,--start-group -L${MKLLIB} -lmkl_gf_lp64 -lmkl_sequential -lmkl_core -Wl,--end-group -lpthread -lm -ldl
+#for Intel compilers
+FFT = -Wl,-rpath ${MKLLIB} -L${MKLLIB} -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
+
+#LIBARIES
+LIBS= -L$L -lgenfft $(FFT) $(BLAS)
+
+
+########################################################################
+# standard CFLAGS
+CFLAGS = -I$I -I.
+
+#############################################################################
+# the archiver
+AR = ar
+
+#############################################################################
+# ar FLAGS
+ARFLAGS = rv
+
+#############################################################################
+# ranlib definition
+RANLIB = ar -s
+# on Sun SOLARIS use:
+#RANLIB = ranlib
+
+
+.SUFFIXES : .o .c .cc .f .a .F90
+.c.o :
+ $(CC) -c $(CFLAGS) $(OPTC) $<
+.c.a :
+ $(CC) -c $(CFLAGS) $(OPTC) $<
+ $(AR) $(ARFLAGS) $@ $*.o
+ rm -f $*.o
+.o.a :
+ $(AR) $(ARFLAGS) $@ $*.o
+ rm -f $*.o
+.f.o :
+ $(FC) -c $(FFLAGS) $(OPTF) $<
+.F90.o :
+ $(FC) -c $(FFLAGS) $(OPTF) $<
+.f.a :
+ $(FC) -c $(FFLAGS) -I$I $<
+ $(AR) $(ARFLAGS) $@ $*.o
+ rm -f $*.o
+.cc.a :
+ $(C++) -c $(C++FLAGS) -I$I $<
+ $(AR) $(ARFLAGS) $@ $*.o
+ rm -f $*.o
+.cc.o :
+ $(C++) -c $(C++FLAGS) $<
+
diff --git a/Makefile b/Makefile
index 74e45de..ebce817 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ all: mkdirs
cd FFTlib ; $(MAKE)
cd fdelmodc ; $(MAKE) install
cd utils ; $(MAKE) install
- cd marchenko ; $(MAKE) install
+ cd marchenko ; $(MAKE) install
cd corrvir ; $(MAKE) install
cd raytime ; $(MAKE) install
cd MDD ; $(MAKE) install
@@ -18,7 +18,7 @@ clean:
cd FFTlib ; $(MAKE) $@
cd fdelmodc ; $(MAKE) $@
cd utils ; $(MAKE) $@
- cd marchenko ; $(MAKE) $@
+ cd marchenko ; $(MAKE) $@
cd corrvir ; $(MAKE) $@
cd raytime ; $(MAKE) $@
cd MDD ; $(MAKE) $@
@@ -27,7 +27,7 @@ realclean:
cd FFTlib ; $(MAKE) $@
cd fdelmodc ; $(MAKE) $@
cd utils ; $(MAKE) $@
- cd marchenko ; $(MAKE) $@
+ cd marchenko ; $(MAKE) $@
cd corrvir ; $(MAKE) $@
cd raytime ; $(MAKE) $@
cd MDD ; $(MAKE) $@
diff --git a/marchenko/demo/WS15/README.1 b/marchenko/demo/WS15/README.1
index 8894f24..a758b34 100644
--- a/marchenko/demo/WS15/README.1
+++ b/marchenko/demo/WS15/README.1
@@ -1,7 +1,7 @@
*1* Source the setup.sh script that you can find on
-source /home/users/jan/WS15/setup.sh
+source /home/users/jan/WS15setup.sh
The setup.sh script completes the following tasks:
@@ -25,7 +25,8 @@ cp Make_include_swan Make_include
make clean
make
-After succesfull complication the code can be used running the exercises and demo's
+After succesfull complication the code can be used for running the exercises and demo's
+
*3* Running jobs
@@ -35,9 +36,23 @@ An example script can be found on:
/home/users/jan/WS15/job.pbs
in job.pbs leave the number of nodes set to 1, the code is not MPI parallel and cannot use more than 1 node. The number of OMP_NUM_THREADS can be changed and is currently set to 40.
-Add the ...scr command as the last line in the aprun command line and submit the job to the queue with 'qsub job.pbs'
+Add the ...scr command (that can be found in the demo directories) as the last line in the aprun command line and submit the job to the queue with 'qsub job.pbs'
+
+*4* Display of results:
+
+suximage < result.su perc=99 &
+
+if X11 display 1s too slow:
+
+supsimage < result.su perc=99 > results.eps
+convert results.eps ~/results.png
+
+copy png file to local machine
+
+scp trxx@swan.cray.com:~/*png .
+
-*4* Usefull job monitor commands specific for Cray
+*5* Usefull job monitor commands specific for Cray
apstat : status of applications running
xtnodestat : placement of application on system
diff --git a/marchenko/demo/WS15/README.2 b/marchenko/demo/WS15/README.2
index 44bf68c..c51bcb7 100644
--- a/marchenko/demo/WS15/README.2
+++ b/marchenko/demo/WS15/README.2
@@ -1,9 +1,21 @@
-Marchenko demo/twoD: reproduce the figures from the paper, but woth a 2D model.
+Marchenko demo/twoD: reproduce the figures from the paper, but with a 2D model.
cd /lus/scratch/$USER/OpenSource/marchenko/demo/twoD
-cp ~jan/WS15/job.pbs .
# Copy pre-computed shots
cp -rp /lus/scratch/jan/OpenSource/marchenko/demo/twoD/shots .
+cp /lus/scratch/$USER/OpenSource/marchenko/demo/WS15/job.pbs .
+
+Adapt job.pbs to
+
+Compute the intial focusing operator placed at a depth of 1100 m by running
+
+initialFocus.scr
+
+and run the marchenko program
+
+marchenko.scr
+
+
diff --git a/marchenko/demo/WS15/job.pbs b/marchenko/demo/WS15/job.pbs
index c9bff77..13eef27 100755
--- a/marchenko/demo/WS15/job.pbs
+++ b/marchenko/demo/WS15/job.pbs
@@ -12,6 +12,7 @@ set -x
cd $PBS_O_WORKDIR
ulimit -s unlimited
+export KMP_AFFINITY=disabled
export OMP_NUM_THREADS=40
starttime=`date +%s%N`
diff --git a/marchenko/demo/invisible/model.scr b/marchenko/demo/invisible/model.scr
index cbd96eb..22f93aa 100755
--- a/marchenko/demo/invisible/model.scr
+++ b/marchenko/demo/invisible/model.scr
@@ -25,7 +25,7 @@ makemod sizex=6000 sizez=1500 dx=$dx dz=$dx cp0=1000 ro0=1000 \
#define wavelet for modeling R
makewave w=fw fmin=0 flef=5 frig=80 fmax=100 dt=$dt file_out=wavefw.su nt=4096 t0=0.3 scale=0 scfft=1
-export OMP_NUM_THREADS=8
+export OMP_NUM_THREADS=12
#Model shot record in middle of model
fdelmodc \
diff --git a/marchenko/demo/twoD/initialFocus.scr b/marchenko/demo/twoD/initialFocus.scr
new file mode 100755
index 0000000..ebf4f4b
--- /dev/null
+++ b/marchenko/demo/twoD/initialFocus.scr
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+export PATH=$HOME/src/OpenSource/bin:$PATH:
+
+dx=2.5
+dt=0.0005
+
+makemod sizex=6000 sizez=2000 dx=$dx dz=$dx cp0=1900 ro0=1200 \
+ orig=-3000,0 file_base=synclDown.su verbose=2 \
+ intt=def x=-3000,500,3000 z=195,195,195 poly=1 cp=1950 ro=3700 \
+ intt=def x=-3000,3000 z=600,600 poly=0 cp=2050 ro=1750 \
+ intt=def x=-3000,3000 z=680,680 poly=0 cp=2150 ro=2220 \
+ intt=def x=-3000,3000 z=780,780 poly=0 cp=2230 ro=1700 \
+ intt=def x=-3000,-2200,-1500,0,1300,2100,3000 z=520,580,680,840,680,600,500 poly=2 cp=2400 ro=2800 \
+
+makewave fp=20 dt=$dt file_out=wave.su nt=4096 t0=0.1 scale=1
+
+dxshot=10
+ishot=300
+nshots=301
+
+export KMP_AFFINITY=disabled
+export OMP_NUM_THREADS=16
+mkdir -p shots
+mkdir -p jobs
+
+while (( ishot < nshots ))
+do
+
+ (( xsrc = -3000 + ${ishot}*${dxshot} ))
+ echo xsrc=$xsrc
+ file_rcv=shots/iniFocus_z1100_x${xsrc}.su
+
+fdelmodc \
+ file_cp=synclDown_cp.su ischeme=1 iorder=4 \
+ file_den=synclDown_ro.su \
+ file_src=wave.su \
+ file_rcv=$file_rcv \
+ src_type=1 \
+ src_orient=1 \
+ src_injectionrate=1 \
+ rec_type_vz=0 \
+ rec_type_p=1 \
+ rec_int_vz=2 \
+ rec_delay=0.1 \
+ dtrcv=0.004 \
+ verbose=2 \
+ tmod=2.100 \
+ dxrcv=10.0 \
+ xrcv1=-3000 xrcv2=3000 \
+ zrcv1=0 zrcv2=0 \
+ xsrc=$xsrc zsrc=1100 \
+ ntaper=200 \
+ left=2 right=2 top=2 bottom=2
+
+ (( ishot = $ishot + 1))
+done
+
+
+
diff --git a/marchenko/demo/twoD/initialFocus_pbs.scr b/marchenko/demo/twoD/initialFocus_pbs.scr
index eb4e0c1..22df20c 100755
--- a/marchenko/demo/twoD/initialFocus_pbs.scr
+++ b/marchenko/demo/twoD/initialFocus_pbs.scr
@@ -33,17 +33,18 @@ do
cat << EOF > jobs/pbs_$ishot.job
#!/bin/bash
#
-#PBS -q medium
+#PBS -q bw44-sm
#PBS -N mod_${xsrc}
#PBS -j eo
-#PBS -m n
-#PBS -l nodes=1
+#PBS -l place=scatter,select=1
+#PBS -l walltime=00:10:00
#PBS -V
export PATH=\$HOME/src/OpenSource/bin:\$PATH:
cd \$PBS_O_WORKDIR
-export OMP_NUM_THREADS=4
+export KMP_AFFINITY=disabled
+export OMP_NUM_THREADS=16
fdelmodc \
file_cp=synclDown_cp.su ischeme=1 iorder=4 \
--
GitLab