[gmx-users] MPI-specific compile problem finding fftwf_plan_r2r_1d in libfftw3f.a -- undefined reference to `_intel_fast_memset'

[Christopher Neale]

I am using intel compilers 17.0.4 to compile gromacs 5.1.2.

When I compile in serial, the fftw is found:

-- Checking for module 'fftw3f'
--   Found fftw3f, version 3.3.3
-- Looking for fftwf_plan_r2r_1d in /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a
-- Looking for fftwf_plan_r2r_1d in /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a - found
-- Looking for fftwf_have_simd_sse in /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a
-- Looking for fftwf_have_simd_sse in /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a - not found
-- Looking for fftwf_have_sse in /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a
-- Looking for fftwf_have_sse in /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a - not found
-- Looking for fftwf_have_simd_sse2 in /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a
-- Looking for fftwf_have_simd_sse2 in /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a - found
-- Using external FFT library - FFTW3

But when I compile MPI, fftw is not found:

-- Checking for module 'fftw3f'
--   Found fftw3f, version 3.3.3
-- Looking for fftwf_plan_r2r_1d in /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a
-- Looking for fftwf_plan_r2r_1d in /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a - not found
CMake Error at cmake/FindFFTW.cmake:101 (message):
  Could not find fftwf_plan_r2r_1d in
  /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a,


And when I look at CMakeFiles/CMakeError.log I find error messages not indicated in the successful serial compilation:

Determining if the function fftwf_plan_r2r_1d exists in the /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a failed with the following output:
Change Dir: /net/scratch3/cneale/exe/KODIAK/GROMACS/build/gromacs-5.1.2/mpi_intel/CMakeFiles/CMakeTmp

Run Build Command:"/usr/bin/gmake" "cmTC_5838c/fast"
/usr/bin/gmake -f CMakeFiles/cmTC_5838c.dir/build.make CMakeFiles/cmTC_5838c.dir/build
gmake[1]: Entering directory `/net/scratch3/cneale/exe/KODIAK/GROMACS/build/gromacs-5.1.2/mpi_intel/CMakeFiles/CMakeTmp'
Building C object CMakeFiles/cmTC_5838c.dir/CheckFunctionExists.c.o
/usr/projects/hpcsoft/toss3/common/intel-clusterstudio/2017.1.024/impi/2017.1.132/intel64/bin/mpicc   -Wundef -Wextra -Wno-missing-field-initializers -Wno-sign-compare -Wpointer-arith -Wall -Wno-unused -Wunused-value -Wunused-parameter  -DCHECK_FUNCTION_EXISTS=fftwf_plan_r2r_1d   -o CMakeFiles/cmTC_5838c.dir/CheckFunctionExists.c.o   -c /turquoise/usr/projects/hpcsoft/toss3/common/cmake/3.9.0/share/cmake-3.9/Modules/CheckFunctionExists.c
Linking C executable cmTC_5838c
/turquoise/usr/projects/hpcsoft/toss3/common/cmake/3.9.0/bin/cmake -E cmake_link_script CMakeFiles/cmTC_5838c.dir/link.txt --verbose=1
/usr/projects/hpcsoft/toss3/common/intel-clusterstudio/2017.1.024/impi/2017.1.132/intel64/bin/mpicc  -Wundef -Wextra -Wno-missing-field-initializers -Wno-sign-compare -Wpointer-arith -Wall -Wno-unused -Wunused-value -Wunused-parameter  -DCHECK_FUNCTION_EXISTS=fftwf_plan_r2r_1d    -rdynamic CMakeFiles/cmTC_5838c.dir/CheckFunctionExists.c.o  -o cmTC_5838c /net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a -lm
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(dht-rader.o): In function `apply':
dht-rader.c:(.text+0x104): undefined reference to `_intel_fast_memset'
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(dht-rader.o): In function `awake':
dht-rader.c:(.text+0x6f6): undefined reference to `_intel_fast_memset'
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(rank0.o): In function `apply_memcpy':
rank0.c:(.text+0x660): undefined reference to `_intel_fast_memcpy'
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(rank0.o): In function `apply_memcpy_loop':
rank0.c:(.text+0x7aa): undefined reference to `_intel_fast_memcpy'
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(rank0.o): In function `memcpy_loop':
rank0.c:(.text+0x8c7): undefined reference to `_intel_fast_memcpy'
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(vrank3-transpose.o): In function `apply_gcd':
vrank3-transpose.c:(.text+0xac): undefined reference to `_intel_fast_memcpy'
vrank3-transpose.c:(.text+0x137): undefined reference to `_intel_fast_memcpy'
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(vrank3-transpose.o): In function `apply_cut':
vrank3-transpose.c:(.text+0x5f7): undefined reference to `_intel_fast_memmove'
vrank3-transpose.c:(.text+0x661): undefined reference to `_intel_fast_memcpy'
vrank3-transpose.c:(.text+0x6b9): undefined reference to `_intel_fast_memcpy'
vrank3-transpose.c:(.text+0x748): undefined reference to `_intel_fast_memmove'
vrank3-transpose.c:(.text+0x803): undefined reference to `_intel_fast_memcpy'
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(vrank3-transpose.o): In function `apply_toms513':
vrank3-transpose.c:(.text+0xf0a): undefined reference to `_intel_fast_memset'
vrank3-transpose.c:(.text+0x1001): undefined reference to `_intel_fast_memcpy'
vrank3-transpose.c:(.text+0x101e): undefined reference to `_intel_fast_memcpy'
vrank3-transpose.c:(.text+0x112b): undefined reference to `_intel_fast_memcpy'
vrank3-transpose.c:(.text+0x1155): undefined reference to `_intel_fast_memcpy'
vrank3-transpose.c:(.text+0x11e0): undefined reference to `_intel_fast_memcpy'
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(vrank3-transpose.o):vrank3-transpose.c:(.text+0x11fd): more undefined references to `_intel_fast_memcpy' follow
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(trig.o): In function `cexpl_sincos':
trig.c:(.text+0x1a4): undefined reference to `__libm_sse2_sincos'
/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec/lib/libfftw3f.a(trig.o): In function `fftwf_mktriggen':
trig.c:(.text+0x5ce): undefined reference to `__svml_sincos2'
trig.c:(.text+0x777): undefined reference to `__svml_sincos2'
trig.c:(.text+0xa52): undefined reference to `__svml_sincos2'
trig.c:(.text+0xc63): undefined reference to `__svml_sincos2'
collect2: error: ld returned 1 exit status
gmake[1]: *** [cmTC_5838c] Error 1
gmake[1]: Leaving directory `/net/scratch3/cneale/exe/KODIAK/GROMACS/build/gromacs-5.1.2/mpi_intel/CMakeFiles/CMakeTmp'
gmake: *** [cmTC_5838c/fast] Error 2

##########

Here is the diff between my two compilation scripts:

bash-4.2$ diff mpi_intel/cn_compile.sh serial_intel/cn_compile.sh
8d7
< module load intel-mpi/2017.1
11c10
< edir=../../../exec/gromacs-5.1.2/mpi_intel
---
> edir=../../../exec/gromacs-5.1.2/serial_intel
25d23
<
32,33c30,31
<       -DCMAKE_C_COMPILER=mpicc \
<       -DCMAKE_CXX_COMPILER=mpicxx \
---
>       -DCMAKE_C_COMPILER=icc \
>       -DCMAKE_CXX_COMPILER=icpc \

#############

and here are the two compilation scripts in full:

bash-4.2$ cat serial_intel/cn_compile.sh
#!/bin/bash


## set up any variables that are required for cmake or your compiler (this will be different on your cluster)
module purge
module load intel/17.0.4
module load cmake/3.9.0


edir=../../../exec/gromacs-5.1.2/serial_intel
mkdir -p $edir


## set the location of the single precision FFTW isntallation
export FFTW_LOCATION=/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec

##### Nothing below this line usually needs to be changed


## Note: get an error if allow avx2 so specify GMX_SIMD=AVX_256
##       This is likely because the gcc compiler is an old version
#      -DGMX_SIMD=AVX_256 \

{
cmake ../../../source/gromacs-5.1.2 \
      -DCMAKE_PREFIX_PATH=$FFTW_LOCATION \
      -DCMAKE_INSTALL_PREFIX=$(pwd)/$edir \
      -DGMX_X11=OFF \
      -DGMX_MPI=OFF \
      -DCMAKE_C_COMPILER=icc \
      -DCMAKE_CXX_COMPILER=icpc \
      -DGMX_PREFER_STATIC_LIBS=ON \
      -DGMX_GPU=OFF

make -j 8

make install -j 8
} > output.cn_compile 2>&1

#####

bash-4.2$ cat mpi_intel/cn_compile.sh
#!/bin/bash


## set up any variables that are required for cmake or your compiler (this will be different on your cluster)
module purge
module load intel/17.0.4
module load cmake/3.9.0
module load intel-mpi/2017.1


edir=../../../exec/gromacs-5.1.2/mpi_intel
mkdir -p $edir


## set the location of the single precision FFTW isntallation
export FFTW_LOCATION=/net/scratch3/cneale/exe/KODIAK/FFTW/fftw-3.3.4_intel/exec

##### Nothing below this line usually needs to be changed


## Note: get an error if allow avx2 so specify GMX_SIMD=AVX_256
##       This is likely because the gcc compiler is an old version
#      -DGMX_SIMD=AVX_256 \


{
cmake ../../../source/gromacs-5.1.2 \
      -DCMAKE_PREFIX_PATH=$FFTW_LOCATION \
      -DCMAKE_INSTALL_PREFIX=$(pwd)/$edir \
      -DGMX_X11=OFF \
      -DGMX_MPI=OFF \
      -DCMAKE_C_COMPILER=mpicc \
      -DCMAKE_CXX_COMPILER=mpicxx \
      -DGMX_PREFER_STATIC_LIBS=ON \
      -DGMX_GPU=OFF

make -j 8

make install -j 8
} > output.cn_compile 2>&1

###########

Thank you for your help,
Chris.