Hadoop native libraries not found on OS/X - macos

Question

I have downloaded hadoop source code from github and compiled with the native option:
mvn package -Pdist,native -DskipTests -Dtar -Dmaven.javadoc.skip=true
I then copied the .dylib files to the $HADOOP_HOME/lib
cp -p hadoop-common-project/hadoop-common/target/hadoop-common-2.7.1/lib/native/*.dylib /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/lib
The LD_LIBRARY_PATH was updated and hdfs restarted:
echo $LD_LIBRARY_PATH
/usr/local/Cellar/hadoop/2.7.2/libexec/lib:
/usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/lib:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home//jre/lib
(Note: this also means that the answer to Hadoop “Unable to load native-hadoop library for your platform” error on docker-spark? does not work for me..)
But checknative still returns uniformly false:
$stop-dfs.sh && start-dfs.sh && hadoop checknative
16/06/13 16:12:32 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Stopping namenodes on [sparkbook]
sparkbook: stopping namenode
localhost: stopping datanode
Stopping secondary namenodes [0.0.0.0]
0.0.0.0: stopping secondarynamenode
16/06/13 16:12:50 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/06/13 16:12:50 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Starting namenodes on [sparkbook]
sparkbook: starting namenode, logging to /usr/local/Cellar/hadoop/2.7.2/libexec/logs/hadoop-macuser-namenode-sparkbook.out
localhost: starting datanode, logging to /usr/local/Cellar/hadoop/2.7.2/libexec/logs/hadoop-macuser-datanode-sparkbook.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /usr/local/Cellar/hadoop/2.7.2/libexec/logs/hadoop-macuser-secondarynamenode-sparkbook.out
16/06/13 16:13:05 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/06/13 16:13:05 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Native library checking:
hadoop: false
zlib: false
snappy: false
lz4: false
bzip2: false
openssl: false

Answer 1

To get this working on a fresh install of macOS 10.12, I had to do the following:
Install build dependencies using homebrew:
brew install cmake maven openssl protobuf#2.5 snappy
Check out hadoop source code
git clone https://github.com/apache/hadoop.git
cd hadoop
git checkout rel/release-2.7.3
Apply the below patch to the build:
diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
index 942b19c..8b34881 100644
--- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt
+++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
## -16,6 +16,8 ##
# limitations under the License.
#
+SET(CUSTOM_OPENSSL_PREFIX /usr/local/opt/openssl)
+
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
# Default to release builds
## -116,8 +118,8 ## set(T main/native/src/test/org/apache/hadoop)
GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME)
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
-set_find_shared_library_version("1")
-find_package(BZip2 QUIET)
+set_find_shared_library_version("1.0")
+find_package(BZip2 REQUIRED)
if (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
GET_FILENAME_COMPONENT(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME)
set(BZIP2_SOURCE_FILES
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
index d2ddf89..ac8e351 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
+++ b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
## -17,4 +17,8 ##
<!-- Put site-specific property overrides in this file. -->
<configuration>
+<property>
+<name>io.compression.codec.bzip2.library</name>
+<value>libbz2.dylib</value>
+</property>
</configuration>
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml
index 34c0110..70f23a4 100644
--- a/hadoop-tools/hadoop-pipes/pom.xml
+++ b/hadoop-tools/hadoop-pipes/pom.xml
## -52,7 +52,7 ##
<mkdir dir="${project.build.directory}/native"/>
<exec executable="cmake" dir="${project.build.directory}/native"
failonerror="true">
- <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
+ <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl"/>
</exec>
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
<arg line="VERBOSE=1"/>
Build hadoop from source:
mvn package -Pdist,native -DskipTests -Dtar -Dmaven.javadoc.skip=true
Specify JAVA_LIBRARY_PATH when running hadoop:
$ JAVA_LIBRARY_PATH=/usr/local/opt/openssl/lib:/opt/local/lib:/usr/lib hadoop-dist/target/hadoop-2.7.3/bin/hadoop checknative -a
16/10/14 20:16:32 INFO bzip2.Bzip2Factory: Successfully loaded & initialized native-bzip2 library libbz2.dylib
16/10/14 20:16:32 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library
Native library checking:
hadoop: true /Users/admin/Desktop/hadoop/hadoop-dist/target/hadoop-2.7.3/lib/native/libhadoop.dylib
zlib: true /usr/lib/libz.1.dylib
snappy: true /usr/local/lib/libsnappy.1.dylib
lz4: true revision:99
bzip2: true /usr/lib/libbz2.1.0.dylib
openssl: true /usr/local/opt/openssl/lib/libcrypto.dylib

Answer 2

There are some missing steps in #andrewdotn's response above:
1) For step (3), create the patch by adding the text posted to a text file e.g. "patch.txt", and then execute "git apply patch.txt"
2) In addition to copying the files as directed by javadba, certain applications also require that you set:
export HADOOP_OPTS="-Djava.library.path=${HADOOP_HOME}/lib/native"
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${HADOOP_HOME}/lib/native
export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:${HADOOP_HOME}/lib/native

Answer 3

The needed step is to copy the *.dylib from the git sources build dir into the $HADOOP_HOME/<common dir>lib dir for your platform . For OS/X installed via brew it is:
cp /git/hadoop/hadoop-dist/target/hadoop-2.7.1/lib/native/ /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/
We can see the required libs there now:
$ll /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/*.dylib
-rwxr-xr-x 1 macuser staff 149100 Jun 13 13:44 /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.dylib
-rwxr-xr-x 1 macuser staff 149100 Jun 13 13:44 /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.1.0.0.dylib
And now the hadoop checknative command works:
$hadoop checknative
6/06/15 09:10:59 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library
Native library checking:
hadoop: true /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.dylib
zlib: true /usr/lib/libz.1.dylib
snappy: false
lz4: true revision:99
bzip2: false
openssl: false build does not support openssl.

Answer 4

As an update to #andrewdotn answer, here is the patch.txt file to be used with Hadoop 2.8.1:
diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
index c93bfe78546..e8918f9ca29 100644
--- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt
+++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
## -20,6 +20,8 ##
# CMake configuration.
#
+SET(CUSTOM_OPENSSL_PREFIX /usr/local/opt/openssl)
+
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/..)
## -50,8 +52,8 ## get_filename_component(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME)
# Look for bzip2.
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
-hadoop_set_find_shared_library_version("1")
-find_package(BZip2 QUIET)
+hadoop_set_find_shared_library_version("1.0")
+find_package(BZip2 REQUIRED)
if(BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
get_filename_component(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME)
set(BZIP2_SOURCE_FILES
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
index d2ddf893e49..ac8e351f1c8 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
+++ b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
## -17,4 +17,8 ##
<!-- Put site-specific property overrides in this file. -->
<configuration>
+<property>
+<name>io.compression.codec.bzip2.library</name>
+<value>libbz2.dylib</value>
+</property>
</configuration>
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml
index 8aafad0f7eb..d4832542265 100644
--- a/hadoop-tools/hadoop-pipes/pom.xml
+++ b/hadoop-tools/hadoop-pipes/pom.xml
## -55,7 +55,7 ##
<mkdir dir="${project.build.directory}/native"/>
<exec executable="cmake" dir="${project.build.directory}/native"
failonerror="true">
- <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
+ <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl"/>
</exec>
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
<arg line="VERBOSE=1"/>