Hadoop DataNode not starting -Apache Bigtop install using puppet with kerberos - hadoop

Question

I am trying to deploy hadoop using Apache Bigtop 3.1.1 with puppet.Hadoop version is 3.2.4 .
OS I am using is CentOS 7.
Deployment works fine without kerberos. But with kerberos, hadoop datanode not stating up. Tried to start it manually with
sudo systemctl start hadoop-hdfs-datanode.service and
this is the error it gives:
Nov 21 12:15:59 master.local hadoop-hdfs-datanode[13646]: ERROR: You must be a privileged user in order to run a secure service. Nov 21 12:16:04 master.local hadoop-hdfs-datanode[13646]: Failed to start Hadoop datanode. Return value: 3[FAILED] Nov 21 12:16:04 master.local systemd[1]: hadoop-hdfs-datanode.service: control process exited, code=exited status=3 Nov 21 12:16:04 master.local systemd[1]: Failed to start LSB: Hadoop datanode.
This is my site.yaml file
---
bigtop::hadoop_head_node: "master.local"
hadoop::hadoop_storage_dirs:
- /data/1
- /data/2
- /data/3
- /data/4
hadoop_cluster_node::cluster_components:
- hdfs
- spark
- hive
- tez
- sqoop
- zookeeper
- kafka
- livy
- oozie
- zeppelin
- solrcloud
- kerberos
- httpfs
bigtop::bigtop_repo_uri: "http://10.42.65.70:90/bigtop/3.1.1/rpm/"
# - "https://archive.apache.org/dist/bigtop/bigtop-3.1.1/repos/centos-7/"
hadoop::common_hdfs::hadoop_http_authentication_signature_secret: "FaztheBits123!"
# Kerberos
hadoop::hadoop_security_authentication: "kerberos"
kerberos::krb_site::domain: "bigtop.apache.org"
kerberos::krb_site::realm: "BIGTOP.APACHE.ORG"
kerberos::krb_site::kdc_server: "%{hiera('bigtop::hadoop_head_node')}"
kerberos::krb_site::kdc_port: "88"
kerberos::krb_site::admin_port: "749"
kerberos::krb_site::keytab_export_dir: "/var/lib/bigtop_keytabs"
hadoop::common_hdfs::hadoop_http_authentication_type: "%{hiera('hadoop::hadoop_security_authentication')}"
# to enable tez in hadoop, uncomment the lines below
hadoop::common::use_tez: true
hadoop::common_mapred_app::mapreduce_framework_name: "yarn-tez"
# to enable tez in hive, uncomment the lines below
hadoop_hive::common_config::hive_execution_engine: "tez"
I tried changing workers in /etc/hadoop/conf/workers with proper hostnames. My core-site.xml is this:
<configuration>
<property>
<!-- URI of NN. Fully qualified. No IP.-->
<name>fs.defaultFS</name>
<value>hdfs://master.local:8020</value>
</property>
<property>
<name>hadoop.security.authentication</name>
<value>kerberos</value>
</property>
<property>
<name>hadoop.security.authorization</name>
<value>true</value>
</property>
<property>
<name>hadoop.proxyuser.hive.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.groups</name>
<value>hudson,testuser,root,hadoop,jenkins,oozie,hive,httpfs,users</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.groups</name>
<value>hudson,testuser,root,hadoop,jenkins,oozie,hive,httpfs,users</value>
</property>
<property>
<name>hadoop.proxyuser.oozie.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.oozie.groups</name>
<value>hudson,testuser,root,hadoop,jenkins,oozie,hive,httpfs,users</value>
</property>
<!-- enable proper authentication instead of static mock authentication as
Dr. Who -->
<property>
<name>hadoop.http.filter.initializers</name>
<value>org.apache.hadoop.security.AuthenticationFilterInitializer</value>
</property>
<!-- disable anonymous access -->
<property>
<name>hadoop.http.authentication.simple.anonymous.allowed</name>
<value>false</value>
</property>
<!-- enable kerberos authentication -->
<property>
<name>hadoop.http.authentication.type</name>
<value>kerberos</value>
</property>
<property>
<name>hadoop.http.authentication.kerberos.principal</name>
<value>HTTP/_HOST#BIGTOP.APACHE.ORG</value>
</property>
<property>
<name>hadoop.http.authentication.kerberos.keytab</name>
<value>/etc/HTTP.keytab</value>
</property>
<!-- provide secret for cross-service-cross-machine cookie -->
<property>
<name>hadoop.http.authentication.signature.secret.file</name>
<value>/etc/hadoop/conf/hadoop-http-authentication-signature-secret</value>
</property>
<!-- make all services on all hosts use the same cookie domain -->
<property>
<name>hadoop.http.authentication.cookie.domain</name>
<value>local</value>
</property>
<property>
<name>hadoop.security.key.provider.path</name>
<value>kms://http#master.local:9600/kms</value>
</property>
</configuration>
My hdfs-site.xml is:
<configuration>
<!-- non HA -->
<property>
<name>dfs.namenode.rpc-address</name>
<value>master.local:8020</value>
</property>
<property>
<name>dfs.namenode.http-address</name>
<value>master.local:50070</value>
</property>
<property>
<name>dfs.namenode.https-address</name>
<value>master.local:50470</value>
</property>
<property>
<name>dfs.block.access.token.enable</name>
<value>true</value>
</property>
<!-- NameNode security config -->
<property>
<name>dfs.https.address</name>
<value>master.local:50475</value>
</property>
<property>
<name>dfs.https.port</name>
<value>50475</value>
</property>
<property>
<name>dfs.namenode.keytab.file</name>
<value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
</property>
<property>
<name>dfs.namenode.kerberos.principal</name>
<value>hdfs/_HOST#BIGTOP.APACHE.ORG</value>
</property>
<property>
<name>dfs.namenode.kerberos.https.principal</name>
<value>host/_HOST#BIGTOP.APACHE.ORG</value>
</property>
<property>
<name>dfs.web.authentication.kerberos.keytab</name>
<value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
</property>
<property>
<name>dfs.web.authentication.kerberos.principal</name>
<value>HTTP/_HOST#BIGTOP.APACHE.ORG</value>
</property>
<!-- Secondary NameNode security config -->
<property>
<name>dfs.secondary.http.address</name>
<value>master.local:0</value>
</property>
<property>
<name>dfs.secondary.https.address</name>
<value>master.local:50495</value>
</property>
<property>
<name>dfs.secondary.https.port</name>
<value>50495</value>
</property>
<property>
<name>dfs.secondary.namenode.keytab.file</name>
<value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
</property>
<property>
<name>dfs.secondary.namenode.kerberos.principal</name>
<value>hdfs/_HOST#BIGTOP.APACHE.ORG</value>
</property>
<property>
<name>dfs.secondary.namenode.kerberos.https.principal</name>
<value>host/_HOST#BIGTOP.APACHE.ORG</value>
</property>
<!-- DataNode security config -->
<property>
<name>dfs.datanode.data.dir.perm</name>
<value>700</value>
</property>
<property>
<name>dfs.datanode.address</name>
<value>0.0.0.0:1004</value>
</property>
<property>
<name>dfs.datanode.http.address</name>
<value>0.0.0.0:1006</value>
</property>
<property>
<name>dfs.datanode.keytab.file</name>
<value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
</property>
<property>
<name>dfs.datanode.kerberos.principal</name>
<value>hdfs/_HOST#BIGTOP.APACHE.ORG</value>
</property>
<property>
<name>dfs.datanode.kerberos.https.principal</name>
<value>host/_HOST#BIGTOP.APACHE.ORG</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:///data/1/hdfs,file:///data/2/hdfs,file:///data/3/hdfs,file:///data/4/hdfs</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:///data/1/namenode,file:///data/2/namenode,file:///data/3/namenode,file:///data/4/namenode</value>
</property>
<property>
<name>dfs.permissions.superusergroup</name>
<value>hadoop</value>
<description>The name of the group of super-users.</description>
</property>
<!-- increase the number of datanode transceivers way above the default of 256
- this is for hbase -->
<property>
<name>dfs.datanode.max.xcievers</name>
<value>4096</value>
</property>
<!-- Configurations for large cluster -->
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
</configuration>
I don't seem to find log for datanode. I can find logs for namenode in
/var/log/hadoop-hdfs but not datanode logs.
What am I doing wrong.