Hadoop

# Licensed to the Apache Software Foundation (ASF) under one

      # or more contributor license agreements.  See the NOTICE file

      # distributed with this work for additional information

      # regarding copyright ownership.  The ASF licenses this file

      # to you under the Apache License, Version 2.0 (the

      # "License"); you may not use this file except in compliance

      # with the License.  You may obtain a copy of the License at

      #

      #     http://www.apache.org/licenses/LICENSE-2.0

      #

      # Unless required by applicable law or agreed to in writing, software

      # distributed under the License is distributed on an "AS IS" BASIS,

      # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

      # See the License for the specific language governing permissions and

      # limitations under the License.

      # Set Hadoop-specific environment variables here.

      ##

      ## THIS FILE ACTS AS THE MASTER FILE FOR ALL HADOOP PROJECTS.

      ## SETTINGS HERE WILL BE READ BY ALL HADOOP COMMANDS.  THEREFORE,

      ## ONE CAN USE THIS FILE TO SET YARN, HDFS, AND MAPREDUCE

      ## CONFIGURATION OPTIONS INSTEAD OF xxx-env.sh.

      ##

      ## Precedence rules:

      ##

      ## {yarn-env.sh|hdfs-env.sh} > hadoop-env.sh > hard-coded defaults

      ##

      ## {YARN_xyz|HDFS_xyz} > HADOOP_xyz > hard-coded defaults

      ##

      # Many of the options here are built from the perspective that users

      # may want to provide OVERWRITING values on the command line.

      # For example:

      #

      #  JAVA_HOME=/usr/java/testing hdfs dfs -ls

      #

      # Therefore, the vast majority (BUT NOT ALL!) of these defaults

      # are configured for substitution and not append.  If append

      # is preferable, modify this file accordingly.

      # Technically, the only required environment variable is JAVA_HOME.

      # All others are optional.  However, the defaults are probably not

      # preferred.  Many sites configure these options outside of Hadoop,

      # such as in /etc/profile.d

      # The java implementation to use. By default, this environment

      # variable is REQUIRED on ALL platforms except OS X!

      export JAVA_HOME={{java_home}}

      # Location of Hadoop.  By default, Hadoop will attempt to determine

      # this location based upon its execution path.

      export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}

      #

      # Out of the box, Hadoop uses jsvc from Apache Commons to launch daemons

      # on privileged ports.  This functionality can be replaced by providing

      # custom functions.  See hadoop-functions.sh for more information.

      #

      # The jsvc implementation to use. Jsvc is required to run secure datanodes

      # that bind to privileged ports to provide authentication of data transfer

      # protocol.  Jsvc is not required if SASL is configured for authentication of

      # data transfer protocol using non-privileged ports.

      export JSVC_HOME={{jsvc_path}}

      # Location of Hadoop's configuration information.  i.e., where this

      # file is living. If this is not defined, Hadoop will attempt to

      # locate it based upon its execution path.

      #

      # NOTE: It is recommend that this variable not be set here but in

      # /etc/profile.d or equivalent.  Some options (such as

      # --config) may react strangely otherwise.

      #

      export HADOOP_CONF_DIR={{hadoop_conf_dir}}

      # The maximum amount of heap to use (Java -Xmx).  If no unit

      # is provided, it will be converted to MB.  Daemons will

      # prefer any Xmx setting in their respective _OPT variable.

      # There is no default; the JVM will autoscale based upon machine

      # memory size.

      export HADOOP_HEAPSIZE={{hadoop_heapsize}}

      export HADOOP_NAMENODE_HEAPSIZE={{namenode_heapsize}}

      # Enable extra debugging of Hadoop's JAAS binding, used to set up

      # Kerberos security.

      # export HADOOP_JAAS_DEBUG=true

      # Extra Java runtime options for all Hadoop commands. We don't support

      # IPv6 yet/still, so by default the preference is set to IPv4.

      # export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"

      # For Kerberos debugging, an extended option set logs more information

      # export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug"

      export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}"

      USER="$(whoami)"

      # Some parts of the shell code may do special things dependent upon

      # the operating system.  We have to set this here. See the next

      # section as to why....

      export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}

      {% if java_version < 8 %}

      SHARED_HDFS_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms$HADOOP_NAMENODE_HEAPSIZE -Xmx$HADOOP_NAMENODE_HEAPSIZE -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT"

      export HDFS_NAMENODE_OPTS="${SHARED_HDFS_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/odp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HDFS_NAMENODE_OPTS}"

      export HDFS_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:OnOutOfMemoryError=\"/usr/odp/current/hadoop-hdfs-datanode/bin/kill-data-node\" -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HDFS_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"

      export HDFS_SECONDARYNAMENODE_OPTS="${SHARED_HDFS_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/odp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HDFS_SECONDARYNAMENODE_OPTS}"

      # The following applies to multiple commands (fs, dfs, fsck, distcp etc)

      export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS"

      {% elif java_version == 8 %}

      SHARED_HDFS_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseG1GC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:InitiatingHeapOccupancyPercent=70 -Xms$HADOOP_NAMENODE_HEAPSIZE -Xmx$HADOOP_NAMENODE_HEAPSIZE -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT"

      export HDFS_NAMENODE_OPTS="${SHARED_HDFS_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/odp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HDFS_NAMENODE_OPTS}"

      export HDFS_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseG1GC -XX:OnOutOfMemoryError=\"/usr/odp/current/hadoop-hdfs-datanode/bin/kill-data-node\" -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HDFS_DATANODE_OPTS} -XX:InitiatingHeapOccupancyPercent=70"

      export HDFS_SECONDARYNAMENODE_OPTS="${SHARED_HDFS_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/odp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HDFS_SECONDARYNAMENODE_OPTS}"

      # The following applies to multiple commands (fs, dfs, fsck, distcp etc)

      export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS"

      {% else %}

      SHARED_HDFS_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseG1GC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xlog:gc*,gc+heap=debug,gc+phases=debug:file={{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'`:time,level,tags -XX:InitiatingHeapOccupancyPercent=70 -Xms$HADOOP_NAMENODE_HEAPSIZE -Xmx$HADOOP_NAMENODE_HEAPSIZE -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT"

      export HDFS_NAMENODE_OPTS="${SHARED_HDFS_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/odp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HDFS_NAMENODE_OPTS}"

      export HDFS_DATANODE_OPTS="-server -XX:OnOutOfMemoryError=\"/usr/odp/current/hadoop-hdfs-datanode/bin/kill-data-node\" -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log  -verbose:gc  -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HDFS_DATANODE_OPTS} "

      export HDFS_SECONDARYNAMENODE_OPTS="${SHARED_HDFS_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/odp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HDFS_SECONDARYNAMENODE_OPTS}"

      # The following applies to multiple commands (fs, dfs, fsck, distcp etc)

      export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS"

      {% endif %}

      {% if security_enabled %}

      export HDFS_NAMENODE_OPTS="$HDFS_NAMENODE_OPTS -Djava.security.auth.login.config={{hadoop_conf_dir}}/hdfs_nn_jaas.conf -Djavax.security.auth.useSubjectCredsOnly=false"

      ###

      # SecondaryNameNode specific parameters

      ###

      # Specify the JVM options to be used when starting the SecondaryNameNode.

      # These options will be appended to the options specified as HADOOP_OPTS

      # and therefore may override any similar flags set in HADOOP_OPTS

      #

      # This is the default:

      export HDFS_SECONDARYNAMENODE_OPTS="$HDFS_SECONDARYNAMENODE_OPTS -Djava.security.auth.login.config={{hadoop_conf_dir}}/hdfs_nn_jaas.conf -Djavax.security.auth.useSubjectCredsOnly=false"

      ###

      # DataNode specific parameters

      ###

      # Specify the JVM options to be used when starting the DataNode.

      # These options will be appended to the options specified as HADOOP_OPTS

      # and therefore may override any similar flags set in HADOOP_OPTS

      #

      # This is the default:

      export HDFS_DATANODE_OPTS="$HDFS_DATANODE_OPTS -Djava.security.auth.login.config={{hadoop_conf_dir}}/hdfs_dn_jaas.conf -Djavax.security.auth.useSubjectCredsOnly=false"

      ###

      # QuorumJournalNode specific parameters

      ###

      # Specify the JVM options to be used when starting the QuorumJournalNode.

      # These options will be appended to the options specified as HADOOP_OPTS

      # and therefore may override any similar flags set in HADOOP_OPTS

      #

      export HDFS_JOURNALNODE_OPTS="$HADOOP_JOURNALNODE_OPTS -Djava.security.auth.login.config={{hadoop_conf_dir}}/hdfs_jn_jaas.conf -Djavax.security.auth.useSubjectCredsOnly=false"

      {% endif %}

      ###

      # NFS3 Gateway specific parameters

      ###

      # Specify the JVM options to be used when starting the NFS3 Gateway.

      # These options will be appended to the options specified as HADOOP_OPTS

      # and therefore may override any similar flags set in HADOOP_OPTS

      #

      export HDFS_NFS3_OPTS="-Xmx{{nfsgateway_heapsize}}m -Dhadoop.security.logger=ERROR,DRFAS ${HDFS_NFS3_OPTS}"

      ###

      # HDFS Balancer specific parameters

      ###

      # Specify the JVM options to be used when starting the HDFS Balancer.

      # These options will be appended to the options specified as HADOOP_OPTS

      # and therefore may override any similar flags set in HADOOP_OPTS

      #

      export HDFS_BALANCER_OPTS="-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}"

      # Should HADOOP_CLASSPATH be first in the official CLASSPATH?

      # export HADOOP_USER_CLASSPATH_FIRST="yes"

      # If HADOOP_USE_CLIENT_CLASSLOADER is set, the classpath along

      # with the main jar are handled by a separate isolated

      # client classloader when 'hadoop jar', 'yarn jar', or 'mapred job'

      # is utilized. If it is set, HADOOP_CLASSPATH and

      # HADOOP_USER_CLASSPATH_FIRST are ignored.

      # export HADOOP_USE_CLIENT_CLASSLOADER=true

      # HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES overrides the default definition of

      # system classes for the client classloader when HADOOP_USE_CLIENT_CLASSLOADER

      # is enabled. Names ending in '.' (period) are treated as package names, and

      # names starting with a '-' are treated as negative matches. For example,

      # export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop."

      # Enable optional, bundled Hadoop features

      # This is a comma delimited list.  It may NOT be overridden via .hadooprc

      # Entries may be added/removed as needed.

      # export HADOOP_OPTIONAL_TOOLS="@@@HADOOP_OPTIONAL_TOOLS@@@"

      # On secure datanodes, user to run the datanode as after dropping privileges

      export HDFS_DATANODE_SECURE_USER=${HDFS_DATANODE_SECURE_USER:-{{hadoop_secure_dn_user}}}

      ###

      # Options for remote shell connectivity

      ###

      # There are some optional components of hadoop that allow for

      # command and control of remote hosts.  For example,

      # start-dfs.sh will attempt to bring up all NNs, DNS, etc.

      # Options to pass to SSH when one of the "log into a host and

      # start/stop daemons" scripts is executed

      export HADOOP_SSH_OPTS="-o BatchMode=yes -o SendEnv=HADOOP_CONF_DIR -o StrictHostKeyChecking=no -o ConnectTimeout=10s"

      # The built-in ssh handler will limit itself to 10 simultaneous connections.

      # For pdsh users, this sets the fanout size ( -f )

      # Change this to increase/decrease as necessary.

      # export HADOOP_SSH_PARALLEL=10

      # Filename which contains all of the hosts for any remote execution

      # helper scripts # such as workers.sh, start-dfs.sh, etc.

      # export HADOOP_WORKERS="${HADOOP_CONF_DIR}/workers"

      ###

      # Options for all daemons

      ###

      #

      #

      # Many options may also be specified as Java properties.  It is

      # very common, and in many cases, desirable, to hard-set these

      # in daemon _OPTS variables.  Where applicable, the appropriate

      # Java property is also identified.  Note that many are re-used

      # or set differently in certain contexts (e.g., secure vs

      # non-secure)

      #

      # Where (primarily) daemon log files are stored.

      # ${HADOOP_HOME}/logs by default.

      # Java property: hadoop.log.dir

      # Where log files are stored.  $HADOOP_HOME/logs by default.

      export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER

      # How many seconds to pause after stopping a daemon

      # export HADOOP_STOP_TIMEOUT=5

      # Where pid files are stored.  /tmp by default.

      export HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER

      ###

      # Secure/privileged execution

      ###

      #

      # This directory contains pids for secure and privileged processes.

      export HADOOP_SECURE_PID_DIR=${HADOOP_SECURE_PID_DIR:-{{hadoop_pid_dir_prefix}}/$HDFS_DATANODE_SECURE_USER}

      #

      # This directory contains the logs for secure and privileged processes.

      # Java property: hadoop.log.dir

      export HADOOP_SECURE_LOG=${HADOOP_SECURE_LOG:-{{hdfs_log_dir_prefix}}/$HDFS_DATANODE_SECURE_USER}

      YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY"

      #

      # When running a secure daemon, the default value of HADOOP_IDENT_STRING

      # ends up being a bit bogus.  Therefore, by default, the code will

      # replace HADOOP_IDENT_STRING with HADOOP_xx_SECURE_USER.  If one wants

      # to keep HADOOP_IDENT_STRING untouched, then uncomment this line.

      # export HADOOP_SECURE_IDENT_PRESERVE="true"

      # A string representing this instance of hadoop. $USER by default.

      # This is used in writing log and pid files, so keep that in mind!

      # Java property: hadoop.id.str

      export HADOOP_IDENT_STRING=$USER

      # Add database libraries

      JAVA_JDBC_LIBS=""

      if [ -d "/usr/share/java" ]; then

      for jarFile in `ls /usr/share/java | grep -E "(mysql|ojdbc|postgresql|sqljdbc)" 2>/dev/null`

      do

      JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile

      done

      fi

      # Add libraries to the hadoop classpath - some may not need a colon as they already include it

      export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}

      # Setting path to hdfs command line

      export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}

      # Mostly required for hadoop 2.0

      export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64

      {% if zk_principal_user is defined %}

      HADOOP_OPTS="-Dzookeeper.sasl.client.username={{zk_principal_user}} $HADOOP_OPTS"

      {% endif %}

      export HADOOP_OPTS="-Dodp.version=$ODP_VERSION $HADOOP_OPTS"

      # Fix temporary bug, when ulimit from conf files is not picked up, without full relogin.

      # Makes sense to fix only when runing DN as root

      if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HDFS_DATANODE_SECURE_USER" ]; then

      {% if is_datanode_max_locked_memory_set %}

      ulimit -l {{datanode_max_locked_memory}}

      {% endif %}

      ulimit -n {{hdfs_user_nofile_limit}}

      fi

      ###

      # ZKFailoverController specific parameters

      ###

      # Specify the JVM options to be used when starting the ZKFailoverController.

      # These options will be appended to the options specified as HADOOP_OPTS

      # and therefore may override any similar flags set in HADOOP_OPTS

      #

      # Enable ACLs on zookeper znodes if required

      {% if hadoop_zkfc_opts is defined %}

      export HDFS_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HDFS_ZKFC_OPTS"

      {% endif %}

      ###

      # HDFS Mover specific parameters

      ###

      # Specify the JVM options to be used when starting the HDFS Mover.

      # These options will be appended to the options specified as HADOOP_OPTS

      # and therefore may override any similar flags set in HADOOP_OPTS

      #

      # export HDFS_MOVER_OPTS=""

      ###

      # Router-based HDFS Federation specific parameters

      # Specify the JVM options to be used when starting the RBF Routers.

      # These options will be appended to the options specified as HADOOP_OPTS

      # and therefore may override any similar flags set in HADOOP_OPTS

      #

      # export HDFS_DFSROUTER_OPTS=""

      ###

      # HDFS StorageContainerManager specific parameters

      ###

      # Specify the JVM options to be used when starting the HDFS Storage Container Manager.

      # These options will be appended to the options specified as HADOOP_OPTS

      # and therefore may override any similar flags set in HADOOP_OPTS

      #

      # export HDFS_STORAGECONTAINERMANAGER_OPTS=""

      ###

      # Advanced Users Only!

      ###

      #

      # When building Hadoop, one can add the class paths to the commands

      # via this special env var:

      # export HADOOP_ENABLE_BUILD_PATHS="true"

      #

      # To prevent accidents, shell commands be (superficially) locked

      # to only allow certain users to execute certain subcommands.

      # It uses the format of (command)_(subcommand)_USER.

      #

      # For example, to limit who can execute the namenode command,

      # export HDFS_NAMENODE_USER=hdfs