[SPARK-717] Enable statsd sink to monitor Spark metrics (#402)

* [SPARK-717] Enable statsd sink to monitor Spark metrics (#290)
- add spark-statsd jar for the StatsDSink
- update spark-statsd jar to use versions Spark 2.2.1 and Scala 2.11 (#393)

Note: must enable UCR to output metrics

conf/metrics.properties.template

@@ -0,0 +1,9 @@
+
+# Enable StatsdSink for all instances by class name
+*.sink.statsd.class=org.apache.spark.metrics.sink.StatsDSink
+*.sink.statsd.prefix=spark
+*.sink.statsd.host=<STATSD_UDP_HOST>
+*.sink.statsd.port=<STATSD_UDP_PORT>
+
+# Enable JVM metrics source for all instances (master, worker, driver and executor) by class name
+*.source.jvm.class=org.apache.spark.metrics.source.JvmSource

conf/spark-env.sh

@@ -12,7 +12,7 @@ mkdir -p "${HADOOP_CONF_DIR}"
 [ -f "${MESOS_SANDBOX}/hdfs-site.xml" ] && cp "${MESOS_SANDBOX}/hdfs-site.xml" "${HADOOP_CONF_DIR}"
 [ -f "${MESOS_SANDBOX}/core-site.xml" ] && cp "${MESOS_SANDBOX}/core-site.xml" "${HADOOP_CONF_DIR}"
 
-
+SPARK_WORKING_DIR=${PWD}
 
 cd $MESOS_SANDBOX
 
@@ -87,6 +87,14 @@ else
     echo "spark-env: StatsD metrics require Mesos UCR. For dispatcher metrics, enable the 'UCR_containerizer' option. For driver metrics, include '--conf spark.mesos.containerizer=mesos' in your run"
 fi
 
+if [ -n "${STATSD_UDP_HOST}" ] && [ -n "${STATSD_UDP_PORT}" ]; then
+    sed -e "s/<STATSD_UDP_HOST>/${STATSD_UDP_HOST}/g" \
+        -e "s/<STATSD_UDP_PORT>/${STATSD_UDP_PORT}/g" \
+        ${SPARK_WORKING_DIR}/conf/metrics.properties.template >${SPARK_WORKING_DIR}/conf/metrics.properties
+else
+    echo "spark-env: No STATSD_UDP environment variables were defined" >&2
+fi
+
 # Options read when launching programs locally with
 # ./bin/run-example or ./bin/spark-submit
 # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files

docker/Dockerfile

@@ -77,6 +77,10 @@ RUN ln -s /var/lib/runit/service/nginx /etc/service/nginx
 ADD dist /opt/spark/dist
 ADD krb5.conf.mustache /etc/
 
+# Add external jar for pushing metrics via StatsD (https://github.com/vidhyaarvind/spark-statsd)
+# current version: Spark 2.2.1, Scala 2.11
+ADD jars/spark-statsd-2.2.1-2.11.jar /opt/spark/dist/jars/  
+
 # required to run as nobody
 RUN chmod -R ugo+rw /etc/nginx
 RUN chmod -R ugo+rw /etc/krb5.conf

docker/runit/service/spark/run

@@ -31,6 +31,14 @@ function set_log_level() {
         ${SPARK_DIST_PATH}/conf/log4j.properties.template >${SPARK_DIST_PATH}/conf/log4j.properties
 }
 
+function set_statsd_address() {
+  if [ "$STATSD_UDP_HOST" != "" ] && [ "$STATSD_UDP_PORT" != "" ]; then
+    sed -e "s/<STATSD_UDP_HOST>/${STATSD_UDP_HOST}/g" \
+        -e "s/<STATSD_UDP_PORT>/${STATSD_UDP_PORT}/g" \
+        ${SPARK_DIST_PATH}/conf/metrics.properties.template >${SPARK_DIST_PATH}/conf/metrics.properties
+  fi
+}
+
 function add_if_non_empty() {
 	if [ -n "$2" ]; then
 		echo "$1=$2" >> ${SPARK_DIST_PATH}/conf/mesos-cluster-dispatcher.properties
@@ -61,6 +69,7 @@ function configure_properties() {
 
 export APPLICATION_WEB_PROXY_BASE="${DISPATCHER_UI_WEB_PROXY_BASE}"
 set_log_level
+set_statsd_address
 export_daemon_opts
 configure_properties
 ZK="master.mesos:2181"