120 lines
3.1 KiB
Nix
120 lines
3.1 KiB
Nix
{ config, lib, pkgs, ... }:
|
|
let sparkConfDir = pkgs.stdenv.mkDerivation {
|
|
name = "spark-config";
|
|
dontUnpack = true;
|
|
installPhase = ''
|
|
# source standard environment
|
|
. $stdenv/setup
|
|
|
|
# shorthands
|
|
base_conf=${pkgs.spark}/lib/${pkgs.spark.untarDir}/conf/
|
|
|
|
# create output dirs for new derivation
|
|
mkdir -p $out/
|
|
|
|
# link unchanged files from the original gnome-session
|
|
for f in $base_conf/*.template ; do
|
|
ln -sf $f $out/
|
|
done
|
|
|
|
# change selected files
|
|
cp $out/log4j.properties{.template,}
|
|
|
|
cat > $out/spark-env.sh <<- STOP
|
|
export JAVA_HOME="${pkgs.jdk8}"
|
|
export SPARK_HOME="${pkgs.spark}/lib/${pkgs.spark.untarDir}"
|
|
export SPARK_DIST_CLASSPATH=$(${pkgs.hadoop}/bin/hadoop classpath)
|
|
export PYSPARK_PYTHON="${pkgs.python3Packages.python}/bin/${pkgs.python3Packages.python.executable}"
|
|
export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH"
|
|
export SPARKR_R_SHELL="${pkgs.R}/bin/R"
|
|
export PATH="\$PATH:${pkgs.R}/bin"
|
|
STOP
|
|
|
|
cat > $out/spark-defaults.conf <<- STOP
|
|
spark.eventLog.enabled true
|
|
spark.eventLog.dir hdfs://localhost:/logs/spark
|
|
spark.history.fs.logDirectory hdfs://localhost:/logs/spark
|
|
STOP
|
|
'';
|
|
};
|
|
in
|
|
{
|
|
|
|
services = {
|
|
spark = {
|
|
master = {
|
|
enable = true;
|
|
restartIfChanged = true;
|
|
};
|
|
worker = {
|
|
enable = true;
|
|
restartIfChanged = true;
|
|
};
|
|
confDir = sparkConfDir;
|
|
};
|
|
|
|
hadoop = {
|
|
coreSite = {
|
|
"fs.defaultFS" = "hdfs://localhost:8020";
|
|
};
|
|
hdfsSite = {
|
|
"dfs.namenode.rpc-bind-host" = "0.0.0.0";
|
|
"dfs.permissions" = "false";
|
|
"dfs.namenode.name.dir" = "/hdfs/dfs/name";
|
|
"dfs.datanode.data.dir" = "/hdfs/dfs/data";
|
|
};
|
|
|
|
hdfs = {
|
|
namenode = {
|
|
enable = true;
|
|
formatOnInit = true;
|
|
restartIfChanged = true;
|
|
};
|
|
datanode = {
|
|
enable = true;
|
|
restartIfChanged = true;
|
|
};
|
|
journalnode = {
|
|
enable = true;
|
|
restartIfChanged = true;
|
|
};
|
|
zkfc = {
|
|
enable = true;
|
|
restartIfChanged = true;
|
|
};
|
|
httpfs = {
|
|
enable = true;
|
|
restartIfChanged = true;
|
|
};
|
|
};
|
|
yarn = {
|
|
resourcemanager.enable = true;
|
|
nodemanager.enable = true;
|
|
};
|
|
};
|
|
};
|
|
|
|
systemd.services.spark-history = {
|
|
path = with pkgs; [ procps openssh nettools ];
|
|
description = "spark history service.";
|
|
after = [ "network.target" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
restartIfChanged = true;
|
|
environment = {
|
|
SPARK_CONF_DIR = sparkConfDir;
|
|
SPARK_LOG_DIR = "/var/log/spark";
|
|
};
|
|
serviceConfig = {
|
|
Type = "forking";
|
|
User = "spark";
|
|
Group = "spark";
|
|
WorkingDirectory = "${pkgs.spark}/lib/${pkgs.spark.untarDir}";
|
|
ExecStart = "${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/start-history-server.sh";
|
|
ExecStop = "${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/stop-history-server.sh";
|
|
TimeoutSec = 300;
|
|
StartLimitBurst = 10;
|
|
Restart = "always";
|
|
};
|
|
};
|
|
|
|
}
|