265 lines
8.3 KiB
Nix
265 lines
8.3 KiB
Nix
{ config, lib, pkgs, ... }:
|
|
let sparkConfDir = pkgs.stdenv.mkDerivation {
|
|
name = "spark-config";
|
|
dontUnpack = true;
|
|
installPhase = ''
|
|
# source standard environment
|
|
. $stdenv/setup
|
|
|
|
# shorthands
|
|
base_conf=${pkgs.spark}/lib/${pkgs.spark.untarDir}/conf/
|
|
|
|
# create output dirs for new derivation
|
|
mkdir -p $out/
|
|
|
|
# link unchanged files from the original gnome-session
|
|
for f in $base_conf/*.template ; do
|
|
ln -sf $f $out/
|
|
done
|
|
|
|
# change selected files
|
|
cp $out/log4j.properties{.template,}
|
|
|
|
cat > $out/spark-env.sh <<- STOP
|
|
export JAVA_HOME="${pkgs.jdk8}"
|
|
export SPARK_HOME="${pkgs.spark}/lib/${pkgs.spark.untarDir}"
|
|
export SPARK_DIST_CLASSPATH=$(${pkgs.hadoop}/bin/hadoop classpath)
|
|
export PYSPARK_PYTHON="${pkgs.python3Packages.python}/bin/${pkgs.python3Packages.python.executable}"
|
|
export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH"
|
|
export SPARKR_R_SHELL="${pkgs.R}/bin/R"
|
|
export PATH="\$PATH:${pkgs.R}/bin"
|
|
STOP
|
|
|
|
cat > $out/spark-defaults.conf <<- STOP
|
|
spark.eventLog.enabled true
|
|
spark.eventLog.dir hdfs://localhost:/logs/spark
|
|
spark.history.fs.logDirectory hdfs://localhost:/logs/spark
|
|
STOP
|
|
'';
|
|
};
|
|
in
|
|
{
|
|
|
|
networking = {
|
|
hosts = {
|
|
"127.0.0.1" = [
|
|
"ds.my.engine"
|
|
"kdc.my.engine"
|
|
"my.engine"
|
|
];
|
|
};
|
|
|
|
};
|
|
|
|
services = {
|
|
spark = {
|
|
master = {
|
|
enable = true;
|
|
restartIfChanged = true;
|
|
};
|
|
worker = {
|
|
enable = true;
|
|
restartIfChanged = true;
|
|
};
|
|
confDir = sparkConfDir;
|
|
};
|
|
|
|
|
|
|
|
hadoop =
|
|
let
|
|
keytab_path = /etc/hadoop.keytab;
|
|
in
|
|
|
|
{
|
|
coreSite = {
|
|
# "fs.defaultFS" = "hdfs://0.0.0.0:8020";
|
|
|
|
# "hadoop.http.authentication.simple.anonymous.allowed" = "false";
|
|
# "hadoop.http.authentication.signature.secret.file" = "/var/lib/hadoop/security/http_secret";
|
|
# "hadoop.http.authentication.type" = "kerberos";
|
|
# "hadoop.http.authentication.kerberos.principal" = "http/my.engine@MY.ENGINE";
|
|
# "hadoop.http.authentication.cookie.domain" = "my.engine";
|
|
|
|
# "hadoop.security.authentication" = "kerberos";
|
|
# "hadoop.security.authorization" = "true";
|
|
# "hadoop.rpc.protection" = "authentication";
|
|
|
|
|
|
# "hadoop.rpc.protection" = "authentication";
|
|
# "hadoop.security.auth_to_local" = ''
|
|
# RULE:[2:$1/$2@$0]([ndj]n/.*@MY.ENGINE)s/.*/hdfs/
|
|
# RULE:[2:$1/$2@$0]([rn]m/.*@MY.ENGINE)s/.*/yarn/
|
|
# RULE:[2:$1/$2@$0](jhs/.*@MY.ENGINE)s/.*/mapred/
|
|
# DEFAULT
|
|
# '';
|
|
# "hadoop.proxyuser.superuser.hosts" = "*"; # TODO: restrict
|
|
# "hadoop.proxyuser.superuser.groups" = "*"; # TODO: restrict
|
|
|
|
"fs.defaultFS" = "hdfs://my.engine:8020";
|
|
|
|
# HDFS IMPERSONATION
|
|
"hadoop.proxyuser.hdfs.hosts" = "*";
|
|
"hadoop.proxyuser.hdfs.groups" = "*";
|
|
|
|
# HIVE IMPERSONATION
|
|
"hadoop.proxyuser.hive.hosts" = "*";
|
|
"hadoop.proxyuser.hive.groups" = "*";
|
|
|
|
# ENABLE AUTHENTICATION
|
|
"hadoop.security.authentication" = "kerberos";
|
|
"hadoop.security.authorization" = "true";
|
|
"hadoop.rpc.protection" = "privacy";
|
|
|
|
"hadoop.security.auth_to_local" = ''
|
|
RULE:[2:$1/$2@$0]([ndj]n/.*@MY\.ENGINE)s/.*/hdfs/
|
|
RULE:[2:$1/$2@$0]([rn]m/.*@MY\.ENGINE)s/.*/yarn/
|
|
RULE:[2:$1/$2@$0](jhs/.*@MY\.ENGINE)s/.*/mapred/
|
|
DEFAULT
|
|
'';
|
|
};
|
|
hdfsSite = {
|
|
# DATA
|
|
"dfs.namenode.name.dir" = "/hdfs/dfs/name";
|
|
"dfs.datanode.data.dir" = "/hdfs/dfs/data";
|
|
|
|
# HDFS SECURITY
|
|
"dfs.block.access.token.enable" = "true";
|
|
|
|
# NAME NODE SECURITY
|
|
"dfs.namenode.keytab.file" = keytab_path;
|
|
"dfs.namenode.kerberos.principal" = "nn/my.engine@MY.ENGINE";
|
|
"dfs.namenode.kerberos.internal.spnego.principal" = "HTTP/my.engine@MY.ENGINE";
|
|
|
|
# SECONDARY NAME NODE SECURITY
|
|
"dfs.secondary.namenode.keytab.file" = keytab_path;
|
|
"dfs.secondary.namenode.kerberos.principal" = "nn/my.engine@MY.ENGINE";
|
|
"dfs.secondary.namenode.kerberos.internal.spnego.principal" = "HTTP/my.engine@MY.ENGINE";
|
|
|
|
# DATA NODE SECURITY
|
|
"dfs.datanode.keytab.file" = keytab_path;
|
|
"dfs.datanode.kerberos.principal" = "dn/my.engine@MY.ENGINE";
|
|
|
|
# WEBHDFS SECURITY
|
|
"dfs.webhdfs.enabled" = "true";
|
|
|
|
# WEB AUTHENTICATION CONFIG
|
|
"dfs.web.authentication.kerberos.principal" = "HTTP/my.engine@MY.ENGINE";
|
|
"dfs.web.authentication.kerberos.keytab" = keytab_path;
|
|
"ignore.secure.ports.for.testing" = "true";
|
|
"dfs.http.policy" = "HTTP_ONLY";
|
|
"dfs.data.transfer.protection" = "privacy";
|
|
|
|
# ## MULTIHOMED
|
|
# "dfs.namenode.rpc-bind-host" = "0.0.0.0";
|
|
# "dfs.namenode.servicerpc-bind-host" = "0.0.0.0";
|
|
# "dfs.namenode.http-bind-host" = "0.0.0.0";
|
|
# "dfs.namenode.https-bind-host" = "0.0.0.0";
|
|
# "dfs.client.use.datanode.hostname" = "true"; # force connection by hostname
|
|
# "dfs.datanode.use.datanode.hostname" = "true"; # force connection by hostname
|
|
|
|
|
|
# "dfs.data.transfer.protection" = "privacy";
|
|
# "hadoop.rpc.protection" = "privacy";
|
|
# "dfs.http.policy" = "HTTP_ONLY";
|
|
# "dfs.datanode.address" = "0.0.0.0:10019";
|
|
# "dfs.datanode.http.address" = "0.0.0.0:10022";
|
|
# "dfs.datanode.https.address" = "0.0.0.0:10023";
|
|
|
|
|
|
# "dfs.datanode.kerberos.principal" = "dn/my.engine@MY.ENGINE";
|
|
# "dfs.datanode.keytab.file" = keytab_path;
|
|
|
|
# "dfs.namenode.kerberos.principal" = "nn/my.engine@MY.ENGINE";
|
|
# "dfs.namenode.keytab.file" = keytab_path;
|
|
|
|
# "dfs.block.access.token.enable" = "true";
|
|
|
|
|
|
};
|
|
yarnSite = {
|
|
# "yarn.acl.enable" = "true";
|
|
# "yarn.admin.acl" = "*"; # TODO: restrict
|
|
};
|
|
extraConfDirs = [ ];
|
|
|
|
hdfs = {
|
|
namenode = { enable = true; formatOnInit = true; restartIfChanged = true; };
|
|
datanode = { enable = true; restartIfChanged = true; };
|
|
journalnode = { enable = true; restartIfChanged = true; };
|
|
zkfc = { enable = true; restartIfChanged = true; };
|
|
httpfs = { enable = true; restartIfChanged = true; };
|
|
};
|
|
yarn = { resourcemanager.enable = true; nodemanager.enable = true; };
|
|
};
|
|
|
|
kerberos_server = {
|
|
enable = true;
|
|
realms."MY.ENGINE".acl = [
|
|
{ principal = "*/admin"; access = "all"; }
|
|
{ principal = "admin"; access = "all"; }
|
|
{ principal = "*/localhost"; access = "all"; }
|
|
{ principal = "*/my.engine"; access = "all"; }
|
|
{ principal = "nn/my.engine"; access = "all"; }
|
|
{ principal = "hdfs"; access = "all"; }
|
|
];
|
|
};
|
|
};
|
|
|
|
krb5 = {
|
|
enable = true;
|
|
realms = {
|
|
"MY.ENGINE" = {
|
|
admin_server = "kdc.my.engine";
|
|
kdc = "kdc.my.engine";
|
|
# default_domain = "my.engine";
|
|
# kpasswd_server = "odin";
|
|
};
|
|
};
|
|
domain_realm = {
|
|
# ".my.engine" = "MY.ENGINE";
|
|
"my.engine" = "MY.ENGINE";
|
|
};
|
|
libdefaults = {
|
|
default_realm = "MY.ENGINE";
|
|
dns_lookup_realm = true;
|
|
dns_lookup_kdc = true;
|
|
ticket_lifetime = "24h";
|
|
renew_lifetime = "7d";
|
|
forwardable = true;
|
|
};
|
|
extraConfig = ''
|
|
[logging]
|
|
default = FILE:/var/log/krb5libs.log
|
|
kdc = FILE:/var/log/krb5kdc.log
|
|
admin_server = FILE:/var/log/kadmind.log
|
|
'';
|
|
};
|
|
|
|
|
|
|
|
|
|
systemd.services.spark-history = {
|
|
path = with pkgs; [ procps openssh nettools ];
|
|
description = "spark history service.";
|
|
after = [ "network.target" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
restartIfChanged = true;
|
|
environment = {
|
|
SPARK_CONF_DIR = sparkConfDir;
|
|
SPARK_LOG_DIR = "/var/log/spark";
|
|
};
|
|
serviceConfig = {
|
|
Type = "forking";
|
|
User = "spark";
|
|
Group = "spark";
|
|
WorkingDirectory = "${pkgs.spark}/lib/${pkgs.spark.untarDir}";
|
|
ExecStart = "${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/start-history-server.sh";
|
|
ExecStop = "${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/stop-history-server.sh";
|
|
TimeoutSec = 300;
|
|
StartLimitBurst = 10;
|
|
Restart = "always";
|
|
};
|
|
};
|
|
|
|
}
|