bertof/nix-dotfiles
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reformat + swap odin SSD

Browse source
This commit is contained in:
Filippo Berto 2022-09-01 15:11:16 +02:00
parent 1bc27dde82
commit 2f3d05a802
No known key found for this signature in database
GPG key ID: FE98AE5EC52B1056
83 changed files with 1448 additions and 931 deletions
Download patch file Download diff file Expand all files Collapse all files

102
nixos_modules/big_data.nix
View file

@ -11,7 +11,6 @@ let
# done
# sudo chown hdfs:hadoop /etc/hadoop.keytab
# sudo kadmin.local -q "ank -randkey spark/my.engine";
# sudo kadmin.local -q "xst -k /etc/spark.keytab spark/my.engine";
# sudo kadmin.local -q "ktrem -k /etc/spark.keytab spark/my.engine old"
@ -82,21 +81,21 @@ in
{
networking = {
hosts = {
"127.0.0.1" = [
"ds.my.engine"
"kdc.my.engine"
"my.engine"
];
};
hosts = { "127.0.0.1" = [ "ds.my.engine" "kdc.my.engine" "my.engine" ]; };
};
services = {
spark = {
package = spark;
master = { enable = true; restartIfChanged = true; };
worker = { enable = true; restartIfChanged = true; };
master = {
enable = true;
restartIfChanged = true;
};
worker = {
enable = true;
restartIfChanged = true;
};
confDir = sparkConfDir;
};
@ -137,12 +136,14 @@ in
# NAME NODE SECURITY
"dfs.namenode.keytab.file" = hadoop_keytab_path;
"dfs.namenode.kerberos.principal" = "nn/my.engine@MY.ENGINE";
"dfs.namenode.kerberos.internal.spnego.principal" = "HTTP/my.engine@MY.ENGINE";
"dfs.namenode.kerberos.internal.spnego.principal" =
"HTTP/my.engine@MY.ENGINE";
# SECONDARY NAME NODE SECURITY
"dfs.secondary.namenode.keytab.file" = hadoop_keytab_path;
"dfs.secondary.namenode.kerberos.principal" = "nn/my.engine@MY.ENGINE";
"dfs.secondary.namenode.kerberos.internal.spnego.principal" = "HTTP/my.engine@MY.ENGINE";
"dfs.secondary.namenode.kerberos.internal.spnego.principal" =
"HTTP/my.engine@MY.ENGINE";
# DATA NODE SECURITY
"dfs.datanode.keytab.file" = hadoop_keytab_path;
@ -156,7 +157,8 @@ in
"dfs.webhdfs.enabled" = "true";
# WEB AUTHENTICATION CONFIG
"dfs.web.authentication.kerberos.principal" = "HTTP/my.engine@MY.ENGINE";
"dfs.web.authentication.kerberos.principal" =
"HTTP/my.engine@MY.ENGINE";
"dfs.web.authentication.kerberos.keytab" = hadoop_keytab_path;
"ignore.secure.ports.for.testing" = "true";
"dfs.http.policy" = "HTTP_ONLY";
@ -173,15 +175,20 @@ in
yarnSite = {
"yarn.nodemanager.admin-env" = "PATH=$PATH";
"yarn.nodemanager.aux-services" = "mapreduce_shuffle";
"yarn.nodemanager.aux-services.mapreduce_shuffle.class" = "org.apache.hadoop.mapred.ShuffleHandler";
"yarn.nodemanager.aux-services.mapreduce_shuffle.class" =
"org.apache.hadoop.mapred.ShuffleHandler";
"yarn.nodemanager.bind-host" = "0.0.0.0";
"yarn.nodemanager.container-executor.class" = "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor";
"yarn.nodemanager.env-whitelist" = "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,LANG,TZ";
"yarn.nodemanager.container-executor.class" =
"org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor";
"yarn.nodemanager.env-whitelist" =
"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,LANG,TZ";
"yarn.nodemanager.linux-container-executor.group" = "hadoop";
"yarn.nodemanager.linux-container-executor.path" = "/run/wrappers/yarn-nodemanager/bin/container-executor";
"yarn.nodemanager.linux-container-executor.path" =
"/run/wrappers/yarn-nodemanager/bin/container-executor";
"yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager";
"yarn.resourcemanager.bind-host" = "0.0.0.0";
"yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
"yarn.resourcemanager.scheduler.class" =
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
"yarn.resourcemanager.keytab" = hadoop_keytab_path;
"yarn.resourcemanager.principal" = "rm/my.engine@MY.ENGINE";
@ -193,12 +200,14 @@ in
"yarn.scheduler.capacity.root.queues" = "default";
"yarn.scheduler.capacity.root.default.capacity" = 100;
# "yarn.scheduler.capacity.root.default.state" = "RUNNING";
"yarn.scheduler.capacity.root.acl_submit_applications" = "hadoop,yarn,mapred,hdfs";
"yarn.scheduler.capacity.root.acl_submit_applications" =
"hadoop,yarn,mapred,hdfs";
};
httpfsSite = {
"kerberos.realm" = "MY.ENGINE";
"httpfs.authentication.type" = "kerberos";
"httpfs.authentication.kerberos.principal " = "HTTP/my.engine@MY.ENGINE";
"httpfs.authentication.kerberos.principal " =
"HTTP/my.engine@MY.ENGINE";
"httpfs.authentication.kerberos.keytab" = hadoop_keytab_path;
"httpfs.hadoop.kerberos.principal " = "HTTP/my.engine@MY.ENGINE";
"httpfs.hadoop.kerberos.keytab" = hadoop_keytab_path;
@ -206,23 +215,52 @@ in
extraConfDirs = [ ];
hdfs = {
namenode = { enable = true; formatOnInit = true; restartIfChanged = true; };
datanode = { enable = true; restartIfChanged = true; };
journalnode = { enable = true; restartIfChanged = true; };
zkfc = { enable = false; restartIfChanged = true; }; # ZOOKEEPER DISABLED, not using High Availability setup
httpfs = { enable = true; restartIfChanged = true; };
namenode = {
enable = true;
formatOnInit = true;
restartIfChanged = true;
};
datanode = {
enable = true;
restartIfChanged = true;
};
journalnode = {
enable = true;
restartIfChanged = true;
};
zkfc = {
enable = false;
restartIfChanged = true;
}; # ZOOKEEPER DISABLED, not using High Availability setup
httpfs = {
enable = true;
restartIfChanged = true;
};
};
yarn = {
resourcemanager = { enable = true; restartIfChanged = true; };
nodemanager = { enable = true; restartIfChanged = true; useCGroups = false; };
resourcemanager = {
enable = true;
restartIfChanged = true;
};
nodemanager = {
enable = true;
restartIfChanged = true;
useCGroups = false;
};
};
};
kerberos_server = {
enable = true;
realms."MY.ENGINE".acl = [
{ principal = "*/admin"; access = "all"; }
{ principal = "*/my.engine"; access = "all"; }
{
principal = "*/admin";
access = "all";
}
{
principal = "*/my.engine";
access = "all";
}
];
};
};
@ -274,8 +312,10 @@ in
User = "spark";
Group = "spark";
WorkingDirectory = "${pkgs.spark}/lib/${pkgs.spark.untarDir}";
ExecStart = "${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/start-history-server.sh";
ExecStop = "${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/stop-history-server.sh";
ExecStart =
"${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/start-history-server.sh";
ExecStop =
"${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/stop-history-server.sh";
TimeoutSec = 300;
StartLimitBurst = 10;
Restart = "always";