bertof/nix-dotfiles
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Extreme cleaup + new nix rice + commit hooks

Browse source
This commit is contained in:
Filippo Berto 2022-06-02 20:16:20 +02:00
parent bc28d964d3
commit c9eeeb9dc4
59 changed files with 319 additions and 400 deletions
Download patch file Download diff file Expand all files Collapse all files

54
nixos_modules/big_data.nix
View file

@ -1,22 +1,22 @@
{ config, lib, pkgs, modulesPath, ... }:
let
setup_scirpt = ''
sudo mkdir -p /hdfs
sudo chown -R hdfs:hadoop /hdfs
# setup_script = ''
# sudo mkdir -p /hdfs
# sudo chown -R hdfs:hadoop /hdfs
for p in {nn,dn,jn,rm,nm,jhs,HTTP}; do
sudo kadmin.local -q "ank -randkey $p/my.engine";
sudo kadmin.local -q "xst -k /etc/hadoop.keytab $p/my.engine";
sudo kadmin.local -q "ktrem -k /etc/hadoop.keytab $p/my.engine old"
done
sudo chown hdfs:hadoop /etc/hadoop.keytab
# for p in {nn,dn,jn,rm,nm,jhs,HTTP}; do
# sudo kadmin.local -q "ank -randkey $p/my.engine";
# sudo kadmin.local -q "xst -k /etc/hadoop.keytab $p/my.engine";
# sudo kadmin.local -q "ktrem -k /etc/hadoop.keytab $p/my.engine old"
# done
# sudo chown hdfs:hadoop /etc/hadoop.keytab
sudo kadmin.local -q "ank -randkey spark/my.engine";
sudo kadmin.local -q "xst -k /etc/spark.keytab spark/my.engine";
sudo kadmin.local -q "ktrem -k /etc/spark.keytab spark/my.engine old"
sudo chown spark:spark /etc/spark.keytab
'';
# sudo kadmin.local -q "ank -randkey spark/my.engine";
# sudo kadmin.local -q "xst -k /etc/spark.keytab spark/my.engine";
# sudo kadmin.local -q "ktrem -k /etc/spark.keytab spark/my.engine old"
# sudo chown spark:spark /etc/spark.keytab
# '';
hadoop_keytab_path = "/etc/hadoop.keytab";
spark_keytab_path = "/etc/spark.keytab";
pysparkPackageSelector = p: with p; [ numpy pyspark ];
@ -63,17 +63,17 @@ let
STOP
cat > $out/spark-defaults.conf <<- STOP
spark.eventLog.enabled true
spark.eventLog.dir hdfs://localhost:/logs/spark
spark.history.fs.logDirectory hdfs://localhost:/logs/spark
# spark.yarn.keytab ${spark_keytab_path}
# spark.yarn.principal spark/my.engine@MY.ENGINE
spark.history.ui.acls.enable true
spark.history.kerberos.enabled true
spark.history.kerberos.keytab ${spark_keytab_path}
spark.history.kerberos.principal spark/my.engine@MY.ENGINE
spark.yarn.appMasterEnv.PYSPARK_PYTHON ${pysparkEnv.outPath}/bin/${pysparkEnv.executable}
spark.yarn.appMasterEnv.PYTHONPATH ${pysparkEnv.outPath}/lib/${pysparkEnv.executable}/site-packages
spark.eventLog.enabled true
spark.eventLog.dir hdfs://localhost:/logs/spark
spark.history.fs.logDirectory hdfs://localhost:/logs/spark
# spark.yarn.keytab ${spark_keytab_path}
# spark.yarn.principal spark/my.engine@MY.ENGINE
spark.history.ui.acls.enable true
spark.history.kerberos.enabled true
spark.history.kerberos.keytab ${spark_keytab_path}
spark.history.kerberos.principal spark/my.engine@MY.ENGINE
spark.yarn.appMasterEnv.PYSPARK_PYTHON ${pysparkEnv.outPath}/bin/${pysparkEnv.executable}
spark.yarn.appMasterEnv.PYTHONPATH ${pysparkEnv.outPath}/lib/${pysparkEnv.executable}/site-packages
spark.executorEnv.PYSPARK_PYTHON ${pysparkEnv.outPath}/bin/${pysparkEnv.executable}
STOP
'';
@ -198,9 +198,9 @@ in
httpfsSite = {
"kerberos.realm" = "MY.ENGINE";
"httpfs.authentication.type" = "kerberos";
"httpfs.authentication.kerberos.principal " = "HTTP/my.engine@MY.ENGINE";
"httpfs.authentication.kerberos.principal " = "HTTP/my.engine@MY.ENGINE";
"httpfs.authentication.kerberos.keytab" = hadoop_keytab_path;
"httpfs.hadoop.kerberos.principal " = "HTTP/my.engine@MY.ENGINE";
"httpfs.hadoop.kerberos.principal " = "HTTP/my.engine@MY.ENGINE";
"httpfs.hadoop.kerberos.keytab" = hadoop_keytab_path;
};
extraConfDirs = [ ];