bertof/nix-dotfiles
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Centralized big_data config

Browse source
This commit is contained in:
Filippo Berto 2022-03-21 09:50:26 +01:00
parent 5a0f9cc5b4
commit 1680b2bc45
3 changed files with 122 additions and 211 deletions
Download patch file Download diff file Expand all files Collapse all files

93
nixos/thor.nix
View file

@ -4,51 +4,13 @@
{ config, pkgs, lib, ... }:
let sparkConfDir = pkgs.stdenv.mkDerivation {
name = "spark-config";
dontUnpack = true;
installPhase = ''
# source standard environment
. $stdenv/setup
# shorthands
base_conf=${pkgs.spark}/lib/${pkgs.spark.untarDir}/conf/
# create output dirs for new derivation
mkdir -p $out/
# link unchanged files from the original gnome-session
for f in $base_conf/*.template ; do
ln -sf $f $out/
done
# change selected files
cp $out/log4j.properties{.template,}
cat > $out/spark-env.sh <<- STOP
export JAVA_HOME="${pkgs.jdk8}"
export SPARK_HOME="${pkgs.spark}/lib/${pkgs.spark.untarDir}"
export SPARK_DIST_CLASSPATH=$(${pkgs.hadoop}/bin/hadoop classpath)
export PYSPARK_PYTHON="${pkgs.python3Packages.python}/bin/${pkgs.python3Packages.python.executable}"
export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH"
export SPARKR_R_SHELL="${pkgs.R}/bin/R"
export PATH="\$PATH:${pkgs.R}/bin"
STOP
cat > $out/spark-defaults.conf <<- STOP
spark.eventLog.enabled true
spark.eventLog.dir hdfs://localhost:/logs/spark
spark.history.fs.logDirectory hdfs://localhost:/logs/spark
STOP
'';
};
in
{
imports = [
<nixos-hardware/common/cpu/amd>
<nixos-hardware/common/pc/ssd>
/etc/nixos/hardware-configuration.nix
./pro_audio.nix
./big_data.nix
];
boot = {
@ -195,59 +157,6 @@ in
# gnome.gnome-remote-desktop.enable = true;
zerotierone = { enable = true; joinNetworks = [ "8056c2e21cf9c753" ]; };
spark = {
master = {
enable = true;
restartIfChanged = true;
};
worker = {
enable = true;
restartIfChanged = true;
};
confDir = sparkConfDir;
};
hadoop = {
coreSite = {
"fs.defaultFS" = "hdfs://localhost:8020";
};
hdfsSite = {
"dfs.namenode.rpc-bind-host" = "0.0.0.0";
"dfs.permissions" = "false";
};
hdfs = {
namenode = {
enable = true;
formatOnInit = true;
restartIfChanged = true;
};
datanode = {
enable = true;
restartIfChanged = true;
};
journalnode = {
enable = true;
restartIfChanged = true;
};
zkfc = {
enable = true;
restartIfChanged = true;
};
httpfs = {
enable = true;
restartIfChanged = true;
};
};
yarn = {
resourcemanager.enable = true;
nodemanager.enable = true;
};
};
ethminer = {
enable = false;
wallet = "0x73b788882e1C182123333f42FFf275B7dd7f51bb";