From 1680b2bc455e86658b47ee6e090ed93d90b96383 Mon Sep 17 00:00:00 2001 From: Filippo Berto Date: Mon, 21 Mar 2022 09:50:26 +0100 Subject: [PATCH] Centralized big_data config --- nixos/base.nix | 120 +-------------------------------------------- nixos/big_data.nix | 120 +++++++++++++++++++++++++++++++++++++++++++++ nixos/thor.nix | 93 +---------------------------------- 3 files changed, 122 insertions(+), 211 deletions(-) create mode 100644 nixos/big_data.nix diff --git a/nixos/base.nix b/nixos/base.nix index 69b4d73..3240307 100644 --- a/nixos/base.nix +++ b/nixos/base.nix @@ -1,45 +1,4 @@ { config, pkgs, lib, ... }: - -let sparkConfDir = pkgs.stdenv.mkDerivation { - name = "spark-config"; - dontUnpack = true; - installPhase = '' - # source standard environment - . $stdenv/setup - - # shorthands - base_conf=${pkgs.spark}/lib/${pkgs.spark.untarDir}/conf/ - - # create output dirs for new derivation - mkdir -p $out/ - - # link unchanged files from the original gnome-session - for f in $base_conf/*.template ; do - ln -sf $f $out/ - done - - # change selected files - cp $out/log4j.properties{.template,} - - cat > $out/spark-env.sh <<- STOP - export JAVA_HOME="${pkgs.jdk8}" - export SPARK_HOME="${pkgs.spark}/lib/${pkgs.spark.untarDir}" - export SPARK_DIST_CLASSPATH=$(${pkgs.hadoop}/bin/hadoop classpath) - export PYSPARK_PYTHON="${pkgs.python3Packages.python}/bin/${pkgs.python3Packages.python.executable}" - export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH" - export SPARKR_R_SHELL="${pkgs.R}/bin/R" - export PATH="\$PATH:${pkgs.R}/bin" - STOP - - cat > $out/spark-defaults.conf <<- STOP - spark.eventLog.enabled true - spark.eventLog.dir hdfs://localhost:/logs/spark - spark.history.fs.logDirectory hdfs://localhost:/logs/spark - STOP - ''; -}; -in - { imports = [ @@ -53,6 +12,7 @@ in # ./defcon.nix # ./mind.nix # ./k3s.nix + ./big_data.nix ]; # Use the systemd-boot EFI boot loader. @@ -252,84 +212,6 @@ in services.power-profiles-daemon.enable = true; # services.teamviewer.enable = true; - services = { - - spark = { - master = { - enable = true; - restartIfChanged = true; - }; - worker = { - enable = true; - restartIfChanged = true; - }; - confDir = sparkConfDir; - }; - - hadoop = { - coreSite = { - "fs.defaultFS" = "hdfs://localhost:8020"; - }; - hdfsSite = { - "dfs.namenode.rpc-bind-host" = "0.0.0.0"; - "dfs.permissions" = "false"; - "dfs.namenode.name.dir" = "/hdfs/dfs/name"; - "dfs.datanode.data.dir" = "/hdfs/dfs/data"; - }; - - hdfs = { - namenode = { - enable = true; - formatOnInit = true; - restartIfChanged = true; - }; - datanode = { - enable = true; - restartIfChanged = true; - }; - journalnode = { - enable = true; - restartIfChanged = true; - }; - zkfc = { - enable = true; - restartIfChanged = true; - }; - httpfs = { - enable = true; - restartIfChanged = true; - }; - }; - yarn = { - resourcemanager.enable = true; - nodemanager.enable = true; - }; - }; - }; - - systemd.services.spark-history = { - path = with pkgs; [ procps openssh nettools ]; - description = "spark history service."; - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; - restartIfChanged = true; - environment = { - SPARK_CONF_DIR = sparkConfDir; - SPARK_LOG_DIR = "/var/log/spark"; - }; - serviceConfig = { - Type = "forking"; - User = "spark"; - Group = "spark"; - WorkingDirectory = "${pkgs.spark}/lib/${pkgs.spark.untarDir}"; - ExecStart = "${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/start-history-server.sh"; - ExecStop = "${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/stop-history-server.sh"; - TimeoutSec = 300; - StartLimitBurst = 10; - Restart = "always"; - }; - }; - # Virtualisation virtualisation = { diff --git a/nixos/big_data.nix b/nixos/big_data.nix new file mode 100644 index 0000000..3d61957 --- /dev/null +++ b/nixos/big_data.nix @@ -0,0 +1,120 @@ +{ config, lib, pkgs, ... }: +let sparkConfDir = pkgs.stdenv.mkDerivation { + name = "spark-config"; + dontUnpack = true; + installPhase = '' + # source standard environment + . $stdenv/setup + + # shorthands + base_conf=${pkgs.spark}/lib/${pkgs.spark.untarDir}/conf/ + + # create output dirs for new derivation + mkdir -p $out/ + + # link unchanged files from the original gnome-session + for f in $base_conf/*.template ; do + ln -sf $f $out/ + done + + # change selected files + cp $out/log4j.properties{.template,} + + cat > $out/spark-env.sh <<- STOP + export JAVA_HOME="${pkgs.jdk8}" + export SPARK_HOME="${pkgs.spark}/lib/${pkgs.spark.untarDir}" + export SPARK_DIST_CLASSPATH=$(${pkgs.hadoop}/bin/hadoop classpath) + export PYSPARK_PYTHON="${pkgs.python3Packages.python}/bin/${pkgs.python3Packages.python.executable}" + export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH" + export SPARKR_R_SHELL="${pkgs.R}/bin/R" + export PATH="\$PATH:${pkgs.R}/bin" + STOP + + cat > $out/spark-defaults.conf <<- STOP + spark.eventLog.enabled true + spark.eventLog.dir hdfs://localhost:/logs/spark + spark.history.fs.logDirectory hdfs://localhost:/logs/spark + STOP + ''; +}; +in +{ + + services = { + spark = { + master = { + enable = true; + restartIfChanged = true; + }; + worker = { + enable = true; + restartIfChanged = true; + }; + confDir = sparkConfDir; + }; + + hadoop = { + coreSite = { + "fs.defaultFS" = "hdfs://localhost:8020"; + }; + hdfsSite = { + "dfs.namenode.rpc-bind-host" = "0.0.0.0"; + "dfs.permissions" = "false"; + "dfs.namenode.name.dir" = "/hdfs/dfs/name"; + "dfs.datanode.data.dir" = "/hdfs/dfs/data"; + }; + + hdfs = { + namenode = { + enable = true; + formatOnInit = true; + restartIfChanged = true; + }; + datanode = { + enable = true; + restartIfChanged = true; + }; + journalnode = { + enable = true; + restartIfChanged = true; + }; + zkfc = { + enable = true; + restartIfChanged = true; + }; + httpfs = { + enable = true; + restartIfChanged = true; + }; + }; + yarn = { + resourcemanager.enable = true; + nodemanager.enable = true; + }; + }; + }; + + systemd.services.spark-history = { + path = with pkgs; [ procps openssh nettools ]; + description = "spark history service."; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + restartIfChanged = true; + environment = { + SPARK_CONF_DIR = sparkConfDir; + SPARK_LOG_DIR = "/var/log/spark"; + }; + serviceConfig = { + Type = "forking"; + User = "spark"; + Group = "spark"; + WorkingDirectory = "${pkgs.spark}/lib/${pkgs.spark.untarDir}"; + ExecStart = "${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/start-history-server.sh"; + ExecStop = "${pkgs.spark}/lib/${pkgs.spark.untarDir}/sbin/stop-history-server.sh"; + TimeoutSec = 300; + StartLimitBurst = 10; + Restart = "always"; + }; + }; + +} diff --git a/nixos/thor.nix b/nixos/thor.nix index a2a8371..7b41f46 100644 --- a/nixos/thor.nix +++ b/nixos/thor.nix @@ -4,51 +4,13 @@ { config, pkgs, lib, ... }: -let sparkConfDir = pkgs.stdenv.mkDerivation { - name = "spark-config"; - dontUnpack = true; - installPhase = '' - # source standard environment - . $stdenv/setup - - # shorthands - base_conf=${pkgs.spark}/lib/${pkgs.spark.untarDir}/conf/ - - # create output dirs for new derivation - mkdir -p $out/ - - # link unchanged files from the original gnome-session - for f in $base_conf/*.template ; do - ln -sf $f $out/ - done - - # change selected files - cp $out/log4j.properties{.template,} - - cat > $out/spark-env.sh <<- STOP - export JAVA_HOME="${pkgs.jdk8}" - export SPARK_HOME="${pkgs.spark}/lib/${pkgs.spark.untarDir}" - export SPARK_DIST_CLASSPATH=$(${pkgs.hadoop}/bin/hadoop classpath) - export PYSPARK_PYTHON="${pkgs.python3Packages.python}/bin/${pkgs.python3Packages.python.executable}" - export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH" - export SPARKR_R_SHELL="${pkgs.R}/bin/R" - export PATH="\$PATH:${pkgs.R}/bin" - STOP - - cat > $out/spark-defaults.conf <<- STOP - spark.eventLog.enabled true - spark.eventLog.dir hdfs://localhost:/logs/spark - spark.history.fs.logDirectory hdfs://localhost:/logs/spark - STOP - ''; -}; -in { imports = [ /etc/nixos/hardware-configuration.nix ./pro_audio.nix + ./big_data.nix ]; boot = { @@ -195,59 +157,6 @@ in # gnome.gnome-remote-desktop.enable = true; zerotierone = { enable = true; joinNetworks = [ "8056c2e21cf9c753" ]; }; - - - spark = { - master = { - enable = true; - restartIfChanged = true; - }; - worker = { - enable = true; - restartIfChanged = true; - }; - confDir = sparkConfDir; - }; - - hadoop = { - coreSite = { - "fs.defaultFS" = "hdfs://localhost:8020"; - }; - hdfsSite = { - "dfs.namenode.rpc-bind-host" = "0.0.0.0"; - "dfs.permissions" = "false"; - }; - - hdfs = { - namenode = { - enable = true; - formatOnInit = true; - restartIfChanged = true; - }; - datanode = { - enable = true; - restartIfChanged = true; - }; - journalnode = { - enable = true; - restartIfChanged = true; - }; - zkfc = { - enable = true; - restartIfChanged = true; - }; - httpfs = { - enable = true; - restartIfChanged = true; - }; - }; - yarn = { - resourcemanager.enable = true; - nodemanager.enable = true; - }; - }; - - ethminer = { enable = false; wallet = "0x73b788882e1C182123333f42FFf275B7dd7f51bb";