From 7e6e99a02676cf4ac6f505344822def3ef5076bf Mon Sep 17 00:00:00 2001 From: Filippo Berto Date: Tue, 19 Apr 2022 08:24:16 +0200 Subject: [PATCH] Add htpfs configuration + workaround for yarn cggroups errors --- nixos_modules/big_data.nix | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/nixos_modules/big_data.nix b/nixos_modules/big_data.nix index 66f665c..9c88a2b 100644 --- a/nixos_modules/big_data.nix +++ b/nixos_modules/big_data.nix @@ -1,4 +1,4 @@ -{ config, lib, pkgs, modulesPath,... }: +{ config, lib, pkgs, modulesPath, ... }: let setup_scirpt = '' sudo mkdir -p /hdfs @@ -132,6 +132,7 @@ in # HDFS SECURITY "dfs.block.access.token.enable" = "true"; + "dfs.cluster.administrators" = "hdfs,HTTP,bertof"; # NAME NODE SECURITY "dfs.namenode.keytab.file" = hadoop_keytab_path; @@ -194,6 +195,14 @@ in # "yarn.scheduler.capacity.root.default.state" = "RUNNING"; "yarn.scheduler.capacity.root.acl_submit_applications" = "hadoop,yarn,mapred,hdfs"; }; + httpfsSite = { + "kerberos.realm" = "MY.ENGINE"; + "httpfs.authentication.type" = "kerberos"; + "httpfs.authentication.kerberos.principal " = "HTTP/my.engine@MY.ENGINE"; + "httpfs.authentication.kerberos.keytab" = hadoop_keytab_path; + "httpfs.hadoop.kerberos.principal " = "HTTP/my.engine@MY.ENGINE"; + "httpfs.hadoop.kerberos.keytab" = hadoop_keytab_path; + }; extraConfDirs = [ ]; hdfs = { @@ -205,7 +214,7 @@ in }; yarn = { resourcemanager = { enable = true; restartIfChanged = true; }; - nodemanager = { enable = true; restartIfChanged = true; }; + nodemanager = { enable = true; restartIfChanged = true; useCGroups = false; }; }; };