bertof/nix-dotfiles
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add htpfs configuration + workaround for yarn cggroups errors

Browse source
This commit is contained in:
Filippo Berto 2022-04-19 08:24:16 +02:00
parent 038bde17f1
commit 7e6e99a026
1 changed files with 11 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

13
nixos_modules/big_data.nix
View file

@ -1,4 +1,4 @@
{ config, lib, pkgs, modulesPath,... }: { config, lib, pkgs, modulesPath, ... }:
let let
setup_scirpt = '' setup_scirpt = ''
sudo mkdir -p /hdfs sudo mkdir -p /hdfs
@ -132,6 +132,7 @@ in
# HDFS SECURITY # HDFS SECURITY
"dfs.block.access.token.enable" = "true"; "dfs.block.access.token.enable" = "true";
"dfs.cluster.administrators" = "hdfs,HTTP,bertof";
# NAME NODE SECURITY # NAME NODE SECURITY
"dfs.namenode.keytab.file" = hadoop_keytab_path; "dfs.namenode.keytab.file" = hadoop_keytab_path;
@ -194,6 +195,14 @@ in
# "yarn.scheduler.capacity.root.default.state" = "RUNNING"; # "yarn.scheduler.capacity.root.default.state" = "RUNNING";
"yarn.scheduler.capacity.root.acl_submit_applications" = "hadoop,yarn,mapred,hdfs"; "yarn.scheduler.capacity.root.acl_submit_applications" = "hadoop,yarn,mapred,hdfs";
}; };
httpfsSite = {
"kerberos.realm" = "MY.ENGINE";
"httpfs.authentication.type" = "kerberos";
"httpfs.authentication.kerberos.principal " = "HTTP/my.engine@MY.ENGINE";
"httpfs.authentication.kerberos.keytab" = hadoop_keytab_path;
"httpfs.hadoop.kerberos.principal " = "HTTP/my.engine@MY.ENGINE";
"httpfs.hadoop.kerberos.keytab" = hadoop_keytab_path;
};
extraConfDirs = [ ]; extraConfDirs = [ ];
hdfs = { hdfs = {
@ -205,7 +214,7 @@ in
}; };
yarn = { yarn = {
resourcemanager = { enable = true; restartIfChanged = true; }; resourcemanager = { enable = true; restartIfChanged = true; };
nodemanager = { enable = true; restartIfChanged = true; }; nodemanager = { enable = true; restartIfChanged = true; useCGroups = false; };
}; };
}; };