Pular para o conteúdo principal

Kernel Space Security

Faça uma revisão sobre kernel space e user space se necessário.

Sabemos que o mais proteger as syscalls é o ponto mais importante. As libs interagem com o syscalls para não precisamos reinventar a roda.

Podemos proteger os syscalls com ferramenta como seccomp e apparmor criando mais uma camada entre o kernel space e o user space. Não devemos confundir com o sandboxes que fica em nível de user space.

Essa camada de proteção fará parte do kernel space aplicando regras para chamadas de syscall.

alt text

Vamos nos aprofundar mais em syscalls

man syscalls

A lista é grande vou apontar algumas que estão no site que podem ser familiar

  • add_key
  • chdir
  • chmod
  • chown
  • chown32
  • chroot
  • connect
  • copy_file_range
  • creat
  • delete_module
  • exit
  • getitimer
  • getpgid
  • getpgrp
  • getpid
  • getsid
  • gettid
  • gettimeofday
  • getuid
  • getxattr
  • kill
  • mkdir
  • mmap
  • mount
  • open
  • read
  • reboot
  • rename
  • rmdir
  • seccomp
  • sendfile
  • signal
  • splice
  • swapoff
  • swapon
  • symlink
  • symlinkat
  • sync
  • sync_file_range
  • sync_file_range2
  • syncfs
  • sysfs
  • sysinfo
  • syslog
  • tee
  • time
  • truncate
  • truncate64
  • ugetrlimit
  • umask
  • umount2
  • uname
  • write

Só para ilustrar, kill por exemplo. Observe que temos a linguagem C por isso o uso da glibc e a chamada de função como deve ser.

alt text

Strace​

o strace é uma ferramenta, costuma vir instalada na maioria das distribuições Linux, que intercepta chamadas para o syscall e faz o lock da chamada através de um processo para podermos olhar a stack trace de chamadas. Muito bom para aprender, diagnosticar e debugar.

root@cks-master:~# ls /
bin  boot  dev  etc  home  lib  lib32  lib64  libx32  lost+found  media  mnt  opt  proc  root  run  sbin  snap  srv  sys  tmp  usr  var

# Aqui terremos todas as chamadas de syscalls feitas para poder executar o ls /
root@cks-master:~# strace ls /
execve("/usr/bin/ls", ["ls", "/"], 0x7ffe6e4d8a18 /* 28 vars */) = 0
brk(NULL)                               = 0x55a637e8c000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffcf3871cc0) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=23997, ...}) = 0
mmap(NULL, 23997, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3b000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@p\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=163200, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7a39000
mmap(NULL, 174600, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a7a0e000
mprotect(0x7ff6a7a14000, 135168, PROT_NONE) = 0
mmap(0x7ff6a7a14000, 102400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7ff6a7a14000
mmap(0x7ff6a7a2d000, 28672, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f000) = 0x7ff6a7a2d000
mmap(0x7ff6a7a35000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x26000) = 0x7ff6a7a35000
mmap(0x7ff6a7a37000, 6664, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7a37000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300A\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32, 848) = 32
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\7\2C\n\357_\243\335\2449\206V>\237\374\304"..., 68, 880) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=2029592, ...}) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32, 848) = 32
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\7\2C\n\357_\243\335\2449\206V>\237\374\304"..., 68, 880) = 68
mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a781c000
mmap(0x7ff6a783e000, 1540096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7ff6a783e000
mmap(0x7ff6a79b6000, 319488, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7ff6a79b6000
mmap(0x7ff6a7a04000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7ff6a7a04000
mmap(0x7ff6a7a0a000, 13920, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7a0a000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\"\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=588488, ...}) = 0
mmap(NULL, 590632, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a778b000
mmap(0x7ff6a778d000, 413696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7ff6a778d000
mmap(0x7ff6a77f2000, 163840, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x67000) = 0x7ff6a77f2000
mmap(0x7ff6a781a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8e000) = 0x7ff6a781a000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \22\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18848, ...}) = 0
mmap(NULL, 20752, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a7785000
mmap(0x7ff6a7786000, 8192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7ff6a7786000
mmap(0x7ff6a7788000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7ff6a7788000
mmap(0x7ff6a7789000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7ff6a7789000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220q\0\0\0\0\0\0"..., 832) = 832
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\232e\273F\236E\241\306\373\317\372\345\270*/\327"..., 68, 824) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=157224, ...}) = 0
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\232e\273F\236E\241\306\373\317\372\345\270*/\327"..., 68, 824) = 68
mmap(NULL, 140408, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a7762000
mmap(0x7ff6a7768000, 69632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7ff6a7768000
mmap(0x7ff6a7779000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7ff6a7779000
mmap(0x7ff6a777f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c000) = 0x7ff6a777f000
mmap(0x7ff6a7781000, 13432, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7781000
close(3)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7760000
arch_prctl(ARCH_SET_FS, 0x7ff6a7761400) = 0
mprotect(0x7ff6a7a04000, 16384, PROT_READ) = 0
mprotect(0x7ff6a777f000, 4096, PROT_READ) = 0
mprotect(0x7ff6a7789000, 4096, PROT_READ) = 0
mprotect(0x7ff6a781a000, 4096, PROT_READ) = 0
mprotect(0x7ff6a7a35000, 4096, PROT_READ) = 0
mprotect(0x55a636cab000, 4096, PROT_READ) = 0
mprotect(0x7ff6a7a6e000, 4096, PROT_READ) = 0
munmap(0x7ff6a7a3b000, 23997)           = 0
set_tid_address(0x7ff6a77616d0)         = 2161540
set_robust_list(0x7ff6a77616e0, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7ff6a7768bf0, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7ff6a7776420}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7ff6a7768c90, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7ff6a7776420}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0x7ffcf3871c10) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7ffcf3871c10)      = -1 ENOENT (No such file or directory)
brk(NULL)                               = 0x55a637e8c000
brk(0x55a637ead000)                     = 0x55a637ead000
openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(3, "nodev\tsysfs\nnodev\ttmpfs\nnodev\tbd"..., 1024) = 418
read(3, "", 1024)                       = 0
close(3)                                = 0
access("/etc/selinux/config", F_OK)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=3035952, ...}) = 0
mmap(NULL, 3035952, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a747a000
close(3)                                = 0
openat(AT_FDCWD, "/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2996, ...}) = 0
read(3, "# Locale name alias data base.\n#"..., 4096) = 2996
read(3, "", 4096)                       = 0
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=252, ...}) = 0
mmap(NULL, 252, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a6d000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/gconv/gconv-modules.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=27002, ...}) = 0
mmap(NULL, 27002, PROT_READ, MAP_SHARED, 3, 0) = 0x7ff6a7473000
close(3)                                = 0
futex(0x7ff6a7a09954, FUTEX_WAKE_PRIVATE, 2147483647) = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MEASUREMENT", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=23, ...}) = 0
mmap(NULL, 23, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a40000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_TELEPHONE", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=47, ...}) = 0
mmap(NULL, 47, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3f000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_ADDRESS", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=131, ...}) = 0
mmap(NULL, 131, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3e000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_NAME", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=62, ...}) = 0
mmap(NULL, 62, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3d000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_PAPER", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=34, ...}) = 0
mmap(NULL, 34, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3c000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MESSAGES", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MESSAGES/SYS_LC_MESSAGES", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=48, ...}) = 0
mmap(NULL, 48, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3b000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MONETARY", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=270, ...}) = 0
mmap(NULL, 270, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7472000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_COLLATE", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1518110, ...}) = 0
mmap(NULL, 1518110, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72ff000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_TIME", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=3360, ...}) = 0
mmap(NULL, 3360, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72fe000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_NUMERIC", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=50, ...}) = 0
mmap(NULL, 50, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72fd000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_CTYPE", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=201272, ...}) = 0
mmap(NULL, 201272, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72cb000
close(3)                                = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=40, ws_col=174, ws_xpixel=0, ws_ypixel=0}) = 0
stat("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
openat(AT_FDCWD, "/", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
getdents64(3, /* 25 entries */, 32768)  = 640
getdents64(3, /* 0 entries */, 32768)   = 0
close(3)                                = 0
fstat(1, {st_mode=S_IFCHR|0600, st_rdev=makedev(0x88, 0), ...}) = 0
write(1, "bin  boot  dev\tetc  home  lib\tli"..., 133bin  boot  dev etc  home  lib lib32  lib64  libx32  lost+found  media  mnt  opt  proc  root  run  sbin  snap srv  sys  tmp  usr  var
) = 133
close(1)                                = 0
close(2)                                = 0
exit_group(0)                           = ?
+++ exited with 0 +++
root@cks-master:~#

# Se quiser um pouco mais verboso
strace -v ls /

Podemos contabilizar quantas vezes uma chamada para cada syscall utilizado no comando.

root@cks-master:~# strace -cw ls /
bin  boot  dev etc  home  lib lib32  lib64  libx32  lost+found  media  mnt  opt  proc  root  run  sbin  snap srv  sys  tmp  usr  var
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 23.58    0.000995          24        40           mmap
 16.55    0.000699          29        24           openat
 12.89    0.000544          20        26           close
 11.64    0.000492          19        25           fstat
  6.77    0.000286         285         1           execve
  4.79    0.000202          22         9           read
  4.58    0.000193          24         8           mprotect
  4.57    0.000193         192         1           write
  3.29    0.000139          17         8           pread64
  1.48    0.000063          31         2           getdents64
  1.28    0.000054          17         3           brk
  1.15    0.000048          24         2         2 statfs
  1.11    0.000047          23         2         2 access
  0.97    0.000041          20         2           rt_sigaction
  0.92    0.000039          19         2           ioctl
  0.86    0.000036          36         1           munmap
  0.81    0.000034          17         2         1 arch_prctl
  0.67    0.000028          28         1           set_robust_list
  0.45    0.000019          18         1           stat
  0.42    0.000018          17         1           rt_sigprocmask
  0.42    0.000018          17         1           set_tid_address
  0.42    0.000018          17         1           futex
  0.40    0.000017          16         1           prlimit64
------ ----------- ----------- --------- --------- ----------------
100.00    0.004222                   164         5 total

Fazendo um cat em um arquivo por exemplo já temos outras chamadas.

root@cks-master:~# echo "hello world" > hello
root@cks-master:~# strace -cw cat hello
hello world
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 58.55    0.004229         211        20           fstat
 11.65    0.000842          38        22           mmap
 11.24    0.000812          42        19           openat
  6.68    0.000482          22        21           close
  4.22    0.000305         305         1           execve
  1.65    0.000119          23         5           read
  1.45    0.000104          17         6           pread64
  1.13    0.000082          27         3           mprotect
  0.94    0.000068          33         2           munmap
  0.83    0.000060          19         3           brk
  0.54    0.000039          39         1           write
  0.52    0.000038          18         2         1 arch_prctl
  0.36    0.000026          25         1         1 access
  0.24    0.000017          17         1           fadvise64
------ ----------- ----------- --------- --------- ----------------
100.00    0.007223                   107         2 total

Diretório /proc​

  • Contém informações e conexões para os processo e do kernel.
  • Podemos utilizar para estudar como os processos trabalham
  • Possui configurações e tarefas administrativas
  • Contém arquivos que não existem fisicamente no disco e são criados de forma volátil.

Vamos utilizar o strace e investigar o etcd.

  1. List Syscalls
  2. Find open files
  3. Read secret value
root@cks-master:~# crictl ps | grep etcd
5e90b3ef71d4e       2e96e5913fc06       4 days ago          Running             etcd                      2                   7e4de6c17bc46       etcd-cks-master

root@cks-master:~#  ps -aux | grep etcd
root        1431  6.6  9.3 1662360 375632 ?      Ssl  Sep02 392:41 kube-apiserver --encryption-provider-config=/etc/kubernetes/etcd/ec.yaml --advertise-address=10.128.0.5 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key

### Aqui o etcd como processo eo pid 1511
root        1511  3.1  1.7 11739196 69280 ?      Ssl  Sep02 185:38 etcd --advertise-client-urls=https://10.128.0.5:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --experimental-initial-corrupt-check=true --experimental-watch-progress-notify-interval=5s --initial-advertise-peer-urls=https://10.128.0.5:2380 --initial-cluster=cks-master=https://10.128.0.5:2380 --key-file=/etc/kubernetes/pki/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://10.128.0.5:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://10.128.0.5:2380 --name=cks-master --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
root     2798918  0.0  0.0   8168  2500 pts/0    S+   19:14   0:00 grep --color=auto etcd

# Para ver as chamadas acontecendo em tempo real utilizamos o -p e passamos o pid do processo e podemos todas as chamadas em andamento.
oot@cks-master:~# strace -p 1511
strace: Process 1511 attached
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = -1 EAGAIN (Resource temporarily unavailable)
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
epoll_pwait(4, [], 128, 0, NULL, 0)     = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
nanosleep({tv_sec=0, tv_nsec=3000}, NULL) = 0
read(115, "\27\3\3\0/\322\330jE\376\224\352\2665\376\304|fEma\337\346\234\10\374\5\252\335?\20{"..., 1408) = 52
read(115, 0xc000ba2b00, 1408)           = -1 EAGAIN (Resource temporarily unavailable)
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
epoll_pwait(4, [], 128, 0, NULL, 0)     = 0
epoll_pwait(4, [], 128, 0, NULL, 0)     = 0
epoll_pwait(4, [{EPOLLIN|EPOLLOUT, {u32=360185859, u64=9154956854616915971}}], 128, 12, NULL, 0) = 1
futex(0x1a26100, FUTEX_WAKE_PRIVATE, 1) = 1

# Para facilitar podemos usar o -f (follow fork) e -cw para gera a lista. É necessário esperar alguns segundos e abortar para ver a saída pois ela nao fica gerando em tempo real.

root@cks-master:~# strace -p 1511 -f -cw
strace: Process 1511 attached with 10 threads
strace: pid 1511: entering, ptrace_syscall_info.op == 2
^Cstrace: Process 1511 detached
strace: Process 1536 detached
strace: Process 1537 detached
strace: Process 1538 detached
strace: Process 1555 detached
strace: Process 1556 detached
strace: Process 1561 detached
strace: Process 1562 detached
strace: Process 1587 detached
strace: Process 135999 detached
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 77.08   23.125434       12019      1924       342 futex
 19.91    5.974530        2970      2011         3 epoll_pwait
  2.59    0.778417         380      2044           nanosleep
  0.16    0.047189         108       433           write # São a maioria
  0.14    0.042748        1257        34           fdatasync
  0.06    0.018214          42       429       176 read # São a maioria
  0.01    0.003903          67        58           tgkill
  0.01    0.002814          48        58         3 rt_sigreturn
  0.01    0.002355          40        58           getpid
  0.01    0.002128          39        54           pwrite64
  0.00    0.001333          27        49           setsockopt
  0.00    0.001005         143         7           close
  0.00    0.000669          47        14         7 accept4
  0.00    0.000654          46        14           epoll_ctl
  0.00    0.000343          28        12           lseek
  0.00    0.000270          38         7           getsockname
  0.00    0.000150          24         6           sched_yield
------ ----------- ----------- --------- --------- ----------------
100.00   30.002157                  7212       531 total

Indo para o processo o que temos.

root@cks-master:/proc/1511# ls
arch_status  cgroup      coredump_filter     environ  gid_map   map_files  mounts      numa_maps      pagemap      root       setgroups     stat     task            uid_map
attr         clear_refs  cpu_resctrl_groups  exe      io        maps       mountstats  oom_adj        patch_state  sched      smaps         statm    timens_offsets  wchan
autogroup    cmdline     cpuset              fd       limits    mem        net         oom_score      personality  schedstat  smaps_rollup  status   timers

# Podemos observar que o exe é um link full path para o binário do etcd que esta sendo executado.
root@cks-master:/proc/1511# ls -lha exe
lrwxrwxrwx 1 root root 0 Sep  2 16:04 exe -> /usr/local/bin/etcd

# O diretório fd contém os arquivos abertos
root@cks-master:/proc/1511# cd fd
root@cks-master:/proc/1511/fd# ls
0    101  105  109  113  117  120  126  13   16  22  26  3   33  37  40  44  48  51  55  59  62  66  7   73  77  80  84  88  91  95  99
1    102  106  110  114  118  121  127  130  17  23  27  30  34  38  41  45  49  52  56  6   63  67  70  74  78  81  85  89  92  96
10   103  107  111  115  119  122  128  14   18  24  28  31  35  39  42  46  5   53  57  60  64  68  71  75  79  82  86  9   93  97
100  104  108  112  116  12   125  129  15   2   25  29  32  36  4   43  47  50  54  58  61  65  69  72  76  8   83  87  90  94  98

# Aqui podemos observar a lista de todos os arquivos e sockets abertos e que são links para
root@cks-master:/proc/1511/fd#  ls -lha
total 0
dr-x------ 2 root root  0 Sep  2 16:04 .
dr-xr-xr-x 9 root root  0 Sep  2 16:04 ..
lrwx------ 1 root root 64 Sep  2 16:04 0 -> /dev/null
l-wx------ 1 root root 64 Sep  2 16:04 1 -> 'pipe:[28837]'
lrwx------ 1 root root 64 Sep  2 16:05 10 -> /var/lib/etcd/member/snap/db # A pasta 10 esta apontando para aqui o db, vamos conferir isso
lrwx------ 1 root root 64 Sep  2 16:05 100 -> 'socket:[34618]'
lrwx------ 1 root root 64 Sep  2 16:05 101 -> 'socket:[34623]'
#...
# Bloco removido para facilitar a leitura (todos sockets)
#...
lrwx------ 1 root root 64 Sep  2 16:05 118 -> 'socket:[34670]'
lrwx------ 1 root root 64 Sep  2 16:05 119 -> 'socket:[34673]'
lr-x------ 1 root root 64 Sep  2 16:05 12 -> /var/lib/etcd/member/wal
lrwx------ 1 root root 64 Sep  2 16:05 120 -> 'socket:[34675]'
lrwx------ 1 root root 64 Sep  2 16:05 121 -> 'socket:[34906]'
#...
# Bloco removido para facilitar a leitura (todos sockets)
#...
lrwx------ 1 root root 64 Sep  2 16:05 128 -> 'socket:[34915]'
lrwx------ 1 root root 64 Sep  2 16:05 129 -> 'socket:[34693]'
l-wx------ 1 root root 64 Sep  6 07:05 13 -> /var/lib/etcd/member/wal/1.tmp
lrwx------ 1 root root 64 Sep  2 16:05 130 -> 'socket:[34696]'
lrwx------ 1 root root 64 Sep  2 16:05 14 -> 'socket:[29116]'
lrwx------ 1 root root 64 Sep  2 16:05 15 -> 'socket:[30017]'
lrwx------ 1 root root 64 Sep  2 16:05 16 -> 'socket:[30018]'
lrwx------ 1 root root 64 Sep  2 16:05 17 -> 'socket:[30019]'
l-wx------ 1 root root 64 Sep  4 13:40 18 -> /var/lib/etcd/member/wal/0000000000000019-000000000023d2b1.wal
l-wx------ 1 root root 64 Sep  2 16:04 2 -> 'pipe:[28838]'
lrwx------ 1 root root 64 Sep  2 16:05 22 -> 'socket:[30039]'
lrwx------ 1 root root 64 Sep  2 16:05 23 -> 'socket:[29373]'
#...
# Bloco removido para facilitar a leitura (todos sockets)
#...
lrwx------ 1 root root 64 Sep  2 16:05 38 -> 'socket:[29444]'
lrwx------ 1 root root 64 Sep  2 16:05 39 -> 'socket:[29447]'
lrwx------ 1 root root 64 Sep  2 16:05 4 -> 'anon_inode:[eventpoll]'
lrwx------ 1 root root 64 Sep  2 16:05 40 -> 'socket:[29450]'
lrwx------ 1 root root 64 Sep  2 16:05 41 -> 'socket:[30102]'
#...
# Bloco removido para facilitar a leitura (todos sockets)
#...

# Vamos tentar um pouco desse arquivo e ver o que temos. Lendo as ultimas 50 linhas podemos ver que temos coisas parecida com yaml algum container.
root@cks-master:/proc/1511/fd# tail -f 10 -n 50
ca.crtca.crt
)'
%
 namespace
v1metadata.namespace��
kubernetes-dashboard-web*docker.io/kubernetesui/dashboard-web:1.4.0" --namespace=kubernetes-dashboard"<--settings-config-map-name=kubernetes-dashboard-web-settings*2
web�>"TCP*:%

GOMAXPROCS�

limits.cpu�
0:(

GOMEMLIMIT�
limits.memory�
0BD

cpu
250m

memory
400Mi
cpu
100m
memory
200MiJ

tmp-volume�/tmp"2JL
kube-api-access-2bjbq�-/var/run/secrets/kubernetes.io/serviceaccount"2j/dev/termination-logr
                                                                                            IfNotPresentz
ALL �08@�����File�Always 2
                          ClusterFirstBkubernetes-dashboard-webJkubernetes-dashboard-webR
cks-workerX`hrR
RuntimeDefault���default-scheduler��6
node.kubernetes.io/not-readyExists�" NoExecute(��8
node.kubernetes.io/unreachableExists�" NoExecute(�����PreemptLowerPriority��
Running1
PodReadyToStartContainersTrue����*2#

InitializedTrue҇��*2
ReadyTrue����*2'
ContainersReadyTrue����*2$

PodScheduledTrue҇��*2�"*
10.128.0.42
           192.168.1.҇��B�
kubernetes-dashboard-web


�����u���Unknownԇ�����:Mcontainerd://5c4a07080b5a1d1551d59414d6496650aee7263ace8e8c2bac85ba2bee9801a0 (2*docker.io/kubernetesui/dashboard-web:1.4.0:ldocker.io/kubernetesui/dashboard-web@sha256:4445b31a2c25c875e2df8ca103a8e3f3275778d10065c7c011f6ca42cd4bec5fBMcontainerd://b78d3a18061a6ac00800b536cdd1017125149b5b47684b1acfeaed0c1f15b4f6Hb

tmp-volume/tmpbR
kube-api-access-2bjbq-/var/run/secrets/kubernetes.io/serviceaccountDisabledJ BurstableZb

192.168.1.7r�


10.128.0.4�"

Vamos tentar criar uma secret e ver o que temos.

root@cks-master:/proc/1511/fd# k create secret generic mynewsecret --from-literal=password=123456
secret/mynewsecret created

# Vamos tentar procurar pela senha
root@cks-master:/proc/1511/fd# cat 10 | grep 123456 # Sem sucesso
root@cks-master:/proc/1511/fd# cat 10 | grep mynewsecret
Binary file (standard input) matches
# Para ler é necessário usar strings, e encontramos isso
root@cks-master:/proc/1511/fd# cat 10 | strings | grep mynewsecret
%/registry/secrets/default/mynewsecret

# Vamos ler 20 linhas depois
root@cks-master:/proc/1511/fd# cat 10 | strings | grep mynewsecret -A20
%/registry/secrets/default/mynewsecret #<<<<
k8s:enc:aescbc:v1:key1: # Existe uma encriptação para esse secret, mas se não houvesse poderíamos ler ela aqui por isso que o grep 123456 não pegou
4O\%s'
zI{N4YE
iqdc
?\wcACz3B
[f3Y
-/registry/events/default/pod.17f2bf3d82110f6f
Event
pod.17f2bf3d82110f6f
default"
*$bd9009e1-832d-465f-ae41-557b7c67b6172
kubelet
Update
FieldsV1:
{"f:count":{},"f:firstTimestamp":{},"f:involvedObject":{},"f:lastTimestamp":{},"f:message":{},"f:reason":{},"f:reportingComponent":{},"f:reportingInstance":{},"f:source":{"f:component":{},"f:host":{}},"f:type":{}}B
default
pod"$8621ad1f-e7b8-4768-9ae3-05b0ec9f2896*
2124582:
spec.containers{pod}
BackOff"aBack-off restarting failed container pod in pod pod_default(8621ad1f-e7b8-4768-9ae3-05b0ec9f2896)*
--
%/registry/secrets/default/mynewsecret
k8s:enc:aescbc:v1:key1:
4O\%s'
zI{N4YE
iqdc
?\wcACz3B
[f3Y
4/registry/leases/kube-system/kube-controller-manager
coordination.k8s.io/v1
Lease
kube-controller-manager
kube-system"
*$ea6e402d-dcd0-4060-8d0a-3ce53002ce4f2
kube-controller-manager
Update
coordination.k8s.io/v1"
FieldsV1:|
z{"f:spec":{"f:acquireTime":{},"f:holderIdentity":{},"f:leaseDurationSeconds":{},"f:leaseTransitions":{},"f:renewTime":{}}}B
/cks-master_c9b0ce4c-37ea-4ce7-afba-3aaecc6693b5
+/registry/leases/kube-node-lease/cks-worker
coordination.k8s.io/v1

Agora vamos colocar uma pod com uma variável hardcoded com essa secret.

# não consigo criar nada no diretório /proc, deixei só pra gente ver
root@cks-master:/proc/1511/fd# k run apache --image=httpd -oyaml --dry-run=client > apache.yaml
-bash: apache.yaml: No such file or directory
root@cks-master:/proc/1511/fd# cd
root@cks-master:~# k run apache --image=httpd -oyaml --dry-run=client > apache.yaml
root@cks-master:~# vim apache.yaml
root@cks-master:~# cat apache.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    run: apache
  name: apache
spec:
  containers:
  - image: httpd
    name: apache
    env: # Adicionado
    - name: PASSWORD
      value: "123456"
    resources: {}
  dnsPolicy: ClusterFirst
  restartPolicy: Always
status: {}

root@cks-master:~# k create -f apache.yaml
pod/apache created

root@cks-master:~# k exec apache -- env | grep PASSWORD
PASSWORD=123456

# Não temos nada pois esta rodando no worker node
root@cks-master:~# ps -aux | grep httpd
root     2830278  0.0  0.0   8168   720 pts/0    S+   20:20   0:00 grep --color=auto httpd
root@cks-master:~# k get pod apache -o wide
NAME     READY   STATUS    RESTARTS   AGE   IP             NODE         NOMINATED NODE   READINESS GATES
apache   1/1     Running   0          16m   192.168.1.36   cks-worker   <none>           <none>

# NO Worker

# Temos o nosso container rodando, mas precisamos do processo
root@cks-worker:~# crictl ps | grep apache
43a100832c2bb       9cb0a23156020       19 minutes ago      Running             apache

# Procuramos por httdp que é o binário que esta sendo executado no container e temos alguns.
root@cks-worker:~# ps -aux | grep httpd
root     1447884  0.0  0.1   5860  4760 ?        Ss   20:05   0:00 httpd -DFOREGROUND
www-data 1447897  0.0  0.0 1210552 3584 ?        Sl   20:05   0:00 httpd -DFOREGROUND
www-data 1447898  0.0  0.0 1210552 3588 ?        Sl   20:05   0:00 httpd -DFOREGROUND
www-data 1447900  0.0  0.0 1210552 3588 ?        Sl   20:05   0:00 httpd -DFOREGROUND
root     1452924  0.0  0.0   8168  2376 pts/0    S+   20:23   0:00 grep --color=auto httpd

# Para saber qual podemos usa o pstree -p e filtrar a saida
# O containerd iniciou o containerd-shim que inicio o main process httpd 1447884 que é o que temos acima
root@cks-worker:~# pstree -p | grep containerd-shim | grep httpd
           |-containerd-shim(1447830)-+-httpd(1447884)-+-httpd(1447897)-+-{httpd}(1447901)

root@cks-worker:~# cd /proc/1447884
root@cks-worker:/proc/1447884# ls
arch_status  cgroup      coredump_filter     environ  gid_map   map_files  mounts      numa_maps      pagemap      root       setgroups     stat     task            uid_map
attr         clear_refs  cpu_resctrl_groups  exe      io        maps       mountstats  oom_adj        patch_state  sched      smaps         statm    timens_offsets  wchan
autogroup    cmdline     cpuset              fd       limits    mem        net         oom_score      personality  schedstat  smaps_rollup  status   timers
auxv         comm        cwd                 fdinfo   loginuid  mountinfo  ns          oom_score_adj  projid_map   sessionid  stack         syscall  timerslack_ns

# Só para mostrar o que o exe esta apontando
root@cks-worker:/proc/1447884# ls -lha exe
lrwxrwxrwx 1 root root 0 Sep  6 20:05 exe -> /usr/local/apache2/bin/httpd

root@cks-worker:/proc/1447884# cat environ
HTTPD_VERSION=2.4.62KUBERNETES_SERVICE_PORT=443KUBERNETES_PORT=tcp://10.96.0.1:443HOSTNAME=apacheHOME=/rootHTTPD_PATCHES=APP1_SERVICE_HOST=10.105.1.235APP2_SERVICE_HOST=10.98.16.26APP1_SERVICE_PORT=80APP1_PORT=tcp://10.105.1.235:80APP2_PORT=tcp://10.98.16.26:80KUBERNETES_PORT_443_TCP_ADDR=10.96.0.1APP2_SERVICE_PORT=80PATH=/usr/local/apache2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binKUBERNETES_PORT_443_TCP_PORT=443HTTPD_SHA256=674188e7bf44ced82da8db522da946849e22080d73d16c93f7f4df89e25729ecKUBERNETES_PORT_443_TCP_PROTO=tcpAPP1_PORT_80_TCP_ADDR=10.105.1.235APP2_PORT_80_TCP_ADDR=10.98.16.26APP1_PORT_80_TCP_PORT=80APP1_PORT_80_TCP_PROTO=tcpAPP2_PORT_80_TCP_PORT=80APP2_PORT_80_TCP_PROTO=tcpHTTPD_PREFIX=/usr/local/apache2KUBERNETES_PORT_443_TCP=tcp://10.96.0.1:443KUBERNETES_SERVICE_PORT_HTTPS=443KUBERNETES_SERVICE_HOST=10.96.0.1PWD=/usr/local/apache2PASSWORD=123456APP1_PORT_80_TCP=tcp://10.105.1.235:80APP2_PORT_80_TCP=tcp://10.98.16.26:80root@cks-worker:/proc/1447884# cat environ  | grep PASSWORD

root@cks-worker:/proc/1447884# cat environ  | grep "PASSWORD"
Binary file (standard input) matches

root@cks-worker:/proc/1447884# cat environ  | strings |  grep "PASSWORD"
PASSWORD=123456

Fazendo uma varredura em todos os processos podemos buscar uma variavel dessa. Elaborando um script melhor para detectar onde cada uma dessa dessa saídas apareceram ficará bem fácil saber o que é que.

root@cks-worker:/proc/1447884# cat /proc/*/environ  | strings |  grep "PASSWORD"
PASSWORD=123456
PASSWORD=123456
PASSWORD=123456
PASSWORD=123456

Veja o perigo que é um acesso root em um node e como é fácil buscar todas as variaveis de ambiente de todos os containers ao mesmo tempo.