Skip to main content

Kernel Space Security

Haz una revisión sobre kernel space y user space si es necesario.

Sabemos que proteger las syscalls es el punto más importante. Las librerías interactúan con las syscalls para no tener que reinventar la rueda.

Podemos proteger las syscalls con herramientas como seccomp y apparmor creando una capa más entre el kernel space y el user space. No debemos confundirlas con los sandboxes que están a nivel de user space.

Esta capa de protección formará parte del kernel space aplicando reglas para llamadas de syscall.

alt text

Vamos a profundizar más en syscalls

man syscalls

La lista es grande, voy a señalar algunas que están en el sitio y que pueden ser familiares

  • add_key
  • chdir
  • chmod
  • chown
  • chown32
  • chroot
  • connect
  • copy_file_range
  • creat
  • delete_module
  • exit
  • getitimer
  • getpgid
  • getpgrp
  • getpid
  • getsid
  • gettid
  • gettimeofday
  • getuid
  • getxattr
  • kill
  • mkdir
  • mmap
  • mount
  • open
  • read
  • reboot
  • rename
  • rmdir
  • seccomp
  • sendfile
  • signal
  • splice
  • swapoff
  • swapon
  • symlink
  • symlinkat
  • sync
  • sync_file_range
  • sync_file_range2
  • syncfs
  • sysfs
  • sysinfo
  • syslog
  • tee
  • time
  • truncate
  • truncate64
  • ugetrlimit
  • umask
  • umount2
  • uname
  • write

Solo para ilustrar, kill por ejemplo. Observa que tenemos el lenguaje C por eso el uso de glibc y la llamada de función como debe ser.

alt text

Strace​

strace es una herramienta, suele venir instalada en la mayoría de las distribuciones Linux, que intercepta llamadas al syscall y hace el bloqueo de la llamada a través de un proceso para que podamos ver el stack trace de llamadas. Muy bueno para aprender, diagnosticar y depurar.

root@cks-master:~# ls /
bin  boot  dev  etc  home  lib  lib32  lib64  libx32  lost+found  media  mnt  opt  proc  root  run  sbin  snap  srv  sys  tmp  usr  var

# Aquí tendremos todas las llamadas de syscalls hechas para poder ejecutar el ls /
root@cks-master:~# strace ls /
execve("/usr/bin/ls", ["ls", "/"], 0x7ffe6e4d8a18 /* 28 vars */) = 0
brk(NULL)                               = 0x55a637e8c000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffcf3871cc0) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=23997, ...}) = 0
mmap(NULL, 23997, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3b000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@p\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=163200, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7a39000
mmap(NULL, 174600, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a7a0e000
mprotect(0x7ff6a7a14000, 135168, PROT_NONE) = 0
mmap(0x7ff6a7a14000, 102400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7ff6a7a14000
mmap(0x7ff6a7a2d000, 28672, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f000) = 0x7ff6a7a2d000
mmap(0x7ff6a7a35000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x26000) = 0x7ff6a7a35000
mmap(0x7ff6a7a37000, 6664, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7a37000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300A\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32, 848) = 32
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\7\2C\n\357_\243\335\2449\206V>\237\374\304"..., 68, 880) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=2029592, ...}) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32, 848) = 32
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\7\2C\n\357_\243\335\2449\206V>\237\374\304"..., 68, 880) = 68
mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a781c000
mmap(0x7ff6a783e000, 1540096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7ff6a783e000
mmap(0x7ff6a79b6000, 319488, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7ff6a79b6000
mmap(0x7ff6a7a04000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7ff6a7a04000
mmap(0x7ff6a7a0a000, 13920, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7a0a000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\"\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=588488, ...}) = 0
mmap(NULL, 590632, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a778b000
mmap(0x7ff6a778d000, 413696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7ff6a778d000
mmap(0x7ff6a77f2000, 163840, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x67000) = 0x7ff6a77f2000
mmap(0x7ff6a781a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8e000) = 0x7ff6a781a000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \22\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18848, ...}) = 0
mmap(NULL, 20752, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a7785000
mmap(0x7ff6a7786000, 8192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7ff6a7786000
mmap(0x7ff6a7788000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7ff6a7788000
mmap(0x7ff6a7789000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7ff6a7789000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220q\0\0\0\0\0\0"..., 832) = 832
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\232e\273F\236E\241\306\373\317\372\345\270*/\327"..., 68, 824) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=157224, ...}) = 0
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\232e\273F\236E\241\306\373\317\372\345\270*/\327"..., 68, 824) = 68
mmap(NULL, 140408, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a7762000
mmap(0x7ff6a7768000, 69632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7ff6a7768000
mmap(0x7ff6a7779000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7ff6a7779000
mmap(0x7ff6a777f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c000) = 0x7ff6a777f000
mmap(0x7ff6a7781000, 13432, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7781000
close(3)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7760000
arch_prctl(ARCH_SET_FS, 0x7ff6a7761400) = 0
mprotect(0x7ff6a7a04000, 16384, PROT_READ) = 0
mprotect(0x7ff6a777f000, 4096, PROT_READ) = 0
mprotect(0x7ff6a7789000, 4096, PROT_READ) = 0
mprotect(0x7ff6a781a000, 4096, PROT_READ) = 0
mprotect(0x7ff6a7a35000, 4096, PROT_READ) = 0
mprotect(0x55a636cab000, 4096, PROT_READ) = 0
mprotect(0x7ff6a7a6e000, 4096, PROT_READ) = 0
munmap(0x7ff6a7a3b000, 23997)           = 0
set_tid_address(0x7ff6a77616d0)         = 2161540
set_robust_list(0x7ff6a77616e0, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7ff6a7768bf0, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7ff6a7776420}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7ff6a7768c90, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7ff6a7776420}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0x7ffcf3871c10) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7ffcf3871c10)      = -1 ENOENT (No such file or directory)
brk(NULL)                               = 0x55a637e8c000
brk(0x55a637ead000)                     = 0x55a637ead000
openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(3, "nodev\tsysfs\nnodev\ttmpfs\nnodev\tbd"..., 1024) = 418
read(3, "", 1024)                       = 0
close(3)                                = 0
access("/etc/selinux/config", F_OK)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=3035952, ...}) = 0
mmap(NULL, 3035952, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a747a000
close(3)                                = 0
openat(AT_FDCWD, "/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2996, ...}) = 0
read(3, "# Locale name alias data base.\n#"..., 4096) = 2996
read(3, "", 4096)                       = 0
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=252, ...}) = 0
mmap(NULL, 252, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a6d000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/gconv/gconv-modules.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=27002, ...}) = 0
mmap(NULL, 27002, PROT_READ, MAP_SHARED, 3, 0) = 0x7ff6a7473000
close(3)                                = 0
futex(0x7ff6a7a09954, FUTEX_WAKE_PRIVATE, 2147483647) = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MEASUREMENT", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=23, ...}) = 0
mmap(NULL, 23, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a40000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_TELEPHONE", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=47, ...}) = 0
mmap(NULL, 47, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3f000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_ADDRESS", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=131, ...}) = 0
mmap(NULL, 131, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3e000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_NAME", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=62, ...}) = 0
mmap(NULL, 62, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3d000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_PAPER", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=34, ...}) = 0
mmap(NULL, 34, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3d000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MESSAGES", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MESSAGES/SYS_LC_MESSAGES", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=48, ...}) = 0
mmap(NULL, 48, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3b000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MONETARY", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=270, ...}) = 0
mmap(NULL, 270, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7472000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_COLLATE", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1518110, ...}) = 0
mmap(NULL, 1518110, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72ff000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_TIME", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=3360, ...}) = 0
mmap(NULL, 3360, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72fe000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_NUMERIC", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=50, ...}) = 0
mmap(NULL, 50, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72fd000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_CTYPE", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=201272, ...}) = 0
mmap(NULL, 201272, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72cb000
close(3)                                = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=40, ws_col=174, ws_xpixel=0, ws_ypixel=0}) = 0
stat("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
openat(AT_FDCWD, "/", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
getdents64(3, /* 25 entries */, 32768)  = 640
getdents64(3, /* 0 entries */, 32768)   = 0
close(3)                                = 0
fstat(1, {st_mode=S_IFCHR|0600, st_rdev=makedev(0x88, 0), ...}) = 0
write(1, "bin  boot  dev\tetc  home  lib\tli"..., 133bin  boot  dev etc  home  lib lib32  lib64  libx32  lost+found  media  mnt  opt  proc  root  run  sbin  snap srv  sys  tmp  usr  var
) = 133
close(1)                                = 0
close(2)                                = 0
exit_group(0)                           = ?
+++ exited with 0 +++
root@cks-master:~#

# Si quieres un poco más verbose
strace -v ls /

Podemos contabilizar cuántas veces se realiza una llamada para cada syscall utilizada en el comando.

root@cks-master:~# strace -cw ls /
bin  boot  dev etc  home  lib lib32  lib64  libx32  lost+found  media  mnt  opt  proc  root  run  sbin  snap srv  sys  tmp  usr  var
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 23.58    0.000995          24        40           mmap
 16.55    0.000699          29        24           openat
 12.89    0.000544          20        26           close
 11.64    0.000492          19        25           fstat
  6.77    0.000286         285         1           execve
  4.79    0.000202          22         9           read
  4.58    0.000193          24         8           mprotect
  4.57    0.000193         192         1           write
  3.29    0.000139          17         8           pread64
  1.48    0.000063          31         2           getdents64
  1.28    0.000054          17         3           brk
  1.15    0.000048          24         2         2 statfs
  1.11    0.000047          23         2         2 access
  0.97    0.000041          20         2           rt_sigaction
  0.92    0.000039          19         2           ioctl
  0.86    0.000036          36         1           munmap
  0.81    0.000034          17         2         1 arch_prctl
  0.67    0.000028          28         1           set_robust_list
  0.45    0.000019          18         1           stat
  0.42    0.000018          17         1           rt_sigprocmask
  0.42    0.000018          17         1           set_tid_address
  0.42    0.000018          17         1           futex
  0.40    0.000017          16         1           prlimit64
------ ----------- ----------- --------- --------- ----------------
100.00    0.004222                   164         5 total

Haciendo un cat en un archivo por ejemplo ya tenemos otras llamadas.

root@cks-master:~# echo "hello world" > hello
root@cks-master:~# strace -cw cat hello
hello world
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 58.55    0.004229         211        20           fstat
 11.65    0.000842          38        22           mmap
 11.24    0.000812          42        19           openat
  6.68    0.000482          22        21           close
  4.22    0.000305         305         1           execve
  1.65    0.000119          23         5           read
  1.45    0.000104          17         6           pread64
  1.13    0.000082          27         3           mprotect
  0.94    0.000068          33         2           munmap
  0.83    0.000060          19         3           brk
  0.54    0.000039          39         1           write
  0.52    0.000038          18         2         1 arch_prctl
  0.36    0.000026          25         1         1 access
  0.24    0.000017          17         1           fadvise64
------ ----------- ----------- --------- --------- ----------------
100.00    0.007223                   107         2 total

Directorio /proc​

  • Contiene información y conexiones para los procesos y del kernel.
  • Podemos utilizarlo para estudiar cómo trabajan los procesos
  • Posee configuraciones y tareas administrativas
  • Contiene archivos que no existen físicamente en disco y son creados de forma volátil.

Vamos a utilizar strace e investigar etcd.

  1. List Syscalls
  2. Find open files
  3. Read secret value
root@cks-master:~# crictl ps | grep etcd
5e90b3ef71d4e       2e96e5913fc06       4 days ago          Running             etcd                      2                   7e4de6c17bc46       etcd-cks-master

root@cks-master:~#  ps -aux | grep etcd
root        1431  6.6  9.3 1662360 375632 ?      Ssl  Sep02 392:41 kube-apiserver --encryption-provider-config=/etc/kubernetes/etcd/ec.yaml --advertise-address=10.128.0.5 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key

### Aquí el etcd como proceso y el pid 1511
root        1511  3.1  1.7 11739196 69280 ?      Ssl  Sep02 185:38 etcd --advertise-client-urls=https://10.128.0.5:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --experimental-initial-corrupt-check=true --experimental-watch-progress-notify-interval=5s --initial-advertise-peer-urls=https://10.128.0.5:2380 --initial-cluster=cks-master=https://10.128.0.5:2380 --key-file=/etc/kubernetes/pki/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://10.128.0.5:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://10.128.0.5:2380 --name=cks-master --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
root     2798918  0.0  0.0   8168  2500 pts/0    S+   19:14   0:00 grep --color=auto etcd

# Para ver las llamadas aconteciendo en tiempo real utilizamos el -p y pasamos el pid del proceso y podemos ver todas las llamadas en curso.
root@cks-master:~# strace -p 1511
strace: Process 1511 attached
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = -1 EAGAIN (Resource temporarily unavailable)
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
epoll_pwait(4, [], 128, 0, NULL, 0)     = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
nanosleep({tv_sec=0, tv_nsec=3000}, NULL) = 0
read(115, "\27\3\3\0/\322\330jE\376\224\352\2665\376\304|fEma\337\346\234\10\374\5\252\335?\20{"..., 1408) = 52
read(115, 0xc000ba2b00, 1408)           = -1 EAGAIN (Resource temporarily unavailable)
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
epoll_pwait(4, [], 128, 0, NULL, 0)     = 0
epoll_pwait(4, [], 128, 0, NULL, 0)     = 0
epoll_pwait(4, [{EPOLLIN|EPOLLOUT, {u32=360185859, u64=9154956854616915971}}], 128, 12, NULL, 0) = 1
futex(0x1a26100, FUTEX_WAKE_PRIVATE, 1) = 1

# Para facilitar podemos usar el -f (follow fork) y -cw para generar la lista. Es necesario esperar algunos segundos y abortar para ver la salida pues ella no se genera en tiempo real.

root@cks-master:~# strace -p 1511 -f -cw
strace: Process 1511 attached with 10 threads
strace: pid 1511: entering, ptrace_syscall_info.op == 2
^Cstrace: Process 1511 detached
strace: Process 1536 detached
strace: Process 1537 detached
strace: Process 1538 detached
strace: Process 1555 detached
strace: Process 1556 detached
strace: Process 1561 detached
strace: Process 1562 detached
strace: Process 1587 detached
strace: Process 135999 detached
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 77.08   23.125434       12019      1924       342 futex
 19.91    5.974530        2970      2011         3 epoll_pwait
  2.59    0.778417         380      2044           nanosleep
  0.16    0.047189         108       433           write # Son la mayoría
  0.14    0.042748        1257        34           fdatasync
  0.06    0.018214          42       429       176 read # Son la mayoría
  0.01    0.003903          67        58           tgkill
  0.01    0.002814          48        58         3 rt_sigreturn
  0.01    0.002355          40        58           getpid
  0.01    0.002128          39        54           pwrite64
  0.00    0.001333          27        49           setsockopt
  0.00    0.001005         143         7           close
  0.00    0.000669          47        14         7 accept4
  0.00    0.000654          46        14           epoll_ctl
  0.00    0.000343          28        12           lseek
  0.00    0.000270          38         7           getsockname
  0.00    0.000150          24         6           sched_yield
------ ----------- ----------- --------- --------- ----------------
100.00   30.002157                  7212       531 total

Yendo al proceso lo que tenemos.

root@cks-master:/proc/1511# ls
arch_status  cgroup      coredump_filter     environ  gid_map   map_files  mounts      numa_maps      pagemap      root       setgroups     stat     task            uid_map
attr         clear_refs  cpu_resctrl_groups  exe      io        maps       mountstats  oom_adj        patch_state  sched      smaps         statm    timens_offsets  wchan
autogroup    cmdline     cpuset              fd       limits    mem        net         oom_score      personality  schedstat  smaps_rollup  status   timers

# Podemos observar que el exe es un link full path para el binario del etcd que está siendo ejecutado.
root@cks-master:/proc/1511# ls -lha exe
lrwxrwxrwx 1 root root 0 Sep  2 16:04 exe -> /usr/local/bin/etcd

# El directorio fd contiene los archivos abiertos
root@cks-master:/proc/1511# cd fd
root@cks-master:/proc/1511/fd# ls
0    101  105  109  113  117  120  126  13   16  22  26  3   33  37  40  44  48  51  55  59  62  66  7   73  77  80  84  88  91  95  99
1    102  106  110  114  118  121  127  130  17  23  27  30  34  38  41  45  49  52  56  6   63  67  70  74  78  81  85  89  92  96
10   103  107  111  115  119  122  128  14   18  24  28  31  35  39  42  46  5   53  57  60  64  68  71  75  79  82  86  9   93  97
100  104  108  112  116  12   125  129  15   2   25  29  32  36  4   43  47  50  54  58  61  65  69  72  76  8   83  87  90  94  98

# Aquí podemos observar la lista de todos los archivos y sockets abiertos y que son enlaces para
root@cks-master:/proc/1511/fd#  ls -lha
total 0
dr-x------ 2 root root  0 Sep  2 16:04 .
dr-xr-xr-x 9 root root  0 Sep  2 16:04 ..
lrwx------ 1 root root 64 Sep  2 16:04 0 -> /dev/null
l-wx------ 1 root root 64 Sep  2 16:04 1 -> 'pipe:[28837]'
lrwx------ 1 root root 64 Sep  2 16:05 10 -> /var/lib/etcd/member/snap/db # La carpeta 10 está apuntando para aquí el db, vamos a verificar esto
lrwx------ 1 root root 64 Sep  2 16:05 100 -> 'socket:[34618]'
lrwx------ 1 root root 64 Sep  2 16:05 101 -> 'socket:[34623]'
#...
# Bloque removido para facilitar la lectura (todos sockets)
#...
lrwx------ 1 root root 64 Sep  2 16:05 118 -> 'socket:[34670]'
lrwx------ 1 root root 64 Sep  2 16:05 119 -> 'socket:[34673]'
lr-x------ 1 root root 64 Sep  2 16:05 12 -> /var/lib/etcd/member/wal
lrwx------ 1 root root 64 Sep  2 16:05 120 -> 'socket:[34675]'
lrwx------ 1 root root 64 Sep  2 16:05 121 -> 'socket:[34906]'
#...
# Bloque removido para facilitar la lectura (todos sockets)
#...
lrwx------ 1 root root 64 Sep  2 16:05 128 -> 'socket:[34915]'
lrwx------ 1 root root 64 Sep  2 16:05 129 -> 'socket:[34693]'
l-wx------ 1 root root 64 Sep  6 07:05 13 -> /var/lib/etcd/member/wal/1.tmp
lrwx------ 1 root root 64 Sep  2 16:05 130 -> 'socket:[34696]'
lrwx------ 1 root root 64 Sep  2 16:05 14 -> 'socket:[29116]'
lrwx------ 1 root root 64 Sep  2 16:05 15 -> 'socket:[30017]'
lrwx------ 1 root root 64 Sep  2 16:05 16 -> 'socket:[30018]'
lrwx------ 1 root root 64 Sep  2 16:05 17 -> 'socket:[30019]'
l-wx------ 1 root root 64 Sep  4 13:40 18 -> /var/lib/etcd/member/wal/0000000000000019-000000000023d2b1.wal
l-wx------ 1 root root 64 Sep  2 16:04 2 -> 'pipe:[28838]'
lrwx------ 1 root root 64 Sep  2 16:05 22 -> 'socket:[30039]'
lrwx------ 1 root root 64 Sep  2 16:05 23 -> 'socket:[29373]'
#...
# Bloque removido para facilitar la lectura (todos sockets)
#...
lrwx------ 1 root root 64 Sep  2 16:05 38 -> 'socket:[29444]'
lrwx------ 1 root root 64 Sep  2 16:05 39 -> 'socket:[29447]'
lrwx------ 1 root root 64 Sep  2 16:05 4 -> 'anon_inode:[eventpoll]'
lrwx------ 1 root root 64 Sep  2 16:05 40 -> 'socket:[29450]'
lrwx------ 1 root root 64 Sep  2 16:05 41 -> 'socket:[30102]'
#...
# Bloque removido para facilitar la lectura (todos sockets)
#...

# Vamos a intentar un poco de este archivo y ver qué tenemos. Leyendo las últimas 50 líneas podemos ver que tenemos cosas parecidas con yaml algún contenedor.
root@cks-master:/proc/1511/fd# tail -f 10 -n 50
ca.crtca.crt
)'
%
 namespace
v1metadata.namespace��
kubernetes-dashboard-web*docker.io/kubernetesui/dashboard-web:1.4.0" --namespace=kubernetes-dashboard"<--settings-config-map-name=kubernetes-dashboard-web-settings*2
web�>"TCP*:%

GOMAXPROCS�

limits.cpu�
0:(

GOMEMLIMIT�
limits.memory�
0BD

cpu
250m

memory
400Mi
cpu
100m
memory
200MiJ

tmp-volume�/tmp"2JL
kube-api-access-2bjbq�-/var/run/secrets/kubernetes.io/serviceaccount"2j/dev/termination-logr
                                                                                            IfNotPresentz
ALL �08@�����File�Always 2
                          ClusterFirstBkubernetes-dashboard-webJkubernetes-dashboard-webR
cks-workerX`hrR
RuntimeDefault���default-scheduler��6
node.kubernetes.io/not-readyExists�" NoExecute(��8
node.kubernetes.io/unreachableExists�" NoExecute(�����PreemptLowerPriority��
Running1
PodReadyToStartContainersTrue����*2#

InitializedTrue҇��*2
ReadyTrue����*2'
ContainersReadyTrue����*2$

PodScheduledTrue҇��*2�"*
10.128.0.42
           192.168.1.҇��B�
kubernetes-dashboard-web


�����u���Unknownԇ�����:Mcontainerd://5c4a07080b5a1d1551d59414d6496650aee7263ace8e8c2bac85ba2bee9801a0 (2*docker.io/kubernetesui/dashboard-web:1.4.0:ldocker.io/kubernetesui/dashboard-web@sha256:4445b31a2c25c875e2df8ca103a8e3f3275778d10065c7c011f6ca42cd4bec5fBMcontainerd://b78d3a18061a6ac00800b536cdd1017125149b5b47684b1acfeaed0c1f15b4f6Hb

tmp-volume/tmpbR
kube-api-access-2bjbq-/var/run/secrets/kubernetes.io/serviceaccountDisabledJ BurstableZb

192.168.1.7r�


10.128.0.4�"

Vamos a intentar crear un secret y ver qué tenemos.

root@cks-master:/proc/1511/fd# k create secret generic mynewsecret --from-literal=password=123456
secret/mynewsecret created

# Vamos a intentar buscar por la contraseña
root@cks-master:/proc/1511/fd# cat 10 | grep 123456 # Sin éxito
root@cks-master:/proc/1511/fd# cat 10 | grep mynewsecret
Binary file (standard input) matches
# Para leer es necesario usar strings, y encontramos esto
root@cks-master:/proc/1511/fd# cat 10 | strings | grep mynewsecret
%/registry/secrets/default/mynewsecret

# Vamos a leer 20 líneas después
root@cks-master:/proc/1511/fd# cat 10 | strings | grep mynewsecret -A20
%/registry/secrets/default/mynewsecret #<<<<
k8s:enc:aescbc:v1:key1: # Existe una encriptación para ese secret, pero si no la hubiera podríamos leerla aquí por eso que el grep 123456 no funcionó
4O\%s'
zI{N4YE
iqdc
?\wcACz3B
[f3Y
-/registry/events/default/pod.17f2bf3d82110f6f
Event
pod.17f2bf3d82110f6f
default"
*$bd9009e1-832d-465f-ae41-557b7c67b6172
kubelet
Update
FieldsV1:
{"f:count":{},"f:firstTimestamp":{},"f:involvedObject":{},"f:lastTimestamp":{},"f:message":{},"f:reason":{},"f:reportingComponent":{},"f:reportingInstance":{},"f:source":{"f:component":{},"f:host":{}},"f:type":{}}B
default
pod"$8621ad1f-e7b8-4768-9ae3-05b0ec9f2896*
2124582:
spec.containers{pod}
BackOff"aBack-off restarting failed container pod in pod pod_default(8621ad1f-e7b8-4768-9ae3-05b0ec9f2896)*
--
%/registry/secrets/default/mynewsecret
k8s:enc:aescbc:v1:key1:
4O\%s'
zI{N4YE
iqdc
?\wcACz3B
[f3Y
4/registry/leases/kube-system/kube-controller-manager
coordination.k8s.io/v1
Lease
kube-controller-manager
kube-system"
*$ea6e402d-dcd0-4060-8d0a-3ce53002ce4f2
kube-controller-manager
Update
coordination.k8s.io/v1"
FieldsV1:|
z{"f:spec":{"f:acquireTime":{},"f:holderIdentity":{},"f:leaseDurationSeconds":{},"f:leaseTransitions":{},"f:renewTime":{}}}B
/cks-master_c9b0ce4c-37ea-4ce7-afba-3aaecc6693b5
+/registry/leases/kube-node-lease/cks-worker
coordination.k8s.io/v1

Ahora vamos a poner un pod con una variable hardcoded con ese secret.

# no puedo crear nada en el directorio /proc, lo dejé solo para que veamos
root@cks-master:/proc/1511/fd# k run apache --image=httpd -oyaml --dry-run=client > apache.yaml
-bash: apache.yaml: No such file or directory
root@cks-master:/proc/1511/fd# cd
root@cks-master:~# k run apache --image=httpd -oyaml --dry-run=client > apache.yaml
root@cks-master:~# vim apache.yaml
root@cks-master:~# cat apache.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    run: apache
  name: apache
spec:
  containers:
  - image: httpd
    name: apache
    env: # Agregado
    - name: PASSWORD
      value: "123456"
    resources: {}
  dnsPolicy: ClusterFirst
  restartPolicy: Always
status: {}

root@cks-master:~# k create -f apache.yaml
pod/apache created

root@cks-master:~# k exec apache -- env | grep PASSWORD
PASSWORD=123456

# No tenemos nada pues está corriendo en el worker node
root@cks-master:~# ps -aux | grep httpd
root     2830278  0.0  0.0   8168   720 pts/0    S+   20:20   0:00 grep --color=auto httpd
root@cks-master:~# k get pod apache -o wide
NAME     READY   STATUS    RESTARTS   AGE   IP             NODE         NOMINATED NODE   READINESS GATES
apache   1/1     Running   0          16m   192.168.1.36   cks-worker   <none>           <none>

# EN EL Worker

# Tenemos nuestro contenedor corriendo, pero necesitamos el proceso
root@cks-worker:~# crictl ps | grep apache
43a100832c2bb       9cb0a23156020       19 minutes ago      Running             apache

# Buscamos por httpd que es el binario que está siendo ejecutado en el contenedor y tenemos algunos.
root@cks-worker:~# ps -aux | grep httpd
root     1447884  0.0  0.1   5860  4760 ?        Ss   20:05   0:00 httpd -DFOREGROUND
www-data 1447897  0.0  0.0 1210552 3584 ?        Sl   20:05   0:00 httpd -DFOREGROUND
www-data 1447898  0.0  0.0 1210552 3588 ?        Sl   20:05   0:00 httpd -DFOREGROUND
www-data 1447900  0.0  0.0 1210552 3588 ?        Sl   20:05   0:00 httpd -DFOREGROUND
root     1452924  0.0  0.0   8168  2376 pts/0    S+   20:23   0:00 grep --color=auto httpd

# Para saber cuál podemos usar el pstree -p y filtrar la salida
# El containerd inició el containerd-shim que inició el main process httpd 1447884 que es el que tenemos arriba
root@cks-worker:~# pstree -p | grep containerd-shim | grep httpd
           |-containerd-shim(1447830)-+-httpd(1447884)-+-httpd(1447897)-+-{httpd}(1447901)

root@cks-worker:~# cd /proc/1447884
root@cks-worker:/proc/1447884# ls
arch_status  cgroup      coredump_filter     environ  gid_map   map_files  mounts      numa_maps      pagemap      root       setgroups     stat     task            uid_map
attr         clear_refs  cpu_resctrl_groups  exe      io        maps       mountstats  oom_adj        patch_state  sched      smaps         statm    timens_offsets  wchan
autogroup    cmdline     cpuset              fd       limits    mem        net         oom_score      personality  schedstat  smaps_rollup  status   timers
auxv         comm        cwd                 fdinfo   loginuid  mountinfo  ns          oom_score_adj  projid_map   sessionid  stack         syscall  timerslack_ns

# Solo para mostrar a qué está apuntando el exe
root@cks-worker:/proc/1447884# ls -lha exe
lrwxrwxrwx 1 root root 0 Sep  6 20:05 exe -> /usr/local/apache2/bin/httpd

root@cks-worker:/proc/1447884# cat environ
HTTPD_VERSION=2.4.62KUBERNETES_SERVICE_PORT=443KUBERNETES_PORT=tcp://10.96.0.1:443HOSTNAME=apacheHOME=/rootHTTPD_PATCHES=APP1_SERVICE_HOST=10.105.1.235APP2_SERVICE_HOST=10.98.16.26APP1_SERVICE_PORT=80APP1_PORT=tcp://10.105.1.235:80APP2_PORT=tcp://10.98.16.26:80KUBERNETES_PORT_443_TCP_ADDR=10.96.0.1APP2_SERVICE_PORT=80PATH=/usr/local/apache2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binKUBERNETES_PORT_443_TCP_PORT=443HTTPD_SHA256=674188e7bf44ced82da8db522da946849e22080d73d16c93f7f4df89e25729ecKUBERNETES_PORT_443_TCP_PROTO=tcpAPP1_PORT_80_TCP_ADDR=10.105.1.235APP2_PORT_80_TCP_ADDR=10.98.16.26APP1_PORT_80_TCP_PORT=80APP1_PORT_80_TCP_PROTO=tcpAPP2_PORT_80_TCP_PORT=80APP2_PORT_80_TCP_PROTO=tcpHTTPD_PREFIX=/usr/local/apache2KUBERNETES_PORT_443_TCP=tcp://10.96.0.1:443KUBERNETES_SERVICE_PORT_HTTPS=443KUBERNETES_SERVICE_HOST=10.96.0.1PWD=/usr/local/apache2PASSWORD=123456APP1_PORT_80_TCP=tcp://10.105.1.235:80APP2_PORT_80_TCP=tcp://10.98.16.26:80root@cks-worker:/proc/1447884# cat environ  | grep PASSWORD

root@cks-worker:/proc/1447884# cat environ  | grep "PASSWORD"
Binary file (standard input) matches

root@cks-worker:/proc/1447884# cat environ  | strings |  grep "PASSWORD"
PASSWORD=123456

Haciendo un escaneo en todos los procesos podemos buscar una variable de estas. Elaborando un script mejor para detectar dónde apareció cada una de estas salidas será bien fácil saber qué es qué.

root@cks-worker:/proc/1447884# cat /proc/*/environ  | strings |  grep "PASSWORD"
PASSWORD=123456
PASSWORD=123456
PASSWORD=123456
PASSWORD=123456

Vea el peligro que es un acceso root en un nodo y cómo es fácil buscar todas las variables de entorno de todos los contenedores al mismo tiempo.