Skip to main content

Kernel Space Security

Review kernel space and user space if necessary.

We know that protecting syscalls is the most important point. Libraries interact with syscalls so we don't have to reinvent the wheel.

We can protect syscalls with tools like seccomp and apparmor by creating an additional layer between kernel space and user space. This should not be confused with sandboxes which operate at the user space level.

This protection layer will be part of kernel space, applying rules for syscall invocations.

alt text

Let's dive deeper into syscalls

man syscalls

The list is large, I'll point out some from the website that may be familiar:

  • add_key
  • chdir
  • chmod
  • chown
  • chown32
  • chroot
  • connect
  • copy_file_range
  • creat
  • delete_module
  • exit
  • getitimer
  • getpgid
  • getpgrp
  • getpid
  • getsid
  • gettid
  • gettimeofday
  • getuid
  • getxattr
  • kill
  • mkdir
  • mmap
  • mount
  • open
  • read
  • reboot
  • rename
  • rmdir
  • seccomp
  • sendfile
  • signal
  • splice
  • swapoff
  • swapon
  • symlink
  • symlinkat
  • sync
  • sync_file_range
  • sync_file_range2
  • syncfs
  • sysfs
  • sysinfo
  • syslog
  • tee
  • time
  • truncate
  • truncate64
  • ugetrlimit
  • umask
  • umount2
  • uname
  • write

Just to illustrate, take kill for example. Notice that we have the C language, which is why we use glibc and function calls as they should be.

alt text

Strace​

strace is a tool, usually pre-installed on most Linux distributions, that intercepts syscall calls and locks the call through a process so we can examine the stack trace of calls. Very good for learning, diagnosing, and debugging.

root@cks-master:~# ls /
bin  boot  dev  etc  home  lib  lib32  lib64  libx32  lost+found  media  mnt  opt  proc  root  run  sbin  snap  srv  sys  tmp  usr  var

# Here we'll see all the syscall calls made to execute ls /
root@cks-master:~# strace ls /
execve("/usr/bin/ls", ["ls", "/"], 0x7ffe6e4d8a18 /* 28 vars */) = 0
brk(NULL)                               = 0x55a637e8c000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffcf3871cc0) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=23997, ...}) = 0
mmap(NULL, 23997, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3b000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@p\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=163200, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7a39000
mmap(NULL, 174600, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a7a0e000
mprotect(0x7ff6a7a14000, 135168, PROT_NONE) = 0
mmap(0x7ff6a7a14000, 102400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7ff6a7a14000
mmap(0x7ff6a7a2d000, 28672, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f000) = 0x7ff6a7a2d000
mmap(0x7ff6a7a35000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x26000) = 0x7ff6a7a35000
mmap(0x7ff6a7a37000, 6664, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7a37000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300A\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32, 848) = 32
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\7\2C\n\357_\243\335\2449\206V>\237\374\304"..., 68, 880) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=2029592, ...}) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32, 848) = 32
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\7\2C\n\357_\243\335\2449\206V>\237\374\304"..., 68, 880) = 68
mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a781c000
mmap(0x7ff6a783e000, 1540096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7ff6a783e000
mmap(0x7ff6a79b6000, 319488, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7ff6a79b6000
mmap(0x7ff6a7a04000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7ff6a7a04000
mmap(0x7ff6a7a0a000, 13920, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7a0a000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\"\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=588488, ...}) = 0
mmap(NULL, 590632, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a778b000
mmap(0x7ff6a778d000, 413696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7ff6a778d000
mmap(0x7ff6a77f2000, 163840, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x67000) = 0x7ff6a77f2000
mmap(0x7ff6a781a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8e000) = 0x7ff6a781a000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \22\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18848, ...}) = 0
mmap(NULL, 20752, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a7785000
mmap(0x7ff6a7786000, 8192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7ff6a7786000
mmap(0x7ff6a7788000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7ff6a7788000
mmap(0x7ff6a7789000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7ff6a7789000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220q\0\0\0\0\0\0"..., 832) = 832
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\232e\273F\236E\241\306\373\317\372\345\270*/\327"..., 68, 824) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=157224, ...}) = 0
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\232e\273F\236E\241\306\373\317\372\345\270*/\327"..., 68, 824) = 68
mmap(NULL, 140408, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff6a7762000
mmap(0x7ff6a7768000, 69632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7ff6a7768000
mmap(0x7ff6a7779000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7ff6a7779000
mmap(0x7ff6a777f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c000) = 0x7ff6a777f000
mmap(0x7ff6a7781000, 13432, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7781000
close(3)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff6a7760000
arch_prctl(ARCH_SET_FS, 0x7ff6a7761400) = 0
mprotect(0x7ff6a7a04000, 16384, PROT_READ) = 0
mprotect(0x7ff6a777f000, 4096, PROT_READ) = 0
mprotect(0x7ff6a7789000, 4096, PROT_READ) = 0
mprotect(0x7ff6a781a000, 4096, PROT_READ) = 0
mprotect(0x7ff6a7a35000, 4096, PROT_READ) = 0
mprotect(0x55a636cab000, 4096, PROT_READ) = 0
mprotect(0x7ff6a7a6e000, 4096, PROT_READ) = 0
munmap(0x7ff6a7a3b000, 23997)           = 0
set_tid_address(0x7ff6a77616d0)         = 2161540
set_robust_list(0x7ff6a77616e0, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7ff6a7768bf0, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7ff6a7776420}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7ff6a7768c90, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7ff6a7776420}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0x7ffcf3871c10) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7ffcf3871c10)      = -1 ENOENT (No such file or directory)
brk(NULL)                               = 0x55a637e8c000
brk(0x55a637ead000)                     = 0x55a637ead000
openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(3, "nodev\tsysfs\nnodev\ttmpfs\nnodev\tbd"..., 1024) = 418
read(3, "", 1024)                       = 0
close(3)                                = 0
access("/etc/selinux/config", F_OK)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=3035952, ...}) = 0
mmap(NULL, 3035952, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a747a000
close(3)                                = 0
openat(AT_FDCWD, "/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2996, ...}) = 0
read(3, "# Locale name alias data base.\n#"..., 4096) = 2996
read(3, "", 4096)                       = 0
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=252, ...}) = 0
mmap(NULL, 252, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a6d000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/gconv/gconv-modules.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=27002, ...}) = 0
mmap(NULL, 27002, PROT_READ, MAP_SHARED, 3, 0) = 0x7ff6a7473000
close(3)                                = 0
futex(0x7ff6a7a09954, FUTEX_WAKE_PRIVATE, 2147483647) = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MEASUREMENT", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=23, ...}) = 0
mmap(NULL, 23, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a40000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_TELEPHONE", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=47, ...}) = 0
mmap(NULL, 47, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3f000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_ADDRESS", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=131, ...}) = 0
mmap(NULL, 131, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3e000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_NAME", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=62, ...}) = 0
mmap(NULL, 62, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3d000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_PAPER", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=34, ...}) = 0
mmap(NULL, 34, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3c000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MESSAGES", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MESSAGES/SYS_LC_MESSAGES", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=48, ...}) = 0
mmap(NULL, 48, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7a3b000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_MONETARY", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=270, ...}) = 0
mmap(NULL, 270, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a7472000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_COLLATE", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1518110, ...}) = 0
mmap(NULL, 1518110, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72ff000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_TIME", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=3360, ...}) = 0
mmap(NULL, 3360, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72fe000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_NUMERIC", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=50, ...}) = 0
mmap(NULL, 50, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72fd000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/locale/C.UTF-8/LC_CTYPE", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=201272, ...}) = 0
mmap(NULL, 201272, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff6a72cb000
close(3)                                = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=40, ws_col=174, ws_xpixel=0, ws_ypixel=0}) = 0
stat("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
openat(AT_FDCWD, "/", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
getdents64(3, /* 25 entries */, 32768)  = 640
getdents64(3, /* 0 entries */, 32768)   = 0
close(3)                                = 0
fstat(1, {st_mode=S_IFCHR|0600, st_rdev=makedev(0x88, 0), ...}) = 0
write(1, "bin  boot  dev\tetc  home  lib\tli"..., 133bin  boot  dev etc  home  lib lib32  lib64  libx32  lost+found  media  mnt  opt  proc  root  run  sbin  snap srv  sys  tmp  usr  var
) = 133
close(1)                                = 0
close(2)                                = 0
exit_group(0)                           = ?
+++ exited with 0 +++
root@cks-master:~#

# For more verbose output
strace -v ls /

We can count how many times each syscall was invoked in the command.

root@cks-master:~# strace -cw ls /
bin  boot  dev etc  home  lib lib32  lib64  libx32  lost+found  media  mnt  opt  proc  root  run  sbin  snap srv  sys  tmp  usr  var
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 23.58    0.000995          24        40           mmap
 16.55    0.000699          29        24           openat
 12.89    0.000544          20        26           close
 11.64    0.000492          19        25           fstat
  6.77    0.000286         285         1           execve
  4.79    0.000202          22         9           read
  4.58    0.000193          24         8           mprotect
  4.57    0.000193         192         1           write
  3.29    0.000139          17         8           pread64
  1.48    0.000063          31         2           getdents64
  1.28    0.000054          17         3           brk
  1.15    0.000048          24         2         2 statfs
  1.11    0.000047          23         2         2 access
  0.97    0.000041          20         2           rt_sigaction
  0.92    0.000039          19         2           ioctl
  0.86    0.000036          36         1           munmap
  0.81    0.000034          17         2         1 arch_prctl
  0.67    0.000028          28         1           set_robust_list
  0.45    0.000019          18         1           stat
  0.42    0.000018          17         1           rt_sigprocmask
  0.42    0.000018          17         1           set_tid_address
  0.42    0.000018          17         1           futex
  0.40    0.000017          16         1           prlimit64
------ ----------- ----------- --------- --------- ----------------
100.00    0.004222                   164         5 total

Doing a cat on a file for example already gives us different calls.

root@cks-master:~# echo "hello world" > hello
root@cks-master:~# strace -cw cat hello
hello world
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 58.55    0.004229         211        20           fstat
 11.65    0.000842          38        22           mmap
 11.24    0.000812          42        19           openat
  6.68    0.000482          22        21           close
  4.22    0.000305         305         1           execve
  1.65    0.000119          23         5           read
  1.45    0.000104          17         6           pread64
  1.13    0.000082          27         3           mprotect
  0.94    0.000068          33         2           munmap
  0.83    0.000060          19         3           brk
  0.54    0.000039          39         1           write
  0.52    0.000038          18         2         1 arch_prctl
  0.36    0.000026          25         1         1 access
  0.24    0.000017          17         1           fadvise64
------ ----------- ----------- --------- --------- ----------------
100.00    0.007223                   107         2 total

The /proc Directory​

  • Contains information and connections to processes and the kernel
  • We can use it to study how processes work
  • Contains configurations and administrative tasks
  • Contains files that don't physically exist on disk and are created volatilely

Let's use strace to investigate etcd.

  1. List Syscalls
  2. Find open files
  3. Read secret value
root@cks-master:~# crictl ps | grep etcd
5e90b3ef71d4e       2e96e5913fc06       4 days ago          Running             etcd                      2                   7e4de6c17bc46       etcd-cks-master

root@cks-master:~#  ps -aux | grep etcd
root        1431  6.6  9.3 1662360 375632 ?      Ssl  Sep02 392:41 kube-apiserver --encryption-provider-config=/etc/kubernetes/etcd/ec.yaml --advertise-address=10.128.0.5 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key

### Here etcd as a process with pid 1511
root        1511  3.1  1.7 11739196 69280 ?      Ssl  Sep02 185:38 etcd --advertise-client-urls=https://10.128.0.5:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --experimental-initial-corrupt-check=true --experimental-watch-progress-notify-interval=5s --initial-advertise-peer-urls=https://10.128.0.5:2380 --initial-cluster=cks-master=https://10.128.0.5:2380 --key-file=/etc/kubernetes/pki/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://10.128.0.5:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://10.128.0.5:2380 --name=cks-master --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
root     2798918  0.0  0.0   8168  2500 pts/0    S+   19:14   0:00 grep --color=auto etcd

# To see calls happening in real-time we use -p and pass the process pid and we can see all ongoing calls
root@cks-master:~# strace -p 1511
strace: Process 1511 attached
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = -1 EAGAIN (Resource temporarily unavailable)
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
epoll_pwait(4, [], 128, 0, NULL, 0)     = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
nanosleep({tv_sec=0, tv_nsec=3000}, NULL) = 0
read(115, "\27\3\3\0/\322\330jE\376\224\352\2665\376\304|fEma\337\346\234\10\374\5\252\335?\20{"..., 1408) = 52
read(115, 0xc000ba2b00, 1408)           = -1 EAGAIN (Resource temporarily unavailable)
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
write(6, "\0", 1)                       = 1
futex(0x1a25d48, FUTEX_WAIT_PRIVATE, 0, NULL) = 0
epoll_pwait(4, [], 128, 0, NULL, 0)     = 0
epoll_pwait(4, [], 128, 0, NULL, 0)     = 0
epoll_pwait(4, [{EPOLLIN|EPOLLOUT, {u32=360185859, u64=9154956854616915971}}], 128, 12, NULL, 0) = 1
futex(0x1a26100, FUTEX_WAKE_PRIVATE, 1) = 1

# To make it easier we can use -f (follow fork) and -cw to generate the list. You need to wait a few seconds and abort to see the output as it doesn't generate in real-time

root@cks-master:~# strace -p 1511 -f -cw
strace: Process 1511 attached with 10 threads
strace: pid 1511: entering, ptrace_syscall_info.op == 2
^Cstrace: Process 1511 detached
strace: Process 1536 detached
strace: Process 1537 detached
strace: Process 1538 detached
strace: Process 1555 detached
strace: Process 1556 detached
strace: Process 1561 detached
strace: Process 1562 detached
strace: Process 1587 detached
strace: Process 135999 detached
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 77.08   23.125434       12019      1924       342 futex
 19.91    5.974530        2970      2011         3 epoll_pwait
  2.59    0.778417         380      2044           nanosleep
  0.16    0.047189         108       433           write # Most common
  0.14    0.042748        1257        34           fdatasync
  0.06    0.018214          42       429       176 read # Most common
  0.01    0.003903          67        58           tgkill
  0.01    0.002814          48        58         3 rt_sigreturn
  0.01    0.002355          40        58           getpid
  0.01    0.002128          39        54           pwrite64
  0.00    0.001333          27        49           setsockopt
  0.00    0.001005         143         7           close
  0.00    0.000669          47        14         7 accept4
  0.00    0.000654          46        14           epoll_ctl
  0.00    0.000343          28        12           lseek
  0.00    0.000270          38         7           getsockname
  0.00    0.000150          24         6           sched_yield
------ ----------- ----------- --------- --------- ----------------
100.00   30.002157                  7212       531 total

Going to the process, what do we have.

root@cks-master:/proc/1511# ls
arch_status  cgroup      coredump_filter     environ  gid_map   map_files  mounts      numa_maps      pagemap      root       setgroups     stat     task            uid_map
attr         clear_refs  cpu_resctrl_groups  exe      io        maps       mountstats  oom_adj        patch_state  sched      smaps         statm    timens_offsets  wchan
autogroup    cmdline     cpuset              fd       limits    mem        net         oom_score      personality  schedstat  smaps_rollup  status   timers

# We can observe that exe is a full path link to the etcd binary being executed
root@cks-master:/proc/1511# ls -lha exe
lrwxrwxrwx 1 root root 0 Sep  2 16:04 exe -> /usr/local/bin/etcd

# The fd directory contains open files
root@cks-master:/proc/1511# cd fd
root@cks-master:/proc/1511/fd# ls
0    101  105  109  113  117  120  126  13   16  22  26  3   33  37  40  44  48  51  55  59  62  66  7   73  77  80  84  88  91  95  99
1    102  106  110  114  118  121  127  130  17  23  27  30  34  38  41  45  49  52  56  6   63  67  70  74  78  81  85  89  92  96
10   103  107  111  115  119  122  128  14   18  24  28  31  35  39  42  46  5   53  57  60  64  68  71  75  79  82  86  9   93  97
100  104  108  112  116  12   125  129  15   2   25  29  32  36  4   43  47  50  54  58  61  65  69  72  76  8   83  87  90  94  98

# Here we can see the list of all open files and sockets which are links to
root@cks-master:/proc/1511/fd#  ls -lha
total 0
dr-x------ 2 root root  0 Sep  2 16:04 .
dr-xr-xr-x 9 root root  0 Sep  2 16:04 ..
lrwx------ 1 root root 64 Sep  2 16:04 0 -> /dev/null
l-wx------ 1 root root 64 Sep  2 16:04 1 -> 'pipe:[28837]'
lrwx------ 1 root root 64 Sep  2 16:05 10 -> /var/lib/etcd/member/snap/db # Folder 10 is pointing to the db here, let's check this
lrwx------ 1 root root 64 Sep  2 16:05 100 -> 'socket:[34618]'
lrwx------ 1 root root 64 Sep  2 16:05 101 -> 'socket:[34623]'
#...
# Block removed for easier reading (all sockets)
#...
lrwx------ 1 root root 64 Sep  2 16:05 118 -> 'socket:[34670]'
lrwx------ 1 root root 64 Sep  2 16:05 119 -> 'socket:[34673]'
lr-x------ 1 root root 64 Sep  2 16:05 12 -> /var/lib/etcd/member/wal
lrwx------ 1 root root 64 Sep  2 16:05 120 -> 'socket:[34675]'
lrwx------ 1 root root 64 Sep  2 16:05 121 -> 'socket:[34906]'
#...
# Block removed for easier reading (all sockets)
#...
lrwx------ 1 root root 64 Sep  2 16:05 128 -> 'socket:[34915]'
lrwx------ 1 root root 64 Sep  2 16:05 129 -> 'socket:[34693]'
l-wx------ 1 root root 64 Sep  6 07:05 13 -> /var/lib/etcd/member/wal/1.tmp
lrwx------ 1 root root 64 Sep  2 16:05 130 -> 'socket:[34696]'
lrwx------ 1 root root 64 Sep  2 16:05 14 -> 'socket:[29116]'
lrwx------ 1 root root 64 Sep  2 16:05 15 -> 'socket:[30017]'
lrwx------ 1 root root 64 Sep  2 16:05 16 -> 'socket:[30018]'
lrwx------ 1 root root 64 Sep  2 16:05 17 -> 'socket:[30019]'
l-wx------ 1 root root 64 Sep  4 13:40 18 -> /var/lib/etcd/member/wal/0000000000000019-000000000023d2b1.wal
l-wx------ 1 root root 64 Sep  2 16:04 2 -> 'pipe:[28838]'
lrwx------ 1 root root 64 Sep  2 16:05 22 -> 'socket:[30039]'
lrwx------ 1 root root 64 Sep  2 16:05 23 -> 'socket:[29373]'
#...
# Block removed for easier reading (all sockets)
#...
lrwx------ 1 root root 64 Sep  2 16:05 38 -> 'socket:[29444]'
lrwx------ 1 root root 64 Sep  2 16:05 39 -> 'socket:[29447]'
lrwx------ 1 root root 64 Sep  2 16:05 4 -> 'anon_inode:[eventpoll]'
lrwx------ 1 root root 64 Sep  2 16:05 40 -> 'socket:[29450]'
lrwx------ 1 root root 64 Sep  2 16:05 41 -> 'socket:[30102]'
#...
# Block removed for easier reading (all sockets)
#...

# Let's try to read a bit from this file and see what we have. Reading the last 50 lines we can see we have something similar to yaml from a container
root@cks-master:/proc/1511/fd# tail -f 10 -n 50
ca.crtca.crt
)'
%
 namespace
v1metadata.namespace��
kubernetes-dashboard-web*docker.io/kubernetesui/dashboard-web:1.4.0" --namespace=kubernetes-dashboard"<--settings-config-map-name=kubernetes-dashboard-web-settings*2
web�>"TCP*:%

GOMAXPROCS�

limits.cpu�
0:(

GOMEMLIMIT�
limits.memory�
0BD

cpu
250m

memory
400Mi
cpu
100m
memory
200MiJ

tmp-volume�/tmp"2JL
kube-api-access-2bjbq�-/var/run/secrets/kubernetes.io/serviceaccount"2j/dev/termination-logr
                                                                                            IfNotPresentz
ALL �08@�����File�Always 2
                          ClusterFirstBkubernetes-dashboard-webJkubernetes-dashboard-webR
cks-workerX`hrR
RuntimeDefault���default-scheduler��6
node.kubernetes.io/not-readyExists�" NoExecute(��8
node.kubernetes.io/unreachableExists�" NoExecute(�����PreemptLowerPriority��
Running1
PodReadyToStartContainersTrue����*2#

InitializedTrue҇��*2
ReadyTrue����*2'
ContainersReadyTrue����*2$

PodScheduledTrue҇��*2�"*
10.128.0.42
           192.168.1.҇��B�
kubernetes-dashboard-web


�����u���Unknownԇ�����:Mcontainerd://5c4a07080b5a1d1551d59414d6496650aee7263ace8e8c2bac85ba2bee9801a0 (2*docker.io/kubernetesui/dashboard-web:1.4.0:ldocker.io/kubernetesui/dashboard-web@sha256:4445b31a2c25c875e2df8ca103a8e3f3275778d10065c7c011f6ca42cd4bec5fBMcontainerd://b78d3a18061a6ac00800b536cdd1017125149b5b47684b1acfeaed0c1f15b4f6Hb

tmp-volume/tmpbR
kube-api-access-2bjbq-/var/run/secrets/kubernetes.io/serviceaccountDisabledJ BurstableZb

192.168.1.7r�


10.128.0.4�"

Let's try to create a secret and see what we have.

root@cks-master:/proc/1511/fd# k create secret generic mynewsecret --from-literal=password=123456
secret/mynewsecret created

# Let's try to search for the password
root@cks-master:/proc/1511/fd# cat 10 | grep 123456 # Unsuccessful
root@cks-master:/proc/1511/fd# cat 10 | grep mynewsecret
Binary file (standard input) matches
# To read we need to use strings, and we find this
root@cks-master:/proc/1511/fd# cat 10 | strings | grep mynewsecret
%/registry/secrets/default/mynewsecret

# Let's read 20 lines after
root@cks-master:/proc/1511/fd# cat 10 | strings | grep mynewsecret -A20
%/registry/secrets/default/mynewsecret #<<<<
k8s:enc:aescbc:v1:key1: # There is encryption for this secret, but if there wasn't we could read it here, which is why grep 123456 didn't find it
4O\%s'
zI{N4YE
iqdc
?\wcACz3B
[f3Y
-/registry/events/default/pod.17f2bf3d82110f6f
Event
pod.17f2bf3d82110f6f
default"
*$bd9009e1-832d-465f-ae41-557b7c67b6172
kubelet
Update
FieldsV1:
{"f:count":{},"f:firstTimestamp":{},"f:involvedObject":{},"f:lastTimestamp":{},"f:message":{},"f:reason":{},"f:reportingComponent":{},"f:reportingInstance":{},"f:source":{"f:component":{},"f:host":{}},"f:type":{}}B
default
pod"$8621ad1f-e7b8-4768-9ae3-05b0ec9f2896*
2124582:
spec.containers{pod}
BackOff"aBack-off restarting failed container pod in pod pod_default(8621ad1f-e7b8-4768-9ae3-05b0ec9f2896)*
--
%/registry/secrets/default/mynewsecret
k8s:enc:aescbc:v1:key1:
4O\%s'
zI{N4YE
iqdc
?\wcACz3B
[f3Y
4/registry/leases/kube-system/kube-controller-manager
coordination.k8s.io/v1
Lease
kube-controller-manager
kube-system"
*$ea6e402d-dcd0-4060-8d0a-3ce53002ce4f2
kube-controller-manager
Update
coordination.k8s.io/v1"
FieldsV1:|
z{"f:spec":{"f:acquireTime":{},"f:holderIdentity":{},"f:leaseDurationSeconds":{},"f:leaseTransitions":{},"f:renewTime":{}}}B
/cks-master_c9b0ce4c-37ea-4ce7-afba-3aaecc6693b5
+/registry/leases/kube-node-lease/cks-worker
coordination.k8s.io/v1

Now let's create a pod with a hardcoded variable with this secret.

# Can't create anything in the /proc directory, just showing for reference
root@cks-master:/proc/1511/fd# k run apache --image=httpd -oyaml --dry-run=client > apache.yaml
-bash: apache.yaml: No such file or directory
root@cks-master:/proc/1511/fd# cd
root@cks-master:~# k run apache --image=httpd -oyaml --dry-run=client > apache.yaml
root@cks-master:~# vim apache.yaml
root@cks-master:~# cat apache.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    run: apache
  name: apache
spec:
  containers:
  - image: httpd
    name: apache
    env: # Added
    - name: PASSWORD
      value: "123456"
    resources: {}
  dnsPolicy: ClusterFirst
  restartPolicy: Always
status: {}

root@cks-master:~# k create -f apache.yaml
pod/apache created

root@cks-master:~# k exec apache -- env | grep PASSWORD
PASSWORD=123456

# Nothing here because it's running on the worker node
root@cks-master:~# ps -aux | grep httpd
root     2830278  0.0  0.0   8168   720 pts/0    S+   20:20   0:00 grep --color=auto httpd
root@cks-master:~# k get pod apache -o wide
NAME     READY   STATUS    RESTARTS   AGE   IP             NODE         NOMINATED NODE   READINESS GATES
apache   1/1     Running   0          16m   192.168.1.36   cks-worker   <none>           <none>

# On Worker

# We have our container running, but we need the process
root@cks-worker:~# crictl ps | grep apache
43a100832c2bb       9cb0a23156020       19 minutes ago      Running             apache

# We search for httpd which is the binary being executed in the container and we have some
root@cks-worker:~# ps -aux | grep httpd
root     1447884  0.0  0.1   5860  4760 ?        Ss   20:05   0:00 httpd -DFOREGROUND
www-data 1447897  0.0  0.0 1210552 3584 ?        Sl   20:05   0:00 httpd -DFOREGROUND
www-data 1447898  0.0  0.0 1210552 3588 ?        Sl   20:05   0:00 httpd -DFOREGROUND
www-data 1447900  0.0  0.0 1210552 3588 ?        Sl   20:05   0:00 httpd -DFOREGROUND
root     1452924  0.0  0.0   8168  2376 pts/0    S+   20:23   0:00 grep --color=auto httpd

# To know which one we can use pstree -p and filter the output
# Containerd started containerd-shim which started the main process httpd 1447884 which is what we have above
root@cks-worker:~# pstree -p | grep containerd-shim | grep httpd
           |-containerd-shim(1447830)-+-httpd(1447884)-+-httpd(1447897)-+-{httpd}(1447901)

root@cks-worker:~# cd /proc/1447884
root@cks-worker:/proc/1447884# ls
arch_status  cgroup      coredump_filter     environ  gid_map   map_files  mounts      numa_maps      pagemap      root       setgroups     stat     task            uid_map
attr         clear_refs  cpu_resctrl_groups  exe      io        maps       mountstats  oom_adj        patch_state  sched      smaps         statm    timens_offsets  wchan
autogroup    cmdline     cpuset              fd       limits    mem        net         oom_score      personality  schedstat  smaps_rollup  status   timers
auxv         comm        cwd                 fdinfo   loginuid  mountinfo  ns          oom_score_adj  projid_map   sessionid  stack         syscall  timerslack_ns

# Just to show what exe is pointing to
root@cks-worker:/proc/1447884# ls -lha exe
lrwxrwxrwx 1 root root 0 Sep  6 20:05 exe -> /usr/local/apache2/bin/httpd

root@cks-worker:/proc/1447884# cat environ
HTTPD_VERSION=2.4.62KUBERNETES_SERVICE_PORT=443KUBERNETES_PORT=tcp://10.96.0.1:443HOSTNAME=apacheHOME=/rootHTTPD_PATCHES=APP1_SERVICE_HOST=10.105.1.235APP2_SERVICE_HOST=10.98.16.26APP1_SERVICE_PORT=80APP1_PORT=tcp://10.105.1.235:80APP2_PORT=tcp://10.98.16.26:80KUBERNETES_PORT_443_TCP_ADDR=10.96.0.1APP2_SERVICE_PORT=80PATH=/usr/local/apache2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binKUBERNETES_PORT_443_TCP_PORT=443HTTPD_SHA256=674188e7bf44ced82da8db522da946849e22080d73d16c93f7f4df89e25729ecKUBERNETES_PORT_443_TCP_PROTO=tcpAPP1_PORT_80_TCP_ADDR=10.105.1.235APP2_PORT_80_TCP_ADDR=10.98.16.26APP1_PORT_80_TCP_PORT=80APP1_PORT_80_TCP_PROTO=tcpAPP2_PORT_80_TCP_PORT=80APP2_PORT_80_TCP_PROTO=tcpHTTPD_PREFIX=/usr/local/apache2KUBERNETES_PORT_443_TCP=tcp://10.96.0.1:443KUBERNETES_SERVICE_PORT_HTTPS=443KUBERNETES_SERVICE_HOST=10.96.0.1PWD=/usr/local/apache2PASSWORD=123456APP1_PORT_80_TCP=tcp://10.105.1.235:80APP2_PORT_80_TCP=tcp://10.98.16.26:80root@cks-worker:/proc/1447884# cat environ  | grep PASSWORD

root@cks-worker:/proc/1447884# cat environ  | grep "PASSWORD"
Binary file (standard input) matches

root@cks-worker:/proc/1447884# cat environ  | strings |  grep "PASSWORD"
PASSWORD=123456

Doing a scan of all processes we can search for such a variable. By crafting a better script to detect where each of these outputs appeared, it will be quite easy to know what is what.

root@cks-worker:/proc/1447884# cat /proc/*/environ  | strings |  grep "PASSWORD"
PASSWORD=123456
PASSWORD=123456
PASSWORD=123456
PASSWORD=123456

See the danger of root access on a node and how easy it is to search for all environment variables of all containers at once.