K8s学习笔记——深入理解容器镜像

学习极客时间上的《深入剖析Kubernetes》

秉持眼过千遍不如手过一遍的原则.

对应章节：07 | 白话容器基础（三）：深入理解容器镜像

以系统调用方式创建namespace实验

ns.c

#define _GNU_SOURCE
#include <sys/mount.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <stdio.h>
#include <sched.h>
#include <signal.h>
#include <unistd.h>
#define STACK_SIZE (1024 * 1024)
static char container_stack[STACK_SIZE];
char* const container_args[] = {
  "/bin/bash",
  NULL
};

int container_main(void* arg)
{
  printf("Container - inside the container!\n");
  execv(container_args[0], container_args);
  printf("Something's wrong!\n");
  return 1;
}

int main()
{
  printf("Parent - start a container!\n");
  int container_pid = clone(container_main, container_stack+STACK_SIZE, CLONE_NEWNS | SIGCHLD , NULL);
  waitpid(container_pid, NULL, 0);
  printf("Parent - container stopped!\n");
  return 0;
}

build ns并进入ns

$ gcc -o ns ns.c
$ ./ns
Parent - start a container!
Container - inside the container!
$ ls /tmp
systemd-private-8bac4934482d484d879054051ff48730-systemd-resolved.service-54yGbQ   vmware-root_563-4281712267
systemd-private-8bac4934482d484d879054051ff48730-systemd-timesyncd.service-ti2Lns
$ exit
exit
Parent - container stopped!

查看进程(在另外一个窗口中执行)

$ ps -aux
...
root     12217  0.0  0.0   5532   720 pts/3    S    06:41   0:00 ./ns
root     12218  0.0  0.3  20312  3980 pts/3    S+   06:41   0:00 /bin/bash
...
$ pstree -g
systemd(1)─┬─VGAuthService(562)
           ├─sshd(1144)─┬─sshd(2307)───bash(2394)
           │            ├─sshd(2448)───bash(2529)
           │            ├─sshd(11698)───bash(11784)───pstree(12265)
           │            └─sshd(11845)───bash(11927)───ns(12217)───bash(12218)
           ...

由上面的操作可见，即使开启了namespace，容器进程看到的文件系统与宿主机一样

重新挂载目录实验

修改代码

修改ns.c的container_main函数，新创建文件ns_new.c

注：因为是在虚拟机上实验，根目录类型默认是shared，所以，需要先重新挂载根目录

int container_main(void* arg)
{
  printf("Container - inside the container!\n");
  // 如果你的机器的根目录的挂载类型是shared，那必须先重新挂载根目录
  mount("", "/", NULL, MS_PRIVATE, "");
  mount("none", "/tmp", "tmpfs", 0, "");
  execv(container_args[0], container_args);
  printf("Something's wrong!\n");
  return 1;
}

编译及测试效果

在容器内

$ gcc -o ns_new ns_new.c
$ ./ns_new
Parent - start a container!
Container - inside the container!
$ ls /tmp
$ mount -l | grep tmpfs
udev on /dev type devtmpfs (rw,nosuid,relatime,size=473204k,nr_inodes=118301,mode=755)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=100928k,mode=755)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=100924k,mode=700)
none on /tmp type tmpfs (rw,relatime)
$ exit
exit
Parent - container stopped!

在宿主机上

$ mount -l | grep tmpfs
udev on /dev type devtmpfs (rw,nosuid,relatime,size=473204k,nr_inodes=118301,mode=755)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=100928k,mode=755)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=100924k,mode=700)

chroot实验

准备

$ mkdir test
$ mkdir -p test/{bin,lib64,lib}
# 拷贝bash和ls命令
$ cp -v /bin/{bash,ls} test/bin/
'/bin/bash' -> 'test/bin/bash'
'/bin/ls' -> 'test/bin/ls'
# 拷贝lib
$ list="$(ldd /bin/ls | egrep -o '/lib.*\.[0-9]')"
$ mkdir test/lib/x86_64-linux-gnu
$ for i in $list; do cp -v "$i" "test${i}"; done
'/lib/x86_64-linux-gnu/libselinux.so.1' -> 'test/lib/x86_64-linux-gnu/libselinux.so.1'
'/lib/x86_64-linux-gnu/libc.so.6' -> 'test/lib/x86_64-linux-gnu/libc.so.6'
'/lib/x86_64-linux-gnu/libpcre.so.3' -> 'test/lib/x86_64-linux-gnu/libpcre.so.3'
'/lib/x86_64-linux-gnu/libdl.so.2' -> 'test/lib/x86_64-linux-gnu/libdl.so.2'
'/lib64/ld-linux-x86-64.so.2' -> 'test/lib64/ld-linux-x86-64.so.2'
'/lib/x86_64-linux-gnu/libpthread.so.0' -> 'test/lib/x86_64-linux-gnu/libpthread.so.0'
$ cp /lib/x86_64-linux-gnu/libtinfo.so.5 test/lib/x86_64-linux-gnu/

chroot

1
2
3

$ chroot test /bin/bash
bash-4.4# ls
bin  lib  lib64

以busybox的镜像为例，同样可以chroot

1
2
3

$ chroot busybox /bin/sh
/ # ls
bin   dev   etc   home  root  tmp   usr   var

UnionFS实验

$ mkdir A
$ mkdir B
$ mkdi^C
$ touch A/t1.txt
$ touch A/t2.txt
$ touch B/t2.txt
$ touch B/t3.txt
$ mkdir C
$ mount -t aufs -o dirs=./A:./B none ./C
$ ls C
t1.txt  t2.txt  t3.txt
$ mount -l | grep aufs
none on /root/bqi/C type aufs (rw,relatime,si=f9e234d74656f278)

此时，可以尝试修改A/t1.txt, A/t2.txt, B/t2.txt, B/t3.txt

$ echo 'This is a test' > A/t1.txt
$ cat C/t1.txt
This is a test
$ echo 'This is a test2' > A/t2.txt
$ cat C/t2.txt
This is a test2
$ cat B/t2.txt
$ echo 'This is test2 for B' > B/t2.txt
$ cat C/t2.txt
This is a test2

docker image解析

$ docker inspect ubuntu
...
        "GraphDriver": {
            "Data": {
                "LowerDir": "/var/lib/docker/overlay2/823e415d4256d05fb0101af4dcc42a4389d44cf6467972d654e93e0cc575cd9b/diff:/var/lib/docker/overlay2/37d3e588905fae55c8a0481e9cda7be36177af874631abb15724c893887e260b/diff:/var/lib/docker/overlay2/40d198d6f624e455800254766eb6a7190ce02442fc48f02f6f16f72105cefd0d/diff",
                "MergedDir": "/var/lib/docker/overlay2/17bd5da0cda20e8ecd1d4955d25f49609ff0d7aa72fe45a0388a357fcc5b625f/merged",
                "UpperDir": "/var/lib/docker/overlay2/17bd5da0cda20e8ecd1d4955d25f49609ff0d7aa72fe45a0388a357fcc5b625f/diff",
                "WorkDir": "/var/lib/docker/overlay2/17bd5da0cda20e8ecd1d4955d25f49609ff0d7aa72fe45a0388a357fcc5b625f/work"
            },
            "Name": "overlay2"
        },
        "RootFS": {
            "Type": "layers",
            "Layers": [
                "sha256:7789f1a3d4e9258fbe5469a8d657deb6aba168d86967063e9b80ac3e1154333f",
                "sha256:9e53fd4895597d04f8871a68caea4c686011e1fbd0be32e57e89ada2ea5c24c4",
                "sha256:2a19bd70fcd4ce7fd73b37b1b2c710f8065817a9db821ff839fe0b4b4560e643",
                "sha256:8891751e0a1733c5c214d17ad2b0040deccbdea0acebb963679735964d516ac2"
            ]
        },
...

你会看到，Ubuntu镜像，在我的环境里面是4层

overlay挂载方式

先启动一个container

$ docker run -it -d ubuntu
$ docker ps
CONTAINER ID        IMAGE               COMMAND             CREATED             STATUS              PORTS               NAMES
a274e38c218a        ubuntu              "/bin/bash"         17 minutes ago      Up 17 minutes                           musing_knuth
$ docker inspect a274e38c218a
...
        "GraphDriver": {
            "Data": {
                "LowerDir": "/var/lib/docker/overlay2/0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2-init/diff:/var/lib/docker/overlay2/17bd5da0cda20e8ecd1d4955d25f49609ff0d7aa72fe45a0388a357fcc5b625f/diff:/var/lib/docker/overlay2/823e415d4256d05fb0101af4dcc42a4389d44cf6467972d654e93e0cc575cd9b/diff:/var/lib/docker/overlay2/37d3e588905fae55c8a0481e9cda7be36177af874631abb15724c893887e260b/diff:/var/lib/docker/overlay2/40d198d6f624e455800254766eb6a7190ce02442fc48f02f6f16f72105cefd0d/diff",
                "MergedDir": "/var/lib/docker/overlay2/0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2/merged",
                "UpperDir": "/var/lib/docker/overlay2/0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2/diff",
                "WorkDir": "/var/lib/docker/overlay2/0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2/work"
            },
            "Name": "overlay2"
        },
...

查看系统挂载表

1
2

$ cat /proc/mounts | grep overlay
overlay /var/lib/docker/overlay2/0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2/merged overlay rw,relatime,lowerdir=/var/lib/docker/overlay2/l/7ZDIR6KYXXF6RMDW3JCBEQUGMH:/var/lib/docker/overlay2/l/TH3OYMS4POUF3S22QNB7UJPORG:/var/lib/docker/overlay2/l/BJILDL5W6H6U7LGVSUUS2QUTGO:/var/lib/docker/overlay2/l/6F6BVIETKMGL5QLJIOCP6CONB3:/var/lib/docker/overlay2/l/P5BANYVEKZJYYPT4M6IFNLAR7Z,upperdir=/var/lib/docker/overlay2/0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2/diff,workdir=/var/lib/docker/overlay2/0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2/work 0 0

可以看到，lowerdir由5个目录共同挂载而成，分别是

7ZDIR6KYXXF6RMDW3JCBEQUGMH
TH3OYMS4POUF3S22QNB7UJPORG
BJILDL5W6H6U7LGVSUUS2QUTGO
6F6BVIETKMGL5QLJIOCP6CONB3
P5BANYVEKZJYYPT4M6IFNLAR7Z

查看overlay2目录下的文件

$ ls /var/lib/docker/overlay2/l/ -l
total 56
lrwxrwxrwx 1 root root 72 May 14 07:09 2L7W765NSNAZAJUW324PTRY6AF -> ../6bd794b03d6772755f61a55ff28f0f20caf1541192c57030b1c0d92e4d3134fa/diff
lrwxrwxrwx 1 root root 72 May 13 07:09 2VHUX6G37XVXLON33KHDZBOVBH -> ../2787c91d4cd57511162a5b17a1ad9cca5204e57b541127223dd11b8c084710bb/diff
lrwxrwxrwx 1 root root 72 Jun 11 05:51 6F6BVIETKMGL5QLJIOCP6CONB3 -> ../37d3e588905fae55c8a0481e9cda7be36177af874631abb15724c893887e260b/diff
lrwxrwxrwx 1 root root 72 May 13 07:24 6P3MYX3ANRVWUUGIBBTJYPGRLP -> ../6d18d5f8aed3820e7500e1f70b3c5d896b90c109977a1097e957667a6b0f48f3/diff
lrwxrwxrwx 1 root root 77 Jun 11 07:53 7ZDIR6KYXXF6RMDW3JCBEQUGMH -> ../0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2-init/diff
lrwxrwxrwx 1 root root 72 Jun 11 05:51 BJILDL5W6H6U7LGVSUUS2QUTGO -> ../823e415d4256d05fb0101af4dcc42a4389d44cf6467972d654e93e0cc575cd9b/diff
lrwxrwxrwx 1 root root 72 May 13 07:24 D26SOVMOFLZLRVVBCVXJVRYAE2 -> ../a74c293f4eb20bef383865bbba97f84a51fd0d894ced280dc1cfe6021be3ae77/diff
lrwxrwxrwx 1 root root 72 Jun 11 07:53 EY46SF3NEYTFL4QOXUZT5YHMA3 -> ../0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2/diff
lrwxrwxrwx 1 root root 72 May 13 07:30 IXP5XYXVUASXI2FOX5UKMYW2JN -> ../c58e315ff14dda2b6ec7f75a3a0a8099dfe269604a0acecf6ecf026c6b56de63/diff
lrwxrwxrwx 1 root root 72 May 14 07:18 N7NRQKX4E62F2JKRQ5JCI3BSSH -> ../d6fed7e45abcf9e0055b9de876a81a5347cdcf364736b0f50053630f8f189e30/diff
lrwxrwxrwx 1 root root 72 May 13 06:16 NJCD6YK72MQFTUSJUMGOEJL4WM -> ../bf1fb537d794b4460c81ae39fc45c3230c22b47e4509a35c282ca15727fe81ac/diff
lrwxrwxrwx 1 root root 72 Jun 11 05:51 P5BANYVEKZJYYPT4M6IFNLAR7Z -> ../40d198d6f624e455800254766eb6a7190ce02442fc48f02f6f16f72105cefd0d/diff
lrwxrwxrwx 1 root root 72 Jun 11 05:51 TH3OYMS4POUF3S22QNB7UJPORG -> ../17bd5da0cda20e8ecd1d4955d25f49609ff0d7aa72fe45a0388a357fcc5b625f/diff
lrwxrwxrwx 1 root root 72 May 13 07:24 YQB24YJNNQSDHU2IDBA2L4LCX4 -> ../c3bee923bb1d5cd56503c976bc8353a6a579698186536f6023524b84373a6834/diff

做个对比

容器ID为a274e38，对应的DIR的ID是0c3ff90
lowerdir=/var/lib/docker/overlay2/l/7ZDIR6KYXXF6RMDW3JCBEQUGMH，实际上指向了0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2-init/diff
1
2
$ ls /var/lib/docker/overlay2/0c3ff90aba26b4197b2293789f75d4e3db7a9213601b8d222b1f0c413c7115b2-init/diff/
dev etc
/var/lib/docker/overlay2/l/TH3OYMS4POUF3S22QNB7UJPORG 实际上指向了17bd5da0cda20e8ecd1d4955d25f49609ff0d7aa72fe45a0388a357fcc5b625f/diff
1
2
$ ls /var/lib/docker/overlay2/17bd5da0cda20e8ecd1d4955d25f49609ff0d7aa72fe45a0388a357fcc5b625f/diff/
run
而17bd5da0cda20e8ecd1d4955d25f49609ff0d7aa72fe45a0388a357fcc5b625f/diff实际上是ubuntu镜像的UpperDir
1
2
$ ls /var/lib/docker/overlay2/17bd5da0cda20e8ecd1d4955d25f49609ff0d7aa72fe45a0388a357fcc5b625f/diff/
run
/var/lib/docker/overlay2/l/BJILDL5W6H6U7LGVSUUS2QUTGO实际上是823e415d4256d05fb0101af4dcc42a4389d44cf6467972d654e93e0cc575cd9b/diff

而823e415d4256d05fb0101af4dcc42a4389d44cf6467972d654e93e0cc575cd9b/diff实际上是ubuntu镜像的LowerDir

1 2	$ ls /var/lib/docker/overlay2/823e415d4256d05fb0101af4dcc42a4389d44cf6467972d654e93e0cc575cd9b/diff/ etc usr var

/var/lib/docker/overlay2/l/6F6BVIETKMGL5QLJIOCP6CONB3实际上是37d3e588905fae55c8a0481e9cda7be36177af874631abb15724c893887e260b/diff
1
2
$ ls /var/lib/docker/overlay2/37d3e588905fae55c8a0481e9cda7be36177af874631abb15724c893887e260b/diff/
var

/var/lib/docker/overlay2/l/P5BANYVEKZJYYPT4M6IFNLAR7Z实际上是40d198d6f624e455800254766eb6a7190ce02442fc48f02f6f16f72105cefd0d/diff

1
2

$ ls /var/lib/docker/overlay2/40d198d6f624e455800254766eb6a7190ce02442fc48f02f6f16f72105cefd0d/diff/
bin  boot  dev  etc  home  lib  lib32  lib64  libx32  media  mnt  opt  proc  root  run  sbin  srv  sys  tmp  usr  var

小结

容器的镜像即rootfs是按照一层一层的组合起来的。
启动容器进程时，将多个增量的rootfs联合挂载成一个完整的rootfs
启动容器时，会只读模式挂载一个init层，以及一个可写的层