Linux 容器运行时之crun分析

随着IT行业的发展，Linux容器一直是比较火的话题，这种轻量级的虚拟机逐渐替代vmware或者hypervisor成为新一代武林盟主。既然是虚拟机，那么它势必满足虚拟机的特点：虚拟机(container)和虚拟机之间相互独立，可以理解为两个不同的设备，一个大概的框图如下：

从上图可见，Linux下的容器是运行在OS之上的虚拟机（OS共享），因此这和传统的vmware和hypervisor并不相同。最初运行这个容器（虚拟机）在Linux下是采用的LXC来运行的，但是后来dotcloud改成了runc（基于go语言），当然github上面也有c语言版的运行时crun：https://github.com/containers/crun。他们二者主要都是基于Linux内核提供的机制cgroup和namespace来实现的。下图为docker的一个模型，

蓝色是Linux下的可执行指令，绿色为守护进程，紫色为虚拟机，下图为google的k8s的模型框图（google引出了CRI接口），

因此，不管从docker或者是k8s（k8s兼容了docker），我们都可以看到一个runc的结点，这个结点就是本章分析的容器运行时，那么什么叫容器运行时（runtime），这是一种和操作系统强相关的运行程序（最初并没有和contaierd分开），也就是主要在对应的操作系统上动态建立一个虚拟机，然后用该虚拟机来运行容器镜像里面的app。因此，我们可以知道，容器镜像里面一般只有app和app依赖的库文件，并不包含操作系统镜像，这也就是容器更轻量级的原因。

一个容器运行时，所需要的环境bundle包必须在bundle里面包含一个满足runtime规范的配置文件config.json和一个目录结构，通常是rootfs，这个config.json可以通过runc spec生成，如下图，

注：运行时的配置文件config.json必须满足运行时规范：https://github.com/opencontainers/runtime-spec; 容器镜像必须满足镜像规范：https://github.com/opencontainers/image-spec，其中runc/crun只需满足运行时规范。

crun的main函数在crun.c中，实现如下：

点击(此处)折叠或打开

int
main (int argc, char **argv)
{
libcrun_error_t err = NULL;
int ret, first_argument;
argp_program_version_hook = print_version;
// C库的参数解析函数，first_argument得到第一个自己解析的索引
argp_parse (&argp, argc, argv, ARGP_IN_ORDER, &first_argument, &arguments);
// 通过名字获取到操作函数，这里是create, ps, start, run , exec等函数操作
command = get_command (argv[first_argument]);
if (command == NULL)
libcrun_fail_with_error (0, "unknown command %s", argv[first_argument]);
// 执行对应的操作函数
ret = command->handler (&arguments, argc - first_argument, argv + first_argument, &err);
if (ret && err)
libcrun_fail_with_error (err->status, "%s", err->msg);
return ret;
}

这里我们只将create函数，实现如下：

点击(此处)折叠或打开

int crun_command_create (struct crun_global_arguments *global_args, int argc, char **argv, libcrun_error_t *err)
{
int first_arg, ret;
libcrun_container_t *container;
cleanup_free char *bundle_cleanup = NULL;
crun_context.preserve_fds = 0;
// 同样的，调用C的解析函数，解析CREATE的子参数
argp_parse (&run_argp, argc, argv, ARGP_IN_ORDER, &first_arg, &crun_context);
crun_assert_n_args (argc - first_arg, 1, 1);
/* 确定bundle的路径. */
if (bundle)
{
if (bundle[0] != '/')
{
bundle_cleanup = realpath (bundle, NULL);
if (bundle_cleanup == NULL)
libcrun_fail_with_error (errno, "realpath `%s` failed", bundle);
bundle = bundle_cleanup;
}
if (chdir (bundle) < 0)
libcrun_fail_with_error (errno, "chdir `%s` failed", bundle);
}
// 初始化上下文环境
ret = init_libcrun_context (&crun_context, argv[first_arg], global_args, err);
if (UNLIKELY (ret < 0))
return ret;
// 解析 config.json数据，并保存到container->def结构当中
container = libcrun_container_load_from_file (config_file, err);
if (container == NULL)
libcrun_fail_with_error (0, "error loading config.json");
crun_context.bundle = bundle ? bundle : ".";
if (getenv ("LISTEN_FDS"))
crun_context.preserve_fds += strtoll (getenv ("LISTEN_FDS"), NULL, 10);
// 通过配置文件和上下文，创建一个容器
return libcrun_container_create (&crun_context, container, 0, err);
}

具体的创建函数，libcrun_container_create函数实现如下：

点击(此处)折叠或打开

int
libcrun_container_create (libcrun_context_t *context, libcrun_container_t *container, unsigned int options, libcrun_error_t *err)
{
runtime_spec_schema_config_schema *def = container->container_def;
int ret;
int container_ready_pipe[2];
cleanup_close int pipefd0 = -1;
cleanup_close int pipefd1 = -1;
cleanup_close int exec_fifo_fd = -1;
context->detach = 1; // 为1表示，crun create 指令不阻塞，直接返回（只有crun run这个detach是0）
container->context = context;
// 检查oci版本
if (def->oci_version && strstr (def->oci_version, "1.0") == NULL)
return crun_make_error (err, 0, "unknown version specified");
// 检查配置文件
ret = check_config_file (def, err);
if (UNLIKELY (ret < 0))
return ret;
if (def->process && def->process->terminal && context->console_socket == NULL)
return crun_make_error (err, 0, "use --console-socket with create when a terminal is used");
// 检查运行时目录，通常为/var/run/crun/*
ret = libcrun_status_check_directories (context->state_root, context->id, err);
if (UNLIKELY (ret < 0))
return ret;
// 创建执行等待fifo，主要是给create/start组合使用，run指令没有这个技能,即start会触发fifo让容器运行
exec_fifo_fd = libcrun_status_create_exec_fifo (context->state_root, context->id, err);
if (UNLIKELY (exec_fifo_fd < 0))
return exec_fifo_fd;
context->fifo_exec_wait_fd = exec_fifo_fd;
exec_fifo_fd = -1;
if ((options & LIBCRUN_RUN_OPTIONS_PREFORK) == 0)
{
// 将config.json从bundle拷贝到工作目录
ret = libcrun_copy_config_file (context->id, context->state_root, context->bundle, err);
if (UNLIKELY (ret < 0))
return ret;
// 创建容器环境，并运行容器
ret = libcrun_container_run_internal (container, context, -1, err);
if (UNLIKELY (ret < 0))
force_delete_container_status (context, def);
return ret;
}
....此处省略，这里不分析的代码....
}

libcrun_container_run_internal实现如下：

点击(此处)折叠或打开

static int
libcrun_container_run_internal (libcrun_container_t *container, libcrun_context_t *context, int container_ready_fd, libcrun_error_t *err)
{
runtime_spec_schema_config_schema *def = container->container_def;
int ret;
pid_t pid;
int detach = context->detach;
cleanup_free char *cgroup_path = NULL;
cleanup_free char *scope = NULL;
cleanup_close int terminal_fd = -1;
cleanup_terminal void *orig_terminal = NULL;
cleanup_close int sync_socket = -1;
cleanup_close int notify_socket = -1;
cleanup_close int socket_pair_0 = -1;
cleanup_close int socket_pair_1 = -1;
cleanup_close int seccomp_fd = -1;
cleanup_close int console_socket_fd = -1;
cleanup_close int hooks_out_fd = -1;
cleanup_close int hooks_err_fd = -1;
int cgroup_mode, cgroup_manager;
char created[35];
uid_t root_uid = -1;
gid_t root_gid = -1;
struct container_entrypoint_s container_args =
{
.container = container,
.context = context,
.terminal_socketpair = {-1, -1},
.console_socket_fd = -1,
.hooks_out_fd = -1,
.hooks_err_fd = -1,
};
if (def->hooks && (def->hooks->prestart_len
|| def->hooks->poststart_len
|| def->hooks->create_runtime_len
|| def->hooks->create_container_len
|| def->hooks->start_container_len))
{
// 打开hook的输入输出fd
ret = open_hooks_output (container, &hooks_out_fd, &hooks_err_fd, err);
if (UNLIKELY (ret < 0))
return ret;
container_args.hooks_out_fd = hooks_out_fd;
container_args.hooks_err_fd = hooks_err_fd;
}
container->context = context;
if (!detach || context->notify_socket)
{
ret = prctl (PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0);
if (UNLIKELY (ret < 0))
return crun_make_error (err, errno, "set child subreaper");
}
if (!context->no_new_keyring)
{ // 创建keyring
ret = libcrun_create_keyring (container->context->id, err);
if (UNLIKELY (ret < 0))
return ret;
}
if (def->process && def->process->terminal && !detach && context->console_socket == NULL)
{
container_args.has_terminal_socket_pair = 1;
ret = create_socket_pair (container_args.terminal_socketpair, err);
if (UNLIKELY (ret < 0))
return crun_error_wrap (err, "create terminal socket");
socket_pair_0 = container_args.terminal_socketpair[0];
socket_pair_1 = container_args.terminal_socketpair[1];
}
// 设置信号为block
ret = block_signals (err);
if (UNLIKELY (ret < 0))
return ret;
if (def->linux && def->linux->seccomp)
{ // 创建seccomp.bpf, 用于系统调用安全检查
ret = open_seccomp_output (context->id, &seccomp_fd, false, context->state_root, err);
if (UNLIKELY (ret < 0))
return ret;
}
container_args.seccomp_fd = seccomp_fd;
if (context->console_socket)
{// 如果指定了本地socket，则用本地socket作为容器的标准输入输出
console_socket_fd = open_unix_domain_client_socket (context->console_socket, 0, err);
if (UNLIKELY (console_socket_fd < 0))
return crun_error_wrap (err, "open console socket");
container_args.console_socket_fd = console_socket_fd;
}
// 获取cgroup版本模式等
cgroup_mode = libcrun_get_cgroup_mode (err);
if (cgroup_mode < 0)
return cgroup_mode;
// 运行一个linux容器，容器运行后，调用container_init函数初始化
pid = libcrun_run_linux_container (container, container_init, &container_args,
&sync_socket, err);
if (UNLIKELY (pid < 0))
return pid;
if (context->fifo_exec_wait_fd < 0 && context->notify_socket)
{
/* Do not open the notify socket here on "create". "start" will take care of it. */
ret = get_notify_fd (context, container, ?ify_socket, err);
if (UNLIKELY (ret < 0))
return ret;
}
if (container_args.terminal_socketpair[1] >= 0)
close_and_reset (&socket_pair_1);
cgroup_manager = CGROUP_MANAGER_CGROUPFS;
if (context->systemd_cgroup)
cgroup_manager = CGROUP_MANAGER_SYSTEMD;
else if (context->force_no_cgroup)
cgroup_manager = CGROUP_MANAGER_DISABLED;
get_root_in_the_userns_for_cgroups (def, container->host_uid, container->host_gid, &root_uid, &root_gid);
{
struct libcrun_cgroup_args cg =
{
.resources = def->linux ? def->linux->resources : NULL,
.annotations = def->annotations,
.cgroup_mode = cgroup_mode,
.path = &cgroup_path,
.scope = &scope,
.cgroup_path = def->linux ? def->linux->cgroups_path : "",
.manager = cgroup_manager,
.pid = pid,
.root_uid = root_uid,
.root_gid = root_gid,
.id = context->id,
.systemd_subgroup = find_systemd_subgroup (container, cgroup_mode),
};
// 打开cgroup
ret = libcrun_cgroup_enter (&cg, err);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
// 配置cgroup
if (def->linux && def->linux->resources)
{
ret = libcrun_update_cgroup_resources (cgroup_mode,
def->linux->resources,
cgroup_path, err);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
}
}
/* sync 1. */ 通知容器可以进行1初始化
ret = sync_socket_send_sync (sync_socket, true, err);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
/* sync 2. */等待容器通知，可以进行2初始化
ret = sync_socket_wait_sync (context, sync_socket, false, err);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
// 执行hook函数
if (def->hooks && def->hooks->prestart_len)
{
ret = do_hooks (def, pid, context->id, false, NULL, "created",
(hook **) def->hooks->prestart,
def->hooks->prestart_len, hooks_out_fd, hooks_err_fd, err);
if (UNLIKELY (ret != 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
}
if (def->hooks && def->hooks->create_runtime_len)
{
ret = do_hooks (def, pid, context->id, false, NULL, "created",
(hook **) def->hooks->create_runtime,
def->hooks->create_runtime_len, hooks_out_fd, hooks_err_fd, err);
if (UNLIKELY (ret != 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
}
// 如果支持系统调用安全检查，这里就产生一个bpf文件
if (seccomp_fd >= 0)
{
unsigned int seccomp_gen_options = 0;
const char *annotation;
annotation = find_annotation (container, "run.oci.seccomp_fail_unknown_syscall");
if (annotation && strcmp (annotation, "0") != 0)
seccomp_gen_options = LIBCRUN_SECCOMP_FAIL_UNKNOWN_SYSCALL;
ret = libcrun_generate_seccomp (container, seccomp_fd, seccomp_gen_options, err);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
close_and_reset (&seccomp_fd);
}
/* sync 3. */ 通知容器进程，可以进行3初始化
ret = sync_socket_send_sync (sync_socket, true, err);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
if (def->process && def->process->terminal && !detach && context->console_socket == NULL)
{ // 如果没有指定本地socket作为容器的终端，这里创建一个
terminal_fd = receive_fd_from_socket (socket_pair_0, err);
if (UNLIKELY (terminal_fd < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
close_and_reset (&socket_pair_0);
// 设置socket的teminal属性
ret = libcrun_setup_terminal_master (terminal_fd, &orig_terminal, err);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
}
/* sync 4. */ 等待容器通知，可以进行4初始化
ret = sync_socket_wait_sync (context, sync_socket, false, err);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
ret = close_and_reset (&sync_socket);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
get_current_timestamp (created);
// 写容器状态，pid是容器的进程id
ret = write_container_status (container, context, pid, cgroup_path, scope, created, err);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
// 执行running hook函数
if (context->fifo_exec_wait_fd < 0 && def->hooks && def->hooks->poststart_len)
{
ret = do_hooks (def, pid, context->id, true, NULL, "running",
(hook **) def->hooks->poststart,
def->hooks->poststart_len, hooks_out_fd, hooks_err_fd, err);
if (UNLIKELY (ret < 0))
return cleanup_watch (context, pid, sync_socket, terminal_fd, err);
}
// 如果detach是true则return;这个wait主要给crun run使用, 对于create就是一个空函数
ret = wait_for_process (pid, context, terminal_fd, notify_socket, container_ready_fd, err);
if (!context->detach)
{
cleanup_watch (context, 0, sync_socket, terminal_fd, err);
crun_error_release (err);
}
return ret;
}

创建一个容器，并运行libcrun_run_linux_container实现如下：

点击(此处)折叠或打开

pid_t
libcrun_run_linux_container (libcrun_container_t *container,
container_entrypoint_t entrypoint,
void *args,
int *sync_socket_out,
libcrun_error_t *err)
{
__attribute__((cleanup (cleanup_free_init_statusp))) struct init_status_s init_status;
runtime_spec_schema_config_schema *def = container->container_def;
cleanup_close int sync_socket_container = -1;
char *notify_socket_env = NULL;
cleanup_close int sync_socket_host = -1;
bool clone_can_create_userns;
int sync_socket[2];
pid_t pid;
size_t i;
int ret;
// 初始化命名空间,打开命名空间fd
ret = configure_init_status (&init_status, container, err);
if (UNLIKELY (ret < 0))
return ret;
get_private_data (container)->unshare_flags = init_status.all_namespaces;
#ifdef CLONE_NEWCGROUP
/* cgroup will be unshared later. Once the process is in the correct cgroup. */
init_status.all_namespaces &= ~CLONE_NEWCGROUP;
#endif
// 创建容器和主进程通信的socket
ret = socketpair (AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, sync_socket);
if (UNLIKELY (ret < 0))
return crun_make_error (err, errno, "socketpair");
sync_socket_host = sync_socket[0];
sync_socket_container = sync_socket[1];
get_uid_gid_from_def (container->container_def,
&container->container_uid,
&container->container_gid);
/* 设置rlimis值 */
if (def->process)
{
ret = libcrun_set_rlimits (def->process->rlimits, def->process->rlimits_len, err);
if (UNLIKELY (ret < 0))
return ret;
}
// 设置oom属性
ret = libcrun_set_oom (container, err);
if (UNLIKELY (ret < 0))
return ret;
// 配置命名空间相关，是否则在新的容器进程支持对应的命名空间
if ((init_status.all_namespaces & CLONE_NEWIPC) && (init_status.all_namespaces & CLONE_NEWUSER))
{
for (i = 0; i < init_status.fd_len; i++)
if (init_status.value[i] == CLONE_NEWIPC)
init_status.join_ipcns = true;
}
if (init_status.all_namespaces & CLONE_NEWPID)
{
init_status.must_fork = true;
for (i = 0; i < init_status.fd_len; i++)
{
if (init_status.value[i] == CLONE_NEWPID)
{
init_status.join_pidns = true;
if (setns (init_status.fd[i], CLONE_NEWPID) == 0)
{
init_status.namespaces_to_unshare &= ~CLONE_NEWPID;
init_status.must_fork = false;
close_and_reset (&init_status.fd[i]);
}
break;
}
}
/* It creates a new PID namespace, without a user namespace, we can try to
join it immediately without another fork. */
if (i == init_status.fd_len && (init_status.all_namespaces & CLONE_NEWUSER) == 0)
{
if (unshare (CLONE_NEWPID) == 0)
{
init_status.namespaces_to_unshare &= ~CLONE_NEWPID;
init_status.must_fork = false;
}
}
}
#ifdef CLONE_NEWTIME
if (init_status.all_namespaces & CLONE_NEWTIME)
init_status.must_fork = true;
#endif
clone_can_create_userns = init_status.fd_len == 0;
if ((init_status.all_namespaces & CLONE_NEWUSER) && init_status.userns_index < 0)
init_status.delayed_userns_create = !clone_can_create_userns || init_status.fd_len > 0;
// 创建一个容器进程
pid = syscall_clone ((init_status.namespaces_to_unshare & (clone_can_create_userns ? CLONE_NEWUSER : 0)) | SIGCHLD, NULL);
if (UNLIKELY (pid < 0))
return crun_make_error (err, errno, "clone");
if (clone_can_create_userns)
init_status.namespaces_to_unshare &= ~CLONE_NEWUSER;
if (pid)
{ // 主进程保存容器额外的信息
ret = save_external_descriptors (container, pid, err);
if (UNLIKELY (ret < 0))
return ret;
// 关闭主进程不需要的socket，即容器用的socket
ret = close_and_reset (&sync_socket_container);
if (UNLIKELY (ret < 0))
return crun_make_error (err, errno, "close");
// 如果是clone_newuser，需要等待容器ready
if (init_status.all_namespaces & CLONE_NEWUSER)
{
if (init_status.delayed_userns_create)
{
ret = expect_success_from_sync_socket (sync_socket_host, err);
if (UNLIKELY (ret < 0))
return ret;
}
if (init_status.userns_index < 0)
{
ret = libcrun_set_usernamespace (container, pid, err);
if (UNLIKELY (ret < 0))
return ret;
ret = TEMP_FAILURE_RETRY (write (sync_socket_host, "1", 1));
if (UNLIKELY (ret < 0))
return crun_make_error (err, errno, "write to sync socket");
}
}
// 如果容器还要进行fork，则需要等待容器fork出来的进程id
if (init_status.must_fork)
{
pid_t grandchild = 0;
ret = expect_success_from_sync_socket (sync_socket_host, err);
if (UNLIKELY (ret < 0))
return ret;
ret = TEMP_FAILURE_RETRY (read (sync_socket_host, &grandchild, sizeof (grandchild)));
if (UNLIKELY (ret < 0))
return crun_make_error (err, errno, "read pid from sync socket");
/* Cleanup the first process. */
waitpid (pid, NULL, 0);
pid = grandchild;
}
ret = expect_success_from_sync_socket (sync_socket_host, err);
if (UNLIKELY (ret < 0))
return ret;
*sync_socket_out = get_and_reset (&sync_socket_host);
// 返回最终容器的进程id
return pid;
}
/* Inside the container process. */
// 容器进程关闭主进程的不用的通信套接字
ret = close_and_reset (&sync_socket_host);
if (UNLIKELY (ret < 0))
return crun_make_error (err, errno, "close");
// 初始化一个命名空间相关的（添加命名空间）
ret = init_container (container, sync_socket_container, &init_status, err);
if (UNLIKELY (ret < 0))
{
char failure = 1;
ret = TEMP_FAILURE_RETRY (write (sync_socket_container, &failure, 1));
if (UNLIKELY (ret < 0))
goto localfail;
send_error_to_sync_socket_and_die (sync_socket_container, false, err);
localfail:
libcrun_fail_with_error ((*err)->status, "%s", (*err)->msg);
_exit (EXIT_FAILURE);
}
else
{
char success = 0;
ret = TEMP_FAILURE_RETRY (write (sync_socket_container, &success, 1));
if (UNLIKELY (ret < 0))
return ret;
}
/* Jump into the specified entrypoint. */
if (container->context->notify_socket)
xasprintf (&notify_socket_env, "NOTIFY_SOCKET=%s/notify", container->context->notify_socket);
// 执行容器的真正初始化，即container_init
entrypoint (args, notify_socket_env, sync_socket_container, err);
/* ENTRYPOINT returns only on an error, fallback here: */
if (*err)
libcrun_fail_with_error ((*err)->status, "%s", (*err)->msg);
_exit (EXIT_FAILURE);
}

容器进程初始化，container_init实现如下：

点击(此处)折叠或打开

static int container_init (void *args, char *notify_socket, int sync_socket,
libcrun_error_t *err)
{
struct container_entrypoint_s *entrypoint_args = args;
int ret;
runtime_spec_schema_config_schema *def = entrypoint_args->container->container_def;
cleanup_free const char *exec_path = NULL;
cleanup_free char *notify_socket_cleanup = notify_socket;
entrypoint_args->sync_socket = sync_socket;
//改变日志输出fd到syncsocket
crun_set_output_handler (log_write_to_sync_socket, args, false);
// 初始化容器，一些组件的设置
ret = container_init_setup (args, notify_socket, sync_socket, &exec_path, err);
if (UNLIKELY (ret < 0))
{
/* If it fails to write the error using the sync socket, then fallback
to stderr. */
if (sync_socket_write_error (sync_socket, err) < 0)
return ret;
crun_error_release (err);
return ret;
}
entrypoint_args->sync_socket = -1;
//解除信号阻塞
ret = unblock_signals (err);
if (UNLIKELY (ret < 0))
return ret;
/* sync 4. */ 通知主进程，可以进行第4步初始化
ret = sync_socket_send_sync (sync_socket, false, err);
if (UNLIKELY (ret < 0))
return ret;
close_and_reset (&sync_socket);
// 如果exec_wait_fd>=0表示这是create调用，则会在这里阻塞，知道命令行调用crun start来解除
if (entrypoint_args->context->fifo_exec_wait_fd >= 0)
{
char buffer[1];
fd_set read_set;
cleanup_close int fd = entrypoint_args->context->fifo_exec_wait_fd;
entrypoint_args->context->fifo_exec_wait_fd = -1;
FD_ZERO (&read_set);
FD_SET (fd, &read_set);
do
{
ret = select (fd + 1, &read_set, NULL, NULL, NULL);
if (UNLIKELY (ret < 0))
return crun_make_error (err, errno, "select");
ret = TEMP_FAILURE_RETRY (read (fd, buffer, sizeof (buffer)));
if (UNLIKELY (ret < 0))
return crun_make_error (err, errno, "read from the exec fifo");
}
while (ret == 0);
close_and_reset (&entrypoint_args->context->fifo_exec_wait_fd);
}
// 设置打印输出到标准错误
crun_set_output_handler (log_write_to_stderr, NULL, false);
if (def->process && def->process->no_new_privileges)
{
char **seccomp_flags = NULL;
size_t seccomp_flags_len = 0;
if (def->linux && def->linux->seccomp)
{
seccomp_flags = def->linux->seccomp->flags;
seccomp_flags_len = def->linux->seccomp->flags_len;
}
// 设置系统调用检查配置
ret = libcrun_apply_seccomp (entrypoint_args->seccomp_fd, seccomp_flags, seccomp_flags_len, err);
if (UNLIKELY (ret < 0))
return ret;
close_and_reset (&entrypoint_args->seccomp_fd);
}
if (UNLIKELY (def->process == NULL))
return crun_make_error (err, 0, "block 'process' not found");
if (UNLIKELY (exec_path == NULL))
return crun_make_error (err, 0, "executable path not specified");
// 运行starting hook函数
if (def->hooks && def->hooks->start_container_len)
{
libcrun_container_t *container = entrypoint_args->container;
ret = do_hooks (def, 0, container->context->id, false, NULL, "starting",
(hook **) def->hooks->start_container,
def->hooks->start_container_len,
entrypoint_args->hooks_out_fd,
entrypoint_args->hooks_err_fd,
err);
if (UNLIKELY (ret != 0))
return ret;
/* Seek stdout/stderr to the end. If the hooks were using the same files,
the container process overwrites what was previously written. */
(void) lseek (1, 0, SEEK_END);
(void) lseek (2, 0, SEEK_END);
}
// 执行容器里面的app，这里开始，用户程序开始运行
execv (exec_path, def->process->args);
if (errno == ENOENT)
return crun_make_error (err, errno, "exec container process (missing dynamic library?) `%s`", exec_path);
return crun_make_error (err, errno, "exec container process `%s`", exec_path);
}

container_init_setup函数实现如下：

点击(此处)折叠或打开

static int
container_init_setup (void *args, char *notify_socket,
int sync_socket, const char **exec_path,
libcrun_error_t *err)
{
struct container_entrypoint_s *entrypoint_args = args;
libcrun_container_t *container = entrypoint_args->container;
int ret;
int has_terminal;
cleanup_close int console_socket = -1;
cleanup_close int console_socketpair = -1;
runtime_spec_schema_config_schema *def = container->container_def;
runtime_spec_schema_config_schema_process_capabilities *capabilities;
cleanup_free char *rootfs = NULL;
int no_new_privs;
// 初始化selinux和apparmor 安全相关的
ret = initialize_security (def->process, err);
if (UNLIKELY (ret < 0))
return ret;
// UP配置回环网络
ret = libcrun_configure_network (container, err);
if (UNLIKELY (ret < 0))
return ret;
// 得到roofs的实际路径
rootfs = realpath (def->root->path, NULL);
if (UNLIKELY (rootfs == NULL))
{
/* If realpath failed for any reason, try the relative directory. */
rootfs = xstrdup (def->root->path);
}
// 得到终端输入输出fd
if (entrypoint_args->terminal_socketpair[0] >= 0)
{
close_and_reset (&entrypoint_args->terminal_socketpair[0]);
console_socketpair = entrypoint_args->terminal_socketpair[1];
}
/* sync 1. */ 等待主进程通知可以开始第1步初始化
ret = sync_socket_wait_sync (NULL, sync_socket, false, err);
if (UNLIKELY (ret < 0))
return ret;
has_terminal = container->container_def->process && container->container_def->process->terminal;
if (has_terminal && entrypoint_args->context->console_socket)
console_socket = entrypoint_args->console_socket_fd;
// 配置sysctl参数
ret = libcrun_set_sysctl (container, err);
if (UNLIKELY (ret < 0))
return ret;
// 挂载需要的文件系统（包括新的rootfs）
ret = libcrun_set_mounts (container, rootfs, err);
if (UNLIKELY (ret < 0))
return ret;
/* sync 2. */ 通知主进程，可以开始2阶段初始化
ret = sync_socket_send_sync (sync_socket, false, err);
if (UNLIKELY (ret < 0))
return ret;
/* sync 3. */ 等待主进程通知，可以进行第3阶段初始化
ret = sync_socket_wait_sync (NULL, sync_socket, false, err);
if (UNLIKELY (ret < 0))
return ret;
// 执行created hook
if (def->hooks && def->hooks->create_container_len)
{
ret = do_hooks (def, 0, container->context->id, false, NULL, "created",
(hook **) def->hooks->create_container,
def->hooks->create_container_len,
entrypoint_args->hooks_out_fd,
entrypoint_args->hooks_err_fd,
err);
if (UNLIKELY (ret != 0))
return ret;
}
// 设置selinux相关的label
if (def->process)
{
ret = libcrun_set_selinux_exec_label (def->process, err);
if (UNLIKELY (ret < 0))
return ret;
ret = libcrun_set_apparmor_profile (def->process, err);
if (UNLIKELY (ret < 0))
return ret;
}
// 关闭多余的fd套件字
ret = close_fds_ge_than (entrypoint_args->context->preserve_fds + 3, err);
if (UNLIKELY (ret < 0))
crun_error_write_warning_and_release (entrypoint_args->context->output_handler_arg, &err);
// 切换到新的rootfs去工作
ret = libcrun_do_pivot_root (container, entrypoint_args->context->no_pivot, rootfs, err);
if (UNLIKELY (ret < 0))
return ret;
// 重新使用/dev/null
ret = libcrun_reopen_dev_null (err);
if (UNLIKELY (ret < 0))
return ret;
if (clearenv ())
return crun_make_error (err, errno, "clearenv");
// 上面清除了环境变量，这里重新设置新的环境变量
if (def->process)
{
size_t i;
for (i = 0; i < def->process->env_len; i++)
if (putenv (def->process->env[i]) < 0)
return crun_make_error (err, errno, "putenv `%s`", def->process->env[i]);
}
if (getenv ("HOME") == NULL)
{
ret = set_home_env (container->container_uid);
if (UNLIKELY (ret < 0 && errno != ENOTSUP))
{
setenv("HOME", "/", 1);
libcrun_warning ("cannot detect HOME environment variable, setting default");
}
}
if (def->process && def->process->cwd)
if (UNLIKELY (chdir (def->process->cwd) < 0))
return crun_make_error (err, errno, "chdir");
// 查找到用户程序路径
if (def->process && def->process->args)
{
*exec_path = find_executable (def->process->args[0], def->process->cwd);
if (UNLIKELY (*exec_path == NULL))
{
if (errno == ENOENT)
return crun_make_error (err, errno, "executable file not found in $PATH");
return crun_make_error (err, errno, "open executable");
}
}
ret = setsid ();
if (UNLIKELY (ret < 0))
return crun_make_error (err, errno, "setsid");
// 如果有终端，将终端fd通知主进程
if (has_terminal)
{
cleanup_close int terminal_fd = -1;
fflush (stderr);
terminal_fd = libcrun_set_terminal (container, err);
if (UNLIKELY (terminal_fd < 0))
return terminal_fd;
if (console_socket >= 0)
{
ret = send_fd_to_socket (console_socket, terminal_fd, err);
if (UNLIKELY (ret < 0))
return ret;
close_and_reset (&console_socket);
}
else if (entrypoint_args->has_terminal_socket_pair && console_socketpair >= 0)
{
ret = send_fd_to_socket (console_socketpair, terminal_fd, err);
if (UNLIKELY (ret < 0))
return ret;
close_and_reset (&console_socketpair);
}
}
// 设置容器主机名字
ret = libcrun_set_hostname (container, err);
if (UNLIKELY (ret < 0))
return ret;
if (container->container_def->linux && container->container_def->linux->personality)
{
ret = libcrun_set_personality (container->container_def->linux->personality, err);
if (UNLIKELY (ret < 0))
return ret;
}
if (def->process->user)
umask (def->process->user->umask_present ? def->process->user->umask : 0022);
// 设置准备好的读取seccomp.bpf规则到系统调用安全检查
if (def->process && !def->process->no_new_privileges)
{
char **seccomp_flags = NULL;
size_t seccomp_flags_len = 0;
if (def->linux && def->linux->seccomp)
{
seccomp_flags = def->linux->seccomp->flags;
seccomp_flags_len = def->linux->seccomp->flags_len;
}
ret = libcrun_apply_seccomp (entrypoint_args->seccomp_fd, seccomp_flags, seccomp_flags_len, err);
if (UNLIKELY (ret < 0))
return ret;
close_and_reset (&entrypoint_args->seccomp_fd);
}
capabilities = def->process ? def->process->capabilities : NULL;
no_new_privs = def->process ? def->process->no_new_privileges : 1;
ret = libcrun_set_caps (capabilities, container->container_uid, container->container_gid, no_new_privs, err);
if (UNLIKELY (ret < 0))
return ret;
if (notify_socket)
{
if (putenv (notify_socket) < 0)
return crun_make_error (err, errno, "putenv `%s`", notify_socket);
}
return 0;
}

wait_for_process等待函数实现如下：

点击(此处)折叠或打开

static int wait_for_process (pid_t pid, libcrun_context_t *context, int terminal_fd, int notify_socket, int container_ready_fd, libcrun_error_t *err)
{
cleanup_close int epollfd = -1;
cleanup_close int signalfd = -1;
int ret, container_exit_code = 0, last_process;
sigset_t mask;
int fds[10];
int levelfds[10];
int levelfds_len = 0;
int fds_len = 0;
container_exit_code = 0;
if (context->pid_file)
{
char buf[12];
size_t buf_len = sprintf (buf, "%d", pid);
ret = write_file (context->pid_file, buf, buf_len, err);
if (UNLIKELY (ret < 0))
return ret;
}
/* crun create 将在这里返回，而crun run将会继续运行. */
if (context->detach && notify_socket < 0)
return 0;
if (container_ready_fd >= 0)
{
ret = 0;
TEMP_FAILURE_RETRY (write (container_ready_fd, &ret, sizeof (ret)));
close_and_reset (&container_ready_fd);
}
sigfillset (&mask);
ret = sigprocmask (SIG_BLOCK, &mask, NULL);
if (UNLIKELY (ret < 0))
return crun_make_error (err, errno, "sigprocmask");
// 将所有发送给当前主进程的信号转换为fd。
signalfd = create_signalfd (&mask, err);
if (UNLIKELY (signalfd < 0))
return signalfd;
// 等待一个子进程退出
ret = reap_subprocesses (pid, &container_exit_code, &last_process, err);
if (UNLIKELY (ret < 0))
return ret;
if (last_process)
return container_exit_code;
// 将终端和systemd的fd加入到select
fds[fds_len++] = signalfd;
if (notify_socket >= 0)
fds[fds_len++] = notify_socket;
if (terminal_fd >= 0)
{
fds[fds_len++] = 0;
levelfds[levelfds_len++] = terminal_fd;
}
fds[fds_len++] = -1;
levelfds[levelfds_len++] = -1;
// 创建epoll
epollfd = epoll_helper (fds, levelfds, err);
if (UNLIKELY (epollfd < 0))
return epollfd;
while (1)
{
struct signalfd_siginfo si;
ssize_t res;
struct epoll_event events[10];
int i, nr_events;
nr_events = TEMP_FAILURE_RETRY (epoll_wait (epollfd, events, 10, -1));
if (UNLIKELY (nr_events < 0))
return crun_make_error (err, errno, "epoll_wait");
for (i = 0; i < nr_events; i++)
{ // 这里做了一个转发，这样crun run可以实时显示容器的输入输出了
if (events[i].data.fd == 0)
{
ret = copy_from_fd_to_fd (0, terminal_fd, 0, err);
if (UNLIKELY (ret < 0))
return crun_error_wrap (err, "copy to terminal fd");
}
else if (events[i].data.fd == terminal_fd)
{
ret = set_blocking_fd (terminal_fd, 0, err);
if (UNLIKELY (ret < 0))
return crun_error_wrap (err, "set terminal fd not blocking");
ret = copy_from_fd_to_fd (terminal_fd, 1, 1, err);
if (UNLIKELY (ret < 0))
return crun_error_wrap (err, "copy from terminal fd");
ret = set_blocking_fd (terminal_fd, 1, err);
if (UNLIKELY (ret < 0))
return crun_error_wrap (err, "set terminal fd blocking");
}// 如果是给systemd的
else if (events[i].data.fd == notify_socket)
{
ret = handle_notify_socket (notify_socket, err);
if (UNLIKELY (ret < 0))
return ret;
if (ret && context->detach)
return 0;
} // 接收的的信号
else if (events[i].data.fd == signalfd)
{
res = TEMP_FAILURE_RETRY (read (signalfd, &si, sizeof (si)));
if (UNLIKELY (res < 0))
return crun_make_error (err, errno, "read from signalfd");
if (si.ssi_signo == SIGCHLD)
{// 表示容器退出，crun run 返回
ret = reap_subprocesses (pid, &container_exit_code, &last_process, err);
if (UNLIKELY (ret < 0))
return ret;
if (last_process)
return container_exit_code;
}
else
{
/* Send any other signal to the child process. */
ret = kill (pid, si.ssi_signo);
}
}
else
{
return crun_make_error (err, 0, "unknown fd from epoll_wait");
}
}
}
return 0;
}

这里贴出C代码编写的容器运行时crun的程序调用栈。

Linux 容器运行时之crun分析-LMLPHP

另外crun, 大概涉及到systemd, seccomp, sysctl, oom, keyring, rlimit, apparmor，selinux, cgroup, namespace(UTS, IPC, PID, NET,MOUNT,USER)等与Linux强相关的组件。

创建容器： crun create container_id (当前目录下必须有config.json和rootfs)
运行容器： crun start container_id
删除容器： crun delete container_id
创建+运行容器： crun run container_id
查看进程id：crun ps container_id

BugMan

Linux 容器运行时之crun分析