1.说明
以前看内核代码觉得难，是因为内核代码涉及到硬件原理与算法结构和层次递进的代码逻辑，现在的应用层因为业务的复杂与代码和内核的交互接口复杂，也变得有些难度了。
这篇文章是继:openbmc dbus架构简析的第二篇文章。
首先贴出来前篇文章的图，与简述内容:
c
* 1.inherit_fds()使用systemd机制获取到socket描述符(请先了解systemd的socket机制原理，会先接手socket服务，accept默认为no,因此需要sd_listen_fds()方法获取socket描述符)
* 2.父子进程通过socketpair(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC | SOCK_NONBLOCK, 0, controller);中的controller[0]和controller[1]进行连接
* 3.sd_bus_set_fd(launcher->bus_controller, controller[0], controller[0]);使用该函数,实际bus->input_fd = input_fd;bus->output_fd = output_fd; 实际即为controller[0]
* 4.sd_bus_call(launcher->bus_controller, m, 0, NULL, NULL);发送socket数据实际依靠的是controller[0],而子进程使用的是controller[1],通道打通，数据可以传递给子进程
* 5.sd_bus_message_append(m, "oh","/org/bus1/DBus/Listener/0",launcher->fd_listen);把launcher->fd_listen 为总体的systemd的监听的socket描述符传递给了子进程。
2.代码分析
2.1 代码使用的描述符
2.1.1 dbus-broker-launch使用的描述符
根据代码，列出来使用到的描述符:
c
static int run(void)
---> launcher_new()
---> r = launcher_open_log(launcher);
---> fd = socket(PF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0); //新建了描述符， fd=4
---> sd_event_default(&launcher->event);
--->...
---> e->epoll_fd = epoll_create1(EPOLL_CLOEXEC); //创建了描述， epoll_fd=5.
---> r = sd_event_add_signal(launcher->event, NULL, SIGTERM, NULL, NULL);
---> r = sd_event_add_signal(launcher->event, NULL, SIGTERM, NULL, NULL);
---> r = sd_event_add_signal(launcher->event, NULL, SIGTERM, NULL, NULL);
---> r = event_make_signal_data(e, sig, &d);
---> r = signalfd(d->fd >= 0 ? d->fd : -1,...) 使用了signalfd创建的fd=6
---> r = launcher_run(launcher);
---> r = launcher_parse_config(launcher, &root, &nss_cache);
---> r = dirwatch_new(&dirwatch);
---> dw->inotify_fd = inotify_init1(IN_CLOEXEC | IN_NONBLOCK); //新建fd=7
---> r = config_parser_read(&parser, rootp, configfile, nss_cache, dirwatch);
---> r = config_parser_include(parser, root, node, nss_cache, dirwatch);
---> _c_cleanup_(c_closep) int fd = -1;
---> r = open(node->include.file->path, O_RDONLY | O_CLOEXEC | O_NOCTTY);
---> ... //这里 fd=8, 实际需要调用close() ？ 使用_c_cleanup_语法实际关闭了
---> r = socketpair(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC | SOCK_NONBLOCK, 0, controller);
---> ...返回了2个套接字 分别为 controller[0] = 8 和 controller[0] = 9如下图，可以看到基本的文件描述符均为默认状态:
参考文档: https://www.jinbuguo.com/systemd/systemd.exec.html#， 描述了如下内容:
因此，fd=0,fd=1,fd=2的结果如下:
因此，对于openbmc，常见的printf,fprintf(stderr...)均会输出到journal中。
- 对于
fd=3继承于systemd，接收外部的服务。 - 对于
fd=4创建socket，用来将数据传递到:/run/systemd/journal/socket - 对于
fd=5, 创建e->epoll_fd - 对于
fd=6,使用signalfd()创建fd - 对于
fd=7,使用inotify_init1()创建fd - 对于
fd=8,fd=9，使用socketpair()创建fd，用来父子进程数据通信
再回到开头的那副图，可以看看子进程与父进程之间，父进程通过sd-bus接口调用将fd(systemd监控转储给dbus-broker的socket)描述符传递给子进程。
继续分析文件:src\launch\launcher.c:
c
static int launcher_add_listener(Launcher *launcher, Policy *policy,...)
---> r = sd_bus_message_new_method_call(launcher->bus_controller, &m,NULL,"/org/bus1/DBus/Broker","org.bus1.DBus.Broker","AddListener");
---> _cleanup_(sd_bus_message_unrefp) sd_bus_message *t = NULL;
---> r = sd_bus_message_new(bus, &t, SD_BUS_MESSAGE_METHOD_CALL);
---> t->header->endian = BUS_NATIVE_ENDIAN;
---> t->header->type = type;
---> t->header->version = bus->message_version;
---> t->allow_fds = bus->can_fds || !IN_SET(bus->state, BUS_HELLO, BUS_RUNNING);
---> r = sd_bus_message_append(m, "oh","/org/bus1/DBus/Listener/0",launcher->fd_listen);
---> r = policy_export(policy, m, system_console_users, n_system_console_users);
---> r = sd_bus_call(launcher->bus_controller, m, 0, NULL, NULL);实际上，这里面的重点是函数:sd_bus_call(),在文件:src\libsystemd\sd-bus\sd-bus.c中:
c
int sd_bus_call(...)
---> bus_write_message()
---> bus_socket_write_message()
---> if (m->n_fds > 0 && *idx == 0)
---> mh.msg_controllen = CMSG_SPACE(sizeof(int) * m->n_fds);
---> mh.msg_control = alloca0(mh.msg_controllen);
---> control = CMSG_FIRSTHDR(&mh);
---> control->cmsg_len = CMSG_LEN(sizeof(int) * m->n_fds);
---> control->cmsg_level = SOL_SOCKET;
---> control->cmsg_type = SCM_RIGHTS;
---> memcpy(CMSG_DATA(control), m->fds, sizeof(int) * m->n_fds);
---> k = sendmsg(bus->output_fd, &mh, MSG_DONTWAIT|MSG_NOSIGNAL);使用如上调用步骤传递文件描述符给另外一个进程，使用的方法是父子进程之间通过sendmsg()调用。
继续查看函数sd_bus_message_append()的调用关系:
c
int sd_bus_message_append(sd_bus_message *m, const char *types, ...)
---> r = sd_bus_message_appendv(m, types, ap); //type是字符串"oh"
---> switch (*t)
---> case SD_BUS_TYPE_UNIX_FD:
---> uint32_t x;
---> x = va_arg(ap, uint32_t);
---> r = sd_bus_message_append_basic(m, *t, &x);
---> u32 = m->n_fds;
---> case SD_BUS_TYPE_OBJECT_PATH:
---> const char *x;
---> x = va_arg(ap, const char*);
---> r = sd_bus_message_append_basic(m, *t, x);这些sd-bus类型有必要列出来，在文件:src\systemd\sd-bus-protocol.h中:
c
enum {
_SD_BUS_TYPE_INVALID = 0,
SD_BUS_TYPE_BYTE = 'y',
SD_BUS_TYPE_BOOLEAN = 'b',
SD_BUS_TYPE_INT16 = 'n',
SD_BUS_TYPE_UINT16 = 'q',
SD_BUS_TYPE_INT32 = 'i',
SD_BUS_TYPE_UINT32 = 'u',
SD_BUS_TYPE_INT64 = 'x',
SD_BUS_TYPE_UINT64 = 't',
SD_BUS_TYPE_DOUBLE = 'd',
SD_BUS_TYPE_STRING = 's',
SD_BUS_TYPE_OBJECT_PATH = 'o',
SD_BUS_TYPE_SIGNATURE = 'g',
SD_BUS_TYPE_UNIX_FD = 'h',
SD_BUS_TYPE_ARRAY = 'a',
SD_BUS_TYPE_VARIANT = 'v',
SD_BUS_TYPE_STRUCT = 'r', /* not actually used in signatures */
SD_BUS_TYPE_STRUCT_BEGIN = '(',
SD_BUS_TYPE_STRUCT_END = ')',
SD_BUS_TYPE_DICT_ENTRY = 'e', /* not actually used in signatures */
SD_BUS_TYPE_DICT_ENTRY_BEGIN = '{',
SD_BUS_TYPE_DICT_ENTRY_END = '}'
};在文件:src\broker\controller.c中，函数:controller_init()调用:
c
int controller_init(Controller *c, Broker *broker, int controller_fd)
---> r = connection_init_server(&controller->connection,
&broker->dispatcher,controller_dispatch_connection,
broker->bus.user,"0123456789abcdef",controller_fd);其中，函数controller_dispatch_connection()定义:
c
static int controller_dispatch_connection(DispatchFile *file)
---> r = connection_dispatch(&controller->connection, dispatch_file_events(file));
---> r = connection_dequeue(&controller->connection, &m);
---> r = socket_dequeue(&connection->socket, &message);
---> r = message_new_incoming(&message, socket->in.header);
---> r = message_parse_metadata(m);
---> r = message_parse_header(message, &message->metadata);
---> switch (field)
---> case DBUS_MESSAGE_FIELD_PATH:
---> case DBUS_MESSAGE_FIELD_INTERFACE:
---> case DBUS_MESSAGE_FIELD_MEMBER:
---> case DBUS_MESSAGE_FIELD_REPLY_SERIAL:
---> case DBUS_MESSAGE_FIELD_UNIX_FDS:
---> c_dvar_read(&v, "<u>)", c_dvar_type_u, &metadata->fields.unix_fds);
---> r = message_parse_body(message, &message->metadata);
---> r = controller_dbus_dispatch(controller, m);
---> switch (message->header->type) {
---> case DBUS_MESSAGE_TYPE_METHOD_CALL:
---> r = controller_dispatch_object(controller,
message_read_serial(message),message->metadata.fields.interface,
message->metadata.fields.member,message->metadata.fields.path,
message->metadata.fields.signature,message);
---> if (strcmp(path, "/org/bus1/DBus/Broker") == 0)
---> controller_dispatch_controller(controller, serial, member, path, signature, message);
---> controller_method_add_listener( ... )
---> listener_fd = fdlist_get(fds, fd_index);
---> r = controller_add_listener(controller, &listener, path, listener_fd, policy);
---> listener_init_with_fd(...)
---> listener->bus = bus;
---> dispatch_file_init(.., listener_dispatch,... ) 这里面的判断:message->header->type来自于src\launch\launcher.c的sd_bus_message_new_method_call( )
最后，调用到函数listener_dispatch():
c
static int listener_dispatch(DispatchFile *file)
---> fd = accept4(listener->socket_fd, NULL, NULL, SOCK_CLOEXEC | SOCK_NONBLOCK);2.1.2 socket takeover
需要首先查阅该篇文章: Zero Downtime Release: Disruption-free Load Balancing of a Multi-Billion User Website.
可参考文件src\util\log.c中的函数log_fd_send():
c
static int log_fd_send(int destination_fd, int payload_fd)
---> control.cmsg.cmsg_level = SOL_SOCKET;
---> control.cmsg.cmsg_type = SCM_RIGHTS;
---> l = sendmsg(destination_fd, &msg, MSG_NOSIGNAL);2.1.3 描述符附录
简单看一下应用程序使用的描述符:
c
root@evb-ast2500:~# ps | grep -i "dbus"
191 messageb 5836 S /usr/bin/dbus-broker-launch --scope system --audit
192 messageb 2756 S dbus-broker --log 4 --controller 9 --machine-id c47c0d3d042848a1908818ca62f0644e --max-bytes 536870912 --max-fds 4096 --max-matches 16384 --audit
344 root 2952 S grep -i dbus
root@evb-ast2500:~#
root@evb-ast2500:~# ls -al /proc/191/fd
dr-x------ 2 root root 13 Feb 27 17:54 .
dr-xr-xr-x 8 messageb messageb 0 Feb 27 17:54 ..
lr-x------ 1 root root 64 Feb 27 17:54 0 -> /dev/null
lrwx------ 1 root root 64 Feb 27 17:54 1 -> socket:[2242]
lrwx------ 1 root root 64 Feb 27 17:55 10 -> anon_inode:[pidfd]
lrwx------ 1 root root 64 Feb 27 17:55 11 -> anon_inode:[timerfd]
lrwx------ 1 root root 64 Feb 27 17:55 12 -> /memfd:dbus-broker-log (deleted)
lrwx------ 1 root root 64 Feb 27 17:54 2 -> socket:[2242]
lrwx------ 1 root root 64 Feb 27 17:54 3 -> socket:[2210]
lrwx------ 1 root root 64 Feb 27 17:54 4 -> socket:[2246]
lrwx------ 1 root root 64 Feb 27 17:55 5 -> anon_inode:[eventpoll]
lrwx------ 1 root root 64 Feb 27 17:55 6 -> anon_inode:[signalfd]
lr-x------ 1 root root 64 Feb 27 17:55 7 -> anon_inode:inotify
lrwx------ 1 root root 64 Feb 27 17:55 8 -> socket:[2254]
lrwx------ 1 root root 64 Feb 27 17:54 9 -> socket:[2303]
root@evb-ast2500:~#2.2 bus的建立
2.2.1 launcher bus
在文件:src\launch\launcher.c中调用如下函数launcher_new():
c
int launcher_new(Launcher **launcherp, int fd_listen, bool audit, const char *configfile, bool user_scope)
---> r = sd_bus_new(&launcher->bus_controller);另外函数:launcher_run()调用如下:
c
int launcher_run(Launcher *launcher)
---> r = sd_bus_set_fd(launcher->bus_controller, controller[0], controller[0]);
---> r = sd_bus_start(launcher->bus_controller);
---> bus_set_state(bus, BUS_OPENING);
---> if (bus->input_fd >= 0)
---> r = bus_start_fd(bus);
---> return bus_send_hello(bus);//直接返回，并未调用"hello"由于开启的服务是:
在函数:launcher_connect()中:
c
static int launcher_connect(Launcher *launcher)
---> r = sd_bus_open_system(&launcher->bus_regular);
---> sd_bus_open_system_with_description(ret, NULL);
---> r = sd_bus_new(&b);
---> r = bus_set_address_system(b);
---> b->bus_client = true;
---> b->is_local = true;
---> r = sd_bus_start(b);
---> bus_set_state(bus, BUS_OPENING);
---> r = bus_start_address(bus);
---> return bus_send_hello(bus);
---> r = sd_bus_message_new_method_call(
bus, &m,"org.freedesktop.DBus","/org/freedesktop/DBus",
"org.freedesktop.DBus","Hello");
---> sd_bus_call_async(bus, NULL, m, hello_callback, NULL, 0);设置系统默认的bus:"unix:path=/run/dbus/system_bus_socket".
在如下调用关系中:
c
static int listener_dispatch(DispatchFile *file)
---> ...
---> r = peer_dispatch(&peer->connection.socket_file);
---> r = peer_dispatch_connection(peer, dispatch_file_events(file) & interest[i]);
---> r = connection_dispatch(&peer->connection, events);
---> r = connection_dequeue(&peer->connection, &m);
---> r = message_parse_metadata(m);
---> r = driver_dispatch(peer, m);
---> r = driver_dispatch_internal(peer, message);
---> if (string_equal(message->metadata.fields.destination, "org.freedesktop.DBus"))
---> driver_dispatch_interface(peer,...)
---> static const DriverInterface interfaces[] = {
---> { "org.freedesktop.DBus", driver_methods },
---> { "org.freedesktop.DBus.Monitoring", monitoring_methods },
---> ...
---> }其中，函数:driver_methods():
c
static const DriverMethod driver_methods[] = {
{ "Hello", false, NULL, driver_method_hello, ...}
static int driver_method_hello(Peer *peer, const char *path, CDVar *in_v, uint32_t serial, CDVar *out_v)
---> peer_register(peer);
---> peer->registered = true;
---> unique_name = address_to_string(&(Address)ADDRESS_INIT_ID(peer->id)); //获取unique name
---> c_dvar_write(out_v, "(s)", unique_name); //将unique name返回
---> r = driver_send_reply(peer, out_v, serial);
---> driver_send_reply_with_fds(peer, var, serial, NULL, 0);
---> r = message_new_outgoing(&message, data, n_data);
---> r = driver_send_unicast(peer, message);
---> r = driver_name_owner_changed(peer->bus, &peer->name_owner_changed_matches, NULL, NULL, peer);
---> r = driver_notify_name_owner_changed(bus, matches, name, old_owner_str, new_owner_str);
---> .fields = {
---> .path = "/org/freedesktop/DBus",
---> .interface = "org.freedesktop.DBus",
---> .member = "NameOwnerChanged",
---> }
3.分析一下权威代码
3.1 LogControl
可以查看网站:https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.LogControl1.html 或者systemd的代码: man\logcontrol-example.c.
注意:此处是建立一个server
c
int main(int argc, char **argv)
---> r = sd_bus_default(&bus);
---> bus_choose_default(&bus_open);
---> *bus_open = sd_bus_open_system;
---> r = bus_set_address_system(b);
---> b->bus_client = true;
---> r = sd_bus_start(b);
---> bus_send_hello(bus);
---> r = sd_bus_message_new_method_call(
bus,&m,"org.freedesktop.DBus",
"/org/freedesktop/DBus","org.freedesktop.DBus","Hello");
---> sd_bus_call_async(bus, NULL, m, hello_callback, NULL, 0);
---> bus_default(bus_open, busp, ret);
---> r = bus_open(&b); //实际调用sd_bus_open_system()
---> r = sd_bus_add_object_vtable(bus, NULL,
"/org/freedesktop/LogControl1","org.freedesktop.LogControl1",vtable,&o);
---> r = sd_bus_request_name(bus, "org.freedesktop.Example", 0);
---> r = sd_bus_call_method(
bus,"org.freedesktop.DBus","/org/freedesktop/DBus",
"org.freedesktop.DBus","RequestName",NULL,&reply,"su",name,param);
---> r = sd_bus_message_read(reply, "u", &ret);
---> for()
---> {
---> r = sd_bus_wait(bus, UINT64_MAX);
---> r = bus_poll(bus, false, timeout_usec);
---> r = sd_bus_process(bus, NULL);
---> bus_process_internal(bus, ret);
---> }另外，systemd的代码:src\busctl\busctl.c是客户端代码，查看如下命令:
c
* $ busctl --user set-property org.freedesktop.Example \
* /org/freedesktop/LogControl1 \
* org.freedesktop.LogControl1 \
* LogLevel \
* "s" debug代码调用如下:
c
static int run(int argc, char *argv[])
---> busctl_main(argc, argv);
---> r = acquire_bus(false, &bus);
---> r = sd_bus_new(&bus);
---> sd_bus_set_description(bus, "busctl");
---> r = sd_bus_set_bus_client(bus, true);
---> bus->bus_client = b;
---> r = sd_bus_set_watch_bind(bus, arg_watch_bind);
---> r = sd_bus_start(bus);
---> bus_send_hello(bus);
---> r = sd_bus_message_new_method_call(bus, &m, argv[1], argv[2],
"org.freedesktop.DBus.Properties", "Set");
---> r = sd_bus_message_append(m, "ss", argv[3], argv[4]);
---> r = sd_bus_message_open_container(m, 'v', argv[5]);
---> r = message_append_cmdline(m, argv[5], &passed_fdset, &p);
---> r = sd_bus_message_close_container(m);
---> r = sd_bus_call(bus, m, arg_timeout, &error, NULL);从上面的代码，有很重要的一个点:
c
bus->bus_client = true所以:
- 针对
dbus-broker这一个后台程序而言，例子中的server与busctl的client均是客户端。
使用BMC可以测试如下命令:
对应调用:
c
int bus_set_address_user(sd_bus *b)
---> a = secure_getenv("DBUS_SESSION_BUS_ADDRESS");
---> e = secure_getenv("XDG_RUNTIME_DIR");可以确定并无这2个环境变量。
对应如下命令:
- 客户端和服务端程序，均需要发送
hello，执行hello调用
- 服务端程序，需要使用
RequestName调用
