FastDDS之共享内存

初始化流程

1. RTPSParticipantImpl初始化

RTPSParticipantImpl::RTPSParticipantImpl(...): ..., , has_shm_transport_(false), ... {
	// ......
#ifdef SHM_TRANSPORT_BUILTIN
        if (!is_intraprocess_only())
        {
            SharedMemTransportDescriptor shm_transport;
            // We assume (Linux) UDP doubles the user socket buffer size in kernel, so
            // the equivalent segment size in SHM would be socket buffer size x 2
            auto segment_size_udp_equivalent =
                    std::max(m_att.sendSocketBufferSize, m_att.listenSocketBufferSize) * 2;
            shm_transport.segment_size(segment_size_udp_equivalent);
            // Use same default max_message_size on both UDP and SHM
            shm_transport.max_message_size(descriptor.max_message_size());
            has_shm_transport_ |= m_network_Factory.RegisterTransport(&shm_transport);
        }
#endif // ifdef SHM_TRANSPORT_BUILTIN
	// ......
}

2. NetworkFactory中注册transport

bool NetworkFactory::RegisterTransport(
        const TransportDescriptorInterface* descriptor,
        const fastrtps::rtps::PropertyPolicy* properties)
{
    bool wasRegistered = false;

    uint32_t minSendBufferSize = std::numeric_limits<uint32_t>::max();

    **std::unique_ptr<TransportInterface> transport(descriptor->create_transport());**

    if (transport)
    {
        **if (transport->init(properties))**
        {
            minSendBufferSize = transport->get_configuration()->min_send_buffer_size();
            mRegisteredTransports.emplace_back(std::move(transport));
            wasRegistered = true;
        }
        // ......
    }

    return wasRegistered;
}

3. 创建SharedMemTransport

TransportInterface* SharedMemTransportDescriptor::create_transport() const
{
    return new SharedMemTransport(*this);
}

4. 初始化SharedMemTransport

bool SharedMemTransport::init(
        const fastrtps::rtps::PropertyPolicy*)
{
    // TODO(Adolfo): Calculate this value from UDP sockets buffers size.
    static constexpr uint32_t shm_default_segment_size = 512 * 1024;

    if (configuration_.segment_size() == 0)
    {
        configuration_.segment_size(shm_default_segment_size);
    }

    if (configuration_.segment_size() < configuration_.max_message_size())
    {
        EPROSIMA_LOG_ERROR(RTPS_MSG_OUT, "max_message_size cannot be greater than segment_size");
        return false;
    }

#ifdef ANDROID
    if (access(BOOST_INTERPROCESS_SHARED_DIR_PATH, W_OK) != F_OK)
    {
        EPROSIMA_LOG_WARNING(RTPS_MSG_OUT,
                "Unable to write on " << BOOST_INTERPROCESS_SHARED_DIR_PATH << ". SHM Transport not enabled");
        return false;
    }
#endif // ifdef ANDROID

    try
    {
        shared_mem_manager_ = SharedMemManager::create(SHM_MANAGER_DOMAIN);
        if (!shared_mem_manager_)
        {
            return false;
        }
        shared_mem_segment_ = shared_mem_manager_->create_segment(configuration_.segment_size(),
                        configuration_.port_queue_capacity());

        // Memset the whole segment to zero in order to force physical map of the buffer
        auto buffer = shared_mem_segment_->alloc_buffer(configuration_.segment_size(),
                        (std::chrono::steady_clock::now() + std::chrono::milliseconds(100)));
        memset(buffer->data(), 0, configuration_.segment_size());
        buffer.reset();

        if (!configuration_.rtps_dump_file().empty())
        {
            auto packets_file_consumer = std::unique_ptr<SHMPacketFileConsumer>(
                new SHMPacketFileConsumer(configuration_.rtps_dump_file()));

            packet_logger_ = std::make_shared<PacketsLog<SHMPacketFileConsumer>>();
            packet_logger_->RegisterConsumer(std::move(packets_file_consumer));
        }
    }
    catch (std::exception& e)
    {
        EPROSIMA_LOG_ERROR(RTPS_MSG_OUT, e.what());
        return false;
    }

    return true;
}

至此，共享内存相关的初始化完成，接下来就可以深入看看共享内存的实现细节了。

接收流程

1. RTPSParticipantImpl创建接收句柄

bool RTPSParticipantImpl::createReceiverResources(
        LocatorList_t& Locator_list,
        bool ApplyMutation,
        bool RegisterReceiver)
{
	// ......
    for (auto it_loc = Locator_list.begin(); it_loc != Locator_list.end(); ++it_loc)
    {
        bool ret = m_network_Factory.BuildReceiverResources(*it_loc, newItemsBuffer, max_receiver_buffer_size);
        if (!ret && ApplyMutation)
        {
            uint32_t tries = 0;
            while (!ret && (tries < m_att.builtin.mutation_tries))
            {
                tries++;
                applyLocatorAdaptRule(*it_loc);
                ret = m_network_Factory.BuildReceiverResources(*it_loc, newItemsBuffer, max_receiver_buffer_size);
            }
        }
        // ......

2. 调用NetworkFactory::BuildReceiverResources，创建ReceiverResource

bool NetworkFactory::BuildReceiverResources(
        Locator_t& local,
        std::vector<std::shared_ptr<ReceiverResource>>& returned_resources_list,
        uint32_t receiver_max_message_size)
{
    // ......
                std::shared_ptr<ReceiverResource> newReceiverResource = std::shared_ptr<ReceiverResource>(
                    new ReceiverResource(*transport, local, max_recv_buffer_size));

                if (newReceiverResource->mValid)
                {
                    returned_resources_list.push_back(newReceiverResource);
                    returnedValue = true;
                }
   // ......

3. 创建ReceiverResource

ReceiverResource::ReceiverResource(...) : ...
{
    // Internal channel is opened and assigned to this resource.
    mValid = transport.OpenInputChannel(locator, this, max_message_size_);
    if (!mValid)
    {
        return; // Invalid resource to be discarded by the factory.
    }

    // Implementation functions are bound to the right transport parameters
    Cleanup = [&transport, locator]()
            {
                transport.CloseInputChannel(locator);
            };
    LocatorMapsToManagedChannel = [&transport, locator](const Locator_t& locatorToCheck) -> bool
            {
                return locator.kind == locatorToCheck.kind && transport.DoInputLocatorsMatch(locator, locatorToCheck);
            };
}

4. 打开发送句柄

bool SharedMemTransport::OpenInputChannel(
        const Locator& locator,
        TransportReceiverInterface* receiver,
        uint32_t maxMsgSize)
{
    // ......
    if (!IsInputChannelOpen(locator))
    {
        try
        {
            auto channel_resource = CreateInputChannelResource(locator, maxMsgSize, receiver);
            input_channels_.push_back(channel_resource);
        }
        // ......
    }

    return true;
}

5. 创建发送句柄。这里根据address里的信息决定是实现多写一读还是多写多读的模式。这里构造port、创建Listener、创建SharedMemChannelResource启动接收线程。

SharedMemChannelResource* SharedMemTransport::CreateInputChannelResource(
        const Locator& locator,
        uint32_t maxMsgSize,
        TransportReceiverInterface* receiver)
{
    (void) maxMsgSize;

    // Multicast locators implies ReadShared (Multiple readers) ports.
    auto open_mode = locator.address[0] == 'M' ? SharedMemGlobal::Port::OpenMode::ReadShared :
            SharedMemGlobal::Port::OpenMode::ReadExclusive;

    return new SharedMemChannelResource(
        shared_mem_manager_->open_port(
            locator.port,
            configuration_.port_queue_capacity(),
            configuration_.healthy_check_timeout_ms(),
            open_mode)->create_listener(),
        locator,
        receiver,
        configuration_.rtps_dump_file());
}

发送流程

1. SharedMemTransport::send

bool SharedMemTransport::send(
        const octet* send_buffer,
        uint32_t send_buffer_size,
        fastrtps::rtps::LocatorsIterator* destination_locators_begin,
        fastrtps::rtps::LocatorsIterator* destination_locators_end,
        const std::chrono::steady_clock::time_point& max_blocking_time_point)
{
    using namespace eprosima::fastdds::statistics::rtps;

#if !defined(_WIN32)
    cleanup_output_ports();
#endif // if !defined(_WIN32)

    fastrtps::rtps::LocatorsIterator& it = *destination_locators_begin;

    bool ret = true;

    std::shared_ptr<SharedMemManager::Buffer> shared_buffer;

    try
    {
        while (it != *destination_locators_end)
        {
            if (IsLocatorSupported(*it))
            {
                // Only copy the first time
                if (shared_buffer == nullptr)
                {
                    remove_statistics_submessage(send_buffer, send_buffer_size);
                    shared_buffer = copy_to_shared_buffer(send_buffer, send_buffer_size, max_blocking_time_point);
                }

                ret &= send(shared_buffer, *it);

                if (packet_logger_ && ret)
                {
                    packet_logger_->QueueLog({packet_logger_->now(), Locator(), *it, shared_buffer});
                }
            }

            ++it;
        }
    }
    catch (const std::exception& e)
    {
        EPROSIMA_LOG_INFO(RTPS_TRANSPORT_SHM, e.what());
        (void)e;

        // Segment overflow with discard policy doesn't return error.
        if (!shared_buffer)
        {
            ret = true;
        }
        else
        {
            ret = false;
        }
    }

    return ret;

}

共享内存实现

SharedMemTransportDescriptor中需要配置的字段如下：

字段名	含义
segment_size_	共享内存segment大小
maxMessageSize	允许传递消息的最大长度
port_queue_capacity_	port的队列容量
healthy_check_timeout_ms_	健康检查的超时时间
rtps_dump_file_	rtps协议dump的文件路径

BufferNode中Status字段说明：

• validity_id：当缓冲区入队到端口队列时，这个validity_id会被复制到该端口的BufferDescription中，如果发送进程需要强制恢复缓冲区，只需要递增validity_id，接收进程必须检查此validity_id与BufferDescription中的validity_id是否相等，若不相等，则说明发送端缓冲区已经失效。
• enqueued_count：每次缓冲区入队到端口队列时，该计数器递增，每次缓冲区从端口队列中弹出进行处理时，该计数器递减。
• processing_count：当监听进程开始处理缓冲区时，处理器计数递增，这样，发送方就知道缓冲区是正在被处理还是仅仅在端口队列中排队