esp32s3中使用双通道通信解决TCP粘包问题

在使用esp32 idf例程中的tcp_server和tcp_client通信测试时发现，

在tcp_server端，接收到一帧数据之后必须马上回复至少一个字节，才能保证每帧数据不粘包，

如果不回复操作，300ms以内的通信时延会导致tcp严重粘包，后续解析这些数据费时费力，

可能跟lwip的回环读写机制有关，这严重打乱了双向通信逻辑。

换一种方式，使用udp广播来作为数据传输通道，使用tcp连接来做状态检测，这样就可以

避免粘包问题。

udp广播服务如下

/**
 * udp服务器，高速通信，控制器控制命令传输通道(不需要应答的)
 * */
static void udp_server_task(void *pvParameters)
{
    unsigned char rx_buffer[128];
    char addr_str[128];
    int addr_family = (int)pvParameters;//ipv4 or ipv6
    int ip_protocol = 0;
    struct sockaddr_in6 dest_addr;

    while (1) {

        if (addr_family == AF_INET) {
            struct sockaddr_in *dest_addr_ip4 = (struct sockaddr_in *)&dest_addr;
            /***
             * 接收广播地址:
             * 192.168.100.1
             * 192.168.100.255
             * 255.255.255.255
             */
            dest_addr_ip4->sin_addr.s_addr = htonl(INADDR_ANY);
            dest_addr_ip4->sin_family = AF_INET;
            dest_addr_ip4->sin_port = htons(UDP_SERVER_PORT);
            ip_protocol = IPPROTO_IP;
        } else if (addr_family == AF_INET6) {
            bzero(&dest_addr.sin6_addr.un, sizeof(dest_addr.sin6_addr.un));
            dest_addr.sin6_family = AF_INET6;
            dest_addr.sin6_port = htons(UDP_SERVER_PORT);
            ip_protocol = IPPROTO_IPV6;
        }

        global_udpsock_handle = socket(addr_family, SOCK_DGRAM, ip_protocol);
        if (global_udpsock_handle < 0) {
            ESP_LOGE(TAG, "Unable to create socket: errno %d", errno);

            /**因为正在创建的时候网络可能还没有完全连接上，不能退出*/
            vTaskDelay(100 / portTICK_PERIOD_MS);//100ms
            continue;
        }
        ESP_LOGI(TAG, "UDP Socket created");

#if defined(CONFIG_LWIP_NETBUF_RECVINFO) && !defined(CONFIG_EXAMPLE_IPV6)
        int enable = 1;
        lwip_setsockopt(sock, IPPROTO_IP, IP_PKTINFO, &enable, sizeof(enable));
#endif

#if defined(CONFIG_EXAMPLE_IPV4) && defined(CONFIG_EXAMPLE_IPV6)
        if (addr_family == AF_INET6) {
            // Note that by default IPV6 binds to both protocols, it is must be disabled
            // if both protocols used at the same time (used in CI)
            int opt = 1;
            setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
            setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt));
        }
#endif
//        // Set timeout 接收广播数据超时时间
//        struct timeval timeout;
//        timeout.tv_sec = 10;
//        timeout.tv_usec = 0;
//        setsockopt (sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);

        /**
         * E (106995) BOT-TAG: Socket unable to bind: errno 112
         * */
        int opt = 1;
        setsockopt(global_udpsock_handle, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));


        int err = bind(global_udpsock_handle, (struct sockaddr *)&dest_addr, sizeof(dest_addr));
        if (err < 0) {
            ESP_LOGE(TAG, "udp Socket unable to bind: errno %d", errno);

            close(global_udpsock_handle);
            /**因为正在创建的时候网络可能还没有完全连接上，不能退出*/
            vTaskDelay(100 / portTICK_PERIOD_MS);//100ms
            continue;
        }
        ESP_LOGI(TAG, "udp Socket bound, port %d",UDP_SERVER_PORT);

        struct sockaddr_storage source_addr; // Large enough for both IPv4 or IPv6
        socklen_t socklen = sizeof(source_addr);

#if defined(CONFIG_LWIP_NETBUF_RECVINFO) && !defined(CONFIG_EXAMPLE_IPV6)
        struct iovec iov;
        struct msghdr msg;
        struct cmsghdr *cmsgtmp;
        u8_t cmsg_buf[CMSG_SPACE(sizeof(struct in_pktinfo))];

        iov.iov_base = rx_buffer;
        iov.iov_len = sizeof(rx_buffer);
        msg.msg_control = cmsg_buf;
        msg.msg_controllen = sizeof(cmsg_buf);
        msg.msg_flags = 0;
        msg.msg_iov = &iov;
        msg.msg_iovlen = 1;
        msg.msg_name = (struct sockaddr *)&source_addr;
        msg.msg_namelen = socklen;
#endif

        ESP_LOGI(TAG, "udp start Waiting for data");

        /**
         * 重新启动udp server可以清除之前接收的缓存数据，防止对下一个连接影响
         * */
        while (1) {

        	//tcp没有有效连接则处于睡眠等待状态
        	if(global_tcpsock_handle <= 0)
        	{
        		vTaskDelay(100 / portTICK_PERIOD_MS);//100ms
                continue;
        	}

#if defined(CONFIG_LWIP_NETBUF_RECVINFO) && !defined(CONFIG_EXAMPLE_IPV6)
            int len = recvmsg(global_udpsock_handle, &msg, 0);
#else
            int len = recvfrom(global_udpsock_handle, rx_buffer, sizeof(rx_buffer) - 1, 0, (struct sockaddr *)&source_addr, &socklen);
#endif
            // Error occurred during receiving
            if (len < 0) {
                ESP_LOGE(TAG, "udp recvfrom failed: errno %d", errno);

                /**
                 * E (615075) BOT-TAG: udp Socket unable to bind: errno 9
                 * */
                //sock关闭后稍等一下，不用立即去创建和bind
                vTaskDelay(100 / portTICK_PERIOD_MS);

                break;
            }
            // Data received
            else {
                // Get the sender's ip address as string
                if (source_addr.ss_family == PF_INET) {
                    inet_ntoa_r(((struct sockaddr_in *)&source_addr)->sin_addr, addr_str, sizeof(addr_str) - 1);
#if defined(CONFIG_LWIP_NETBUF_RECVINFO) && !defined(CONFIG_EXAMPLE_IPV6)
                    for ( cmsgtmp = CMSG_FIRSTHDR(&msg); cmsgtmp != NULL; cmsgtmp = CMSG_NXTHDR(&msg, cmsgtmp) ) {
                        if ( cmsgtmp->cmsg_level == IPPROTO_IP && cmsgtmp->cmsg_type == IP_PKTINFO ) {
                            struct in_pktinfo *pktinfo;
                            pktinfo = (struct in_pktinfo*)CMSG_DATA(cmsgtmp);
                            ESP_LOGI(TAG, "dest ip: %s\n", inet_ntoa(pktinfo->ipi_addr));
                        }
                    }
#endif
                } else if (source_addr.ss_family == PF_INET6) {
                    inet6_ntoa_r(((struct sockaddr_in6 *)&source_addr)->sin6_addr, addr_str, sizeof(addr_str) - 1);
                }


                //rx_buffer[len] = 0; // Null-terminate whatever we received and treat like a string...
                //ESP_LOGI(TAG, "udp Received %d bytes from %s:", len, addr_str);
                //ESP_LOGI(TAG, "%s", rx_buffer);
                //print0x(rx_buffer,len);

                cmd_resolve_high_speed(rx_buffer, len);

//                int err = sendto(global_udpsock_handle, rx_buffer, len, 0, (struct sockaddr *)&source_addr, sizeof(source_addr));
//                if (err < 0) {
//                    ESP_LOGE(TAG, "Error occurred during sending: errno %d", errno);
//                    break;
//                }
            }
        }

//        if (global_udpsock_handle != -1) {
//            ESP_LOGE(TAG, "Shutting down socket and restarting...");
//            shutdown(global_udpsock_handle, 0);
//            close(global_udpsock_handle);
//        }
    }
    vTaskDelete(NULL);
}

tcp状态监听服务如下

/**
 * tcp服务端,慢速通道，处理维护心跳包
 * */
static void tcp_server_task(void *pvParameters)
{
    char addr_str[128];
    int addr_family = (int)pvParameters;//ipv4 or ipv6
    int ip_protocol = 0;
    int keepAlive = 1;
    int option = 1;

    int keepIdle = KEEPALIVE_IDLE;
    int keepInterval = KEEPALIVE_INTERVAL;
    int keepCount = KEEPALIVE_COUNT;
    struct sockaddr_storage dest_addr;

#ifdef CONFIG_EXAMPLE_IPV4
    if (addr_family == AF_INET) {
        struct sockaddr_in *dest_addr_ip4 = (struct sockaddr_in *)&dest_addr;
        dest_addr_ip4->sin_addr.s_addr = htonl(INADDR_ANY);
        dest_addr_ip4->sin_family = AF_INET;
        dest_addr_ip4->sin_port = htons(TCP_SERVER_PORT);
        ip_protocol = IPPROTO_IP;
    }
#endif
#ifdef CONFIG_EXAMPLE_IPV6
    if (addr_family == AF_INET6) {
        struct sockaddr_in6 *dest_addr_ip6 = (struct sockaddr_in6 *)&dest_addr;
        bzero(&dest_addr_ip6->sin6_addr.un, sizeof(dest_addr_ip6->sin6_addr.un));
        dest_addr_ip6->sin6_family = AF_INET6;
        dest_addr_ip6->sin6_port = htons(PORT);
        ip_protocol = IPPROTO_IPV6;
    }
#endif

    int listen_sock = socket(addr_family, SOCK_STREAM, ip_protocol);
    if (listen_sock < 0) {
        ESP_LOGE(TAG, "Unable to create socket: errno %d", errno);
        vTaskDelete(NULL);
        return;
    }
    int opt = 1;
    setsockopt(listen_sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));

#if defined(CONFIG_EXAMPLE_IPV4) && defined(CONFIG_EXAMPLE_IPV6)
    // Note that by default IPV6 binds to both protocols, it is must be disabled
    // if both protocols used at the same time (used in CI)
    setsockopt(listen_sock, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt));
#endif

    ESP_LOGI(TAG, "tcp Socket created");

    int err = bind(listen_sock, (struct sockaddr *)&dest_addr, sizeof(dest_addr));
    if (err != 0) {
        ESP_LOGE(TAG, "tcp Socket unable to bind: errno %d", errno);
        ESP_LOGE(TAG, "IPPROTO: %d", addr_family);
        goto CLEAN_UP;
    }
    ESP_LOGI(TAG, "tcp Socket bound, port %d", TCP_SERVER_PORT);


    err = listen(listen_sock, 1);
    if (err != 0) {
        ESP_LOGE(TAG, "Error occurred during listen: errno %d", errno);
        goto CLEAN_UP;
    }

    while (1) {

        ESP_LOGI(TAG, "Socket listening");

        //建立握手随机数校验标志
        generate_com_check();

        hp_check_load();


        struct sockaddr_storage source_addr; // Large enough for both IPv4 or IPv6
        socklen_t addr_len = sizeof(source_addr);
        global_tcpsock_handle = accept(listen_sock, (struct sockaddr *)&source_addr, &addr_len);
        if (global_tcpsock_handle < 0) {
            ESP_LOGE(TAG, "Unable to accept connection: errno %d", errno);
            break;
        }

        // Set tcp keepalive option
        setsockopt(global_tcpsock_handle, SOL_SOCKET, SO_KEEPALIVE, &keepAlive, sizeof(int));
        setsockopt(global_tcpsock_handle, IPPROTO_TCP, TCP_KEEPIDLE, &keepIdle, sizeof(int));
        setsockopt(global_tcpsock_handle, IPPROTO_TCP, TCP_KEEPINTVL, &keepInterval, sizeof(int));
        setsockopt(global_tcpsock_handle, IPPROTO_TCP, TCP_KEEPCNT, &keepCount, sizeof(int));

        
        setsockopt(global_tcpsock_handle, IPPROTO_TCP, TCP_NODELAY, &option, sizeof(int));


        // Convert ip address to string
#ifdef CONFIG_EXAMPLE_IPV4
        if (source_addr.ss_family == PF_INET) {
            inet_ntoa_r(((struct sockaddr_in *)&source_addr)->sin_addr, addr_str, sizeof(addr_str) - 1);
        }
#endif
#ifdef CONFIG_EXAMPLE_IPV6
        if (source_addr.ss_family == PF_INET6) {
            inet6_ntoa_r(((struct sockaddr_in6 *)&source_addr)->sin6_addr, addr_str, sizeof(addr_str) - 1);
        }
#endif
        ESP_LOGI(TAG, "Socket accepted ip address: %s", addr_str);


        //接收控制器端的下发数据
        do_tcpsock_recv();

        //用户主动关闭sock
        do_tcpsock_close();


        do_udpsock_close();

    }

CLEAN_UP:
    close(listen_sock);
    vTaskDelete(NULL);
}