go 问题记录（日志丢失）

问题描述：

在go程序中，通过执行一个命令启动一个子命令，并通过pipe读取子程序的标准输入和输出，通过scanner默认按行读取，此时如果子程序输出时没有携带'\n'，scanner就不会打印输出，而是会累积到缓存buf上限，最终被丢弃，直到遇到一个\n，然后输出所有的内容，默认buf缓存上限时65536，如果日志打印处还有限制，如glog就限制最大的打印字节数为4096，那么就会导致日志再次丢失。

解决方法：

不适用scanner去按行读取，直接读取管道的内容，然后设置上限，超过时或者遇到'\n'时打印

测试代码：

子程序：

c 复制代码

#include <stdio.h>
#include <unistd.h>

int main() {
int count = 0;

while (1) {
    fprintf(stderr, "%d", count);
    count = (count + 1) % 10;
    usleep(500); // Sleep for 500,000 microseconds (0.5 seconds)
}

    return 0;
}

主程序：

go 复制代码

package main

import (
    "bufio"
    "fmt"
    "os/exec"
    "strings"
    "log"
)

func main() {
    cmd := exec.Command("./test")
    stdout, err := cmd.StdoutPipe()
    if err != nil {
        fmt.Println("Error creating StdoutPipe:", err)
        return
    }
    cmd.Stderr = cmd.Stdout

    err = cmd.Start()
    if err != nil {
        fmt.Println("Error starting command:", err)
        return
    }

    scanner := bufio.NewScanner(stdout)
    // scanner.Split(bufio.ScanBytes)

    // buf := ""
    // for scanner.Scan() {
    // 	   buf += scanner.Text()
    //     if strings.Contains(buf, "\n") || len(buf) >= 256 {
    //         log.Printf("%s", buf)
    //         buf = ""
    //     }
    // }
  	for scanner.Scan() {
    	log.Printf("%s", scanner.Text())
    }

    if err := scanner.Err(); err != nil {
    	fmt.Println("Error reading standard output:", err)
    }

    err = cmd.Wait()
    if err != nil {
    	fmt.Println("Error waiting for command to finish:", err)
    }
}

修改程序：

go 复制代码

package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os/exec"
)

func getReaderSize(rd io.Reader) {
	b, ok := rd.(*bufio.Reader)
	if ok {
		log.Printf("rd size: %d", b.Size())
	} else {
		log.Printf("rd is not bufio.Reader")
	}
}

func main() {
	// Command to execute
	cmd := exec.Command("./test")

	// Create a pipe to capture the standard output of the command
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Println("Error creating StdoutPipe:", err)
		return
	}
	cmd.Stderr = cmd.Stdout

	// Start the command
	err = cmd.Start()
	if err != nil {
		fmt.Println("Error starting command:", err)
		return
	}

	 Create a scanner to read the command's standard output
	//scanner := bufio.NewScanner(stdout)
	//scanner.Split(bufio.ScanBytes)
	//
	 Read and print each line from the output
	//buf := make([]byte, 256)
	//bufLen := 0
	//for scanner.Scan() {
	//	buf[bufLen] = scanner.Bytes()[0]
	//	// buf = append(buf, scanner.Bytes()...)
	//	bufLen += 1
	//	if buf[bufLen-1] == '\n' || bufLen >= 256 {
	//		log.Printf("%s", string(buf[:bufLen]))
	//		bufLen = 0
	//	}
	//}
	//
	 Check for errors in scanning
	//if err := scanner.Err(); err != nil {
	//	fmt.Println("Error reading standard output:", err)
	//}

	// Create a buffered reader to read from the command's stdout
	reader := bufio.NewReaderSize(stdout, 256)
	getReaderSize(stdout)
	log.Printf("reader size: %d", reader.Size())
	 Buffer to store incomplete lines
	//var incompleteLine []byte
	//
	 Buffer to read chunks of bytes
	//chunk := make([]byte, 256)
	//
	//for {
	//	// Read a chunk of bytes
	//	n, err := reader.Read(chunk)
	//	if err != nil {
	//		break // Break the loop when an error occurs (e.g., when the command finishes)
	//	}
	//
	//	// Process each byte in the chunk
	//	for i := 0; i < n; i++ {
	//		b := chunk[i]
	//
	//		// Check for newline or length exceeding 256
	//		if b == '\n' || len(incompleteLine) >= 256 {
	//			// Print the line
	//			log.Printf("%s", incompleteLine)
	//
	//			// Reset the incomplete line buffer
	//			incompleteLine = nil
	//		} else {
	//			// Add the byte to the incomplete line buffer
	//			incompleteLine = append(incompleteLine, b)
	//		}
	//	}
	//}

	for {
		s, err := reader.ReadSlice('\n')
		if err != nil && err != bufio.ErrBufferFull {
			if len(s) > 0 {
				log.Printf("reader err but exist data, reader size: %d, read string size: %d, string: %s", reader.Size(), len(s), string(s))
			}
			fmt.Println("Error reader ReadString:", err)
			break // Break the loop when an error occurs (e.g., when the command finishes)
		}
		log.Printf("reader size: %d, read string size: %d, string: %s", reader.Size(), len(s), string(s))
	}

	// Wait for the command to finish
	err = cmd.Wait()
	if err != nil {
		fmt.Println("Error waiting for command to finish:", err)
	}
}

benchmark test:

go 复制代码

package main

import (
	"strconv"
	"strings"
	"testing"
)

func stringTest1() string {
	var buf string
	for i := 0; i < 256; i++ {
		buf += strconv.Itoa(i)
	}
	return buf
}

func stringTest2() string {
	var buf strings.Builder
	for i := 0; i < 256; i++ {
		buf.Write([]byte(strconv.Itoa(i)))
	}
	return buf.String()
}

func stringTest3() string {
	var buf = make([]byte, 0)
	for i := 0; i < 256; i++ {
		buf = append(buf, []byte(strconv.Itoa(i))...)
	}
	return string(buf)
}

func stringTest4() string {
	var buf = make([]byte, 256)
	for i := 0; i < 256; i++ {
		buf[i] = '1'
	}
	return string(buf)
}

func BenchmarkStringTest1(b *testing.B) {
	for i := 0; i < b.N; i++ {
		stringTest1()
	}
}
func BenchmarkStringTest2(b *testing.B) {
	for i := 0; i < b.N; i++ {
		stringTest2()
	}
}
func BenchmarkStringTest3(b *testing.B) {
	for i := 0; i < b.N; i++ {
		stringTest3()
	}
}
func BenchmarkStringTest4(b *testing.B) {
	for i := 0; i < b.N; i++ {
		stringTest4()
	}
}

benchmark test

cmd:

shell 复制代码

go test -bench . -benchmem
go test -bench=<function>