Merge pull request #232 from ipfs-force-community/fix/dtynn/use_bufio…

…_reader_instead_of_scanner

fix(vsmgr): use bufio.Reader to avoid bufio.Scanner: token too long

venus-sector-manager/cmd/venus-sector-manager/processor/processor.go

@@ -34,18 +34,20 @@ var processorWdPostCmd = &cli.Command{
 
 		plog.Info("ready")
 
-		in := bufio.NewScanner(os.Stdin)
+		in := bufio.NewReaderSize(os.Stdin, 1<<20)
 		out := bufio.NewWriter(os.Stdout)
 		var outMu = &sync.Mutex{}
 
+		var readErr error
 		for {
-			if !in.Scan() {
+			b, err := in.ReadBytes('\n')
+			if err != nil {
+				readErr = err
 				break
 			}
 
 			var req ext.Request
-			b := in.Bytes()
-			err := json.Unmarshal(b, &req)
+			err = json.Unmarshal(b, &req)
 			if err != nil {
 				plog.Warnf("decode incoming request: %s", err)
 				continue
@@ -70,8 +72,8 @@ var processorWdPostCmd = &cli.Command{
 			}()
 		}
 
-		if err := in.Err(); err != nil {
-			plog.Warnf("stdin broken: %s", err)
+		if readErr != nil {
+			plog.Warnf("stdin broken: %s", readErr)
 		}
 
 		return nil

venus-sector-manager/modules/impl/prover/ext/prover.go

@@ -122,12 +122,18 @@ func (sp *subProcessor) handleResponse(ctx context.Context) {
 	splog.Info("response loop start")
 	defer splog.Info("response loop stop")
 
-	scanner := bufio.NewScanner(sp.stdout)
+	reader := bufio.NewReaderSize(sp.stdout, 1<<20)
+	var readErr error
+
+	for {
+		b, err := reader.ReadBytes('\n')
+		if err != nil {
+			readErr = err
+			break
+		}
 
-	for scanner.Scan() {
 		var resp Response
-		b := scanner.Bytes()
-		err := json.Unmarshal(b, &resp)
+		err = json.Unmarshal(b, &resp)
 		if err != nil {
 			splog.Warnf("decode response from stdout of sub: %s", err)
 			continue
@@ -144,8 +150,8 @@ func (sp *subProcessor) handleResponse(ctx context.Context) {
 		}
 	}
 
-	if err := scanner.Err(); err != nil {
-		splog.Warnf("sub stdout scanner error: %s", err)
+	if readErr != nil {
+		splog.Warnf("sub stdout scanner error: %s", readErr)
 	}
 
 }