From d4442af3686032027112437b7b3b2e3a608675f9 Mon Sep 17 00:00:00 2001 From: simulot Date: Fri, 17 Nov 2023 19:20:55 +0100 Subject: [PATCH] fix for #69: Panic: runtime error: slice bounds out of range --- docs/releases.md | 6 ++ immich/metadata/direct.go | 24 ++++--- immich/metadata/quicktime.go | 14 +--- immich/metadata/search.go | 77 ++++++--------------- immich/metadata/search_test.go | 121 ++++++++++++++++++--------------- 5 files changed, 109 insertions(+), 133 deletions(-) diff --git a/docs/releases.md b/docs/releases.md index 91b2e66b..4b074462 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,5 +1,11 @@ # Release notes +## Release 0.8.3 + +### Fiw for #69: Panic: runtime error: slice bounds out of range +Rewriting searchPattern +Add tests + ## Release 0.8.2 ### Fix for #64: segfault when using *.jpg diff --git a/immich/metadata/direct.go b/immich/metadata/direct.go index db586e84..9d4a30a7 100644 --- a/immich/metadata/direct.go +++ b/immich/metadata/direct.go @@ -31,7 +31,8 @@ func GetFileMetaData(fsys fs.FS, name string) (MetaData, error) { // // -func GetFromReader(r io.Reader, ext string) (MetaData, error) { +func GetFromReader(rd io.Reader, ext string) (MetaData, error) { + r := newSliceReader(rd) meta := MetaData{} var err error var dateTaken time.Time @@ -58,10 +59,12 @@ func readExifDateTaken(r io.Reader) (time.Time, error) { return md.DateTaken, err } -// readHEIFDateTaken locate the Exif part and return the date of capture -func readHEIFDateTaken(r io.Reader) (time.Time, error) { +const searchBufferSize = 32 * 1024 - r, err := seekReaderAtPattern(r, []byte{0x45, 0x78, 0x69, 0x66, 0, 0, 0x4d, 0x4d}) +// readHEIFDateTaken locate the Exif part and return the date of capture +func readHEIFDateTaken(r *sliceReader) (time.Time, error) { + b := make([]byte, searchBufferSize) + r, err := searchPattern(r, []byte{0x45, 0x78, 0x69, 0x66, 0, 0, 0x4d, 0x4d}, b) if err != nil { return time.Time{}, err } @@ -74,21 +77,24 @@ func readHEIFDateTaken(r io.Reader) (time.Time, error) { } // readMP4DateTaken locate the mvhd atom and decode the date of capture -func readMP4DateTaken(r io.Reader) (time.Time, error) { +func readMP4DateTaken(r *sliceReader) (time.Time, error) { + b := make([]byte, searchBufferSize) - b, err := searchPattern(r, []byte{'m', 'v', 'h', 'd'}, 60) + r, err := searchPattern(r, []byte{'m', 'v', 'h', 'd'}, b) if err != nil { return time.Time{}, err } - atom, err := decodeMvhdAtom(b) + atom, err := decodeMvhdAtom(r) if err != nil { return time.Time{}, err } return atom.CreationTime, nil } -func readCR3DateTaken(r io.Reader) (time.Time, error) { - r, err := seekReaderAtPattern(r, []byte("CMT1")) +func readCR3DateTaken(r *sliceReader) (time.Time, error) { + b := make([]byte, searchBufferSize) + + r, err := searchPattern(r, []byte("CMT1"), b) if err != nil { return time.Time{}, err } diff --git a/immich/metadata/quicktime.go b/immich/metadata/quicktime.go index 5d3a59e6..2eb9cffa 100644 --- a/immich/metadata/quicktime.go +++ b/immich/metadata/quicktime.go @@ -1,7 +1,6 @@ package metadata import ( - "bytes" "encoding/binary" "time" ) @@ -48,8 +47,7 @@ type MvhdAtom struct { // NextTrackID uint32 } -func decodeMvhdAtom(b []byte) (*MvhdAtom, error) { - r := &sliceReader{Reader: bytes.NewReader(b)} +func decodeMvhdAtom(r *sliceReader) (*MvhdAtom, error) { a := &MvhdAtom{} @@ -94,13 +92,3 @@ func convertTime64(timestamp uint64) time.Time { // Convert the Unix timestamp to time.Time return time.Unix(unixTimestamp, 0) } - -type sliceReader struct { - *bytes.Reader -} - -func (r *sliceReader) ReadSlice(l int) ([]byte, error) { - b := make([]byte, l) - _, err := r.Read(b) - return b, err -} diff --git a/immich/metadata/search.go b/immich/metadata/search.go index a19716c4..31df76c5 100644 --- a/immich/metadata/search.go +++ b/immich/metadata/search.go @@ -1,87 +1,52 @@ package metadata import ( + "bufio" "bytes" "io" ) -const searchBufferSize = 32 * 1024 - -func searchPattern(r io.Reader, pattern []byte, maxDataLen int) ([]byte, error) { - var err error - pos := 0 - // Create a buffer to hold the chunk of dataZ - buffer := make([]byte, searchBufferSize) - ofs := 0 - - var bytesRead int - for { - // Read a chunk of data into the buffer - bytesRead, err = r.Read(buffer[bytesRead-ofs:]) - if err != nil && err != io.EOF { - return nil, err - } - - // Search for the pattern within the buffer - index := bytes.Index(buffer, pattern) - if index >= 0 { - if index < searchBufferSize-maxDataLen { - return buffer[index : index+maxDataLen], nil - } - ofs = index - } else { - ofs = max(bytesRead-maxDataLen-1, 0) - } - - // Check if end of file is reached - if err == io.EOF || ofs > bytesRead { - break - } +type sliceReader struct { + bufio.Reader +} - // Move the remaining bytes of the current buffer to the beginning - copy(buffer, buffer[ofs:bytesRead]) - pos += bytesRead +func newSliceReader(r io.Reader) *sliceReader { + return &sliceReader{ + Reader: *bufio.NewReader(r), } +} - return nil, io.EOF +func (r *sliceReader) ReadSlice(l int) ([]byte, error) { + b := make([]byte, l) + _, err := r.Read(b) + return b, err } -func seekReaderAtPattern(r io.Reader, pattern []byte) (io.Reader, error) { +func searchPattern(r io.Reader, pattern []byte, buffer []byte) (*sliceReader, error) { var err error pos := 0 - // Create a buffer to hold the chunk of dataZ - buffer := make([]byte, searchBufferSize) ofs := 0 var bytesRead int for { // Read a chunk of data into the buffer - bytesRead, err = r.Read(buffer[bytesRead-ofs:]) - if err != nil && err != io.EOF { + bytesRead, err = r.Read(buffer[ofs:]) + if err != nil { return nil, err } // Search for the pattern within the buffer - index := bytes.Index(buffer, pattern) + index := bytes.Index(buffer[:ofs+bytesRead], pattern) if index >= 0 { - if index < searchBufferSize-len(pattern) { - return io.MultiReader(bytes.NewReader(buffer[index:]), r), nil - } - ofs = index - } else { - ofs = bytesRead - len(pattern) - 1 - } - - // Check if end of file is reached - if err == io.EOF { - break + return newSliceReader(io.MultiReader(bytes.NewReader(buffer[index:]), r)), nil } // Move the remaining bytes of the current buffer to the beginning - copy(buffer, buffer[ofs:bytesRead]) + p := bytesRead + ofs - len(pattern) + 1 + + copy(buffer, buffer[p:bytesRead+ofs]) + ofs = len(pattern) - 1 pos += bytesRead } - - return nil, io.EOF } diff --git a/immich/metadata/search_test.go b/immich/metadata/search_test.go index c4d44567..73d821c0 100644 --- a/immich/metadata/search_test.go +++ b/immich/metadata/search_test.go @@ -2,7 +2,6 @@ package metadata import ( "bytes" - "crypto/rand" "io" "reflect" "testing" @@ -10,82 +9,94 @@ import ( func GenRandomBytes(size int) (blk []byte) { blk = make([]byte, size) - rand.Read(blk) + for i := 0; i < size; i++ { + blk[i] = byte(i & 0xff) + } return } func Test_searchPattern(t *testing.T) { - type args struct { - r io.Reader - pattern []byte - maxDataLen int - } + var searchBufferSize = 20 tests := []struct { name string - args args - want []byte - wantErr bool + r io.Reader + pattern []byte + found bool }{ { - name: "notin", - args: args{ - r: bytes.NewReader(append(GenRandomBytes(searchBufferSize/3), "this is the date:2023-08-01T20:20:00 in the middle of the buffer"...)), - pattern: []byte("nothere"), - maxDataLen: 24, - }, - want: nil, - wantErr: true, + name: "notin", + r: bytes.NewReader(GenRandomBytes(searchBufferSize * 3)), + pattern: []byte{5, 4, 3, 2}, + found: false, + }, + { + name: "at end of reader", + r: bytes.NewReader(GenRandomBytes(searchBufferSize * 3)), + pattern: []byte{56, 57, 58, 59}, + found: true, + }, + { + name: "at 1st buffer boundary", + r: bytes.NewReader(GenRandomBytes(searchBufferSize * 3)), + pattern: []byte{18, 19, 20, 21}, + found: true, + }, + { + name: "at 2nd buffer boundary", + r: bytes.NewReader(GenRandomBytes(searchBufferSize * 3)), + pattern: []byte{34, 35, 36, 37}, + found: true, }, { - name: "middle", - args: args{ - r: bytes.NewReader(append(GenRandomBytes(searchBufferSize/3), "this is the date:2023-08-01T20:20:00 in the middle of the buffer"...)), - pattern: []byte("date:"), - maxDataLen: 24, - }, - want: []byte("date:2023-08-01T20:20:00"), - wantErr: false, + name: "not in real", + r: bytes.NewReader(append(GenRandomBytes(searchBufferSize/3), "this is the date:2023-08-01T20:20:00 in the middle of the buffer"...)), + pattern: []byte("nothere"), + found: false, }, { - name: "beginning", - args: args{ - r: bytes.NewReader([]byte("date:2023-08-01T20:20:00 in the middle of the buffer")), - pattern: []byte("date:"), - maxDataLen: 24, - }, - want: []byte("date:2023-08-01T20:20:00"), - wantErr: false, + name: "middle", + r: bytes.NewReader(append(GenRandomBytes(searchBufferSize/3), "this is the date:2023-08-01T20:20:00 in the middle of the buffer"...)), + pattern: []byte("date:"), + found: true, }, { - name: "2ndbuffer", - args: args{ - r: bytes.NewReader(append(GenRandomBytes(3*searchBufferSize), "this is the date:2023-08-01T20:20:00 in the middle of the buffer"...)), - pattern: []byte("date:"), - maxDataLen: 24, - }, - want: []byte("date:2023-08-01T20:20:00"), - wantErr: false, + name: "beginning", + r: bytes.NewReader([]byte("date:2023-08-01T20:20:00 in the middle of the buffer")), + pattern: []byte("date:"), + found: true, }, { - name: "crossing buffer boundaries", - args: args{ - r: bytes.NewReader(append(append(GenRandomBytes(2*searchBufferSize-10), "date:2023-08-01T20:20:00 in the middle of the buffer"...), GenRandomBytes(searchBufferSize-10)...)), - pattern: []byte("date:"), - maxDataLen: 24, - }, - want: []byte("date:2023-08-01T20:20:00"), - wantErr: false, + name: "2ndbuffer", + r: bytes.NewReader(append(GenRandomBytes(3*searchBufferSize), "this is the date:2023-08-01T20:20:00 in the middle of the buffer"...)), + pattern: []byte("date:"), + found: true, + }, + { + name: "crossing buffer boundaries", + r: bytes.NewReader(append(append(GenRandomBytes(2*searchBufferSize-10), "date:2023-08-01T20:20:00 in the middle of the buffer"...), GenRandomBytes(searchBufferSize-10)...)), + pattern: []byte("date:"), + found: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := searchPattern(tt.args.r, tt.args.pattern, tt.args.maxDataLen) - if (err != nil) != tt.wantErr { - t.Errorf("searchPattern() error = %v, wantErr %v", err, tt.wantErr) + b := make([]byte, searchBufferSize) + r, err := searchPattern(tt.r, tt.pattern, b) + if err != nil && tt.found { + t.Errorf("Pattern %v not found", tt.pattern) + return + } + if !tt.found && err == io.EOF { return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("searchPattern() = %v, want %v", got, tt.want) + got, err := r.ReadSlice(len(tt.pattern)) + if err != nil { + t.Errorf("Can't read result: %s", err) + return + } + + if !reflect.DeepEqual(got, tt.pattern) { + t.Errorf("searchPattern() = %v, want %v", got, tt.pattern) } }) }