package buffer

import "github.com/tdewolff/parse/v2/buffer"

Package buffer contains buffer and wrapper types for byte slices. It is useful for writing lexers or other high-performance byte slice handling. The `Reader` and `Writer` types implement the `io.Reader` and `io.Writer` respectively and provide a thinner and faster interface than `bytes.Buffer`. The `Lexer` type is useful for building lexers because it keeps track of the start and end position of a byte selection, and shifts the bytes whenever a valid token is found. The `StreamLexer` does the same, but keeps a buffer pool so that it reads a limited amount at a time, allowing to parse from streaming sources.

Index ¶

• Variables
• type Lexer
• func NewLexer(r io.Reader) *Lexer
• func NewLexerBytes(b []byte) *Lexer
• func (z *Lexer) Bytes() []byte
• func (z *Lexer) Err() error
• func (z *Lexer) Lexeme() []byte
• func (z *Lexer) Move(n int)
• func (z *Lexer) Offset() int
• func (z *Lexer) Peek(pos int) byte
• func (z *Lexer) PeekErr(pos int) error
• func (z *Lexer) PeekRune(pos int) (rune, int)
• func (z *Lexer) Pos() int
• func (z *Lexer) Reset()
• func (z *Lexer) Restore()
• func (z *Lexer) Rewind(pos int)
• func (z *Lexer) Shift() []byte
• func (z *Lexer) Skip()
• type Reader
• func NewReader(buf []byte) *Reader
• func (r *Reader) Bytes() []byte
• func (r *Reader) Len() int
• func (r *Reader) Read(b []byte) (n int, err error)
• func (r *Reader) Reset()
• type StreamLexer
• func NewStreamLexer(r io.Reader) *StreamLexer
• func NewStreamLexerSize(r io.Reader, size int) *StreamLexer
• func (z *StreamLexer) Err() error
• func (z *StreamLexer) Free(n int)
• func (z *StreamLexer) Lexeme() []byte
• func (z *StreamLexer) Move(n int)
• func (z *StreamLexer) Peek(pos int) byte
• func (z *StreamLexer) PeekRune(pos int) (rune, int)
• func (z *StreamLexer) Pos() int
• func (z *StreamLexer) Rewind(pos int)
• func (z *StreamLexer) Shift() []byte
• func (z *StreamLexer) ShiftLen() int
• func (z *StreamLexer) Skip()
• type Writer
• func NewStaticWriter(buf []byte) *Writer
• func NewWriter(buf []byte) *Writer
• func (w *Writer) Bytes() []byte
• func (w *Writer) Close() error
• func (w *Writer) Len() int
• func (w *Writer) Reset()
• func (w *Writer) Write(b []byte) (int, error)

Examples ¶

• NewReader
• NewWriter
• Writer.Reset

Variables ¶

var MinBuf = defaultBufSize

MinBuf specifies the default initial length of internal buffers. Solely here to support old versions of parse.

Types ¶

type Lexer ¶

type Lexer struct {
	// contains filtered or unexported fields
}

Lexer is a buffered reader that allows peeking forward and shifting, taking an io.Reader. It keeps data in-memory until Free, taking a byte length, is called to move beyond the data.

func NewLexer ¶

func NewLexer(r io.Reader) *Lexer

NewLexer returns a new Lexer for a given io.Reader, and uses io.ReadAll to read it into a byte slice. If the io.Reader implements Bytes, that is used instead. It will append a NULL at the end of the buffer.

func NewLexerBytes ¶

func NewLexerBytes(b []byte) *Lexer

NewLexerBytes returns a new Lexer for a given byte slice, and appends NULL at the end. To avoid reallocation, make sure the capacity has room for one more byte.

func (*Lexer) Bytes ¶

func (z *Lexer) Bytes() []byte

Bytes returns the underlying buffer.

func (*Lexer) Err ¶

func (z *Lexer) Err() error

Err returns the error returned from io.Reader or io.EOF when the end has been reached.

func (*Lexer) Lexeme ¶

func (z *Lexer) Lexeme() []byte

Lexeme returns the bytes of the current selection.

func (*Lexer) Move ¶

func (z *Lexer) Move(n int)

Move advances the position.

func (*Lexer) Offset ¶

func (z *Lexer) Offset() int

Offset returns the character position in the buffer.

func (*Lexer) Peek ¶

func (z *Lexer) Peek(pos int) byte

Peek returns the ith byte relative to the end position. Peek returns 0 when an error has occurred, Err returns the error.

func (*Lexer) PeekErr ¶

func (z *Lexer) PeekErr(pos int) error

PeekErr returns the error at position pos. When pos is zero, this is the same as calling Err().

func (*Lexer) PeekRune ¶

func (z *Lexer) PeekRune(pos int) (rune, int)

PeekRune returns the rune and rune length of the ith byte relative to the end position.

func (*Lexer) Pos ¶

func (z *Lexer) Pos() int

Pos returns a mark to which can be rewinded.

func (*Lexer) Reset ¶

func (z *Lexer) Reset()

Reset resets position to the underlying buffer.

func (*Lexer) Restore ¶

func (z *Lexer) Restore()

Restore restores the replaced byte past the end of the buffer by NULL.

func (*Lexer) Rewind ¶

func (z *Lexer) Rewind(pos int)

Rewind rewinds the position to the given position.

func (*Lexer) Shift ¶

func (z *Lexer) Shift() []byte

Shift returns the bytes of the current selection and collapses the position to the end of the selection.

func (*Lexer) Skip ¶

func (z *Lexer) Skip()

Skip collapses the position to the end of the selection.

type Reader ¶

type Reader struct {
	// contains filtered or unexported fields
}

Reader implements an io.Reader over a byte slice.

func NewReader ¶

func NewReader(buf []byte) *Reader

NewReader returns a new Reader for a given byte slice.

{
	r := NewReader([]byte("Lorem ipsum"))
	w := &bytes.Buffer{}
	io.Copy(w, r)
	fmt.Println(w.String())
	// Output: Lorem ipsum
}

Lorem ipsum

func (*Reader) Bytes ¶

func (r *Reader) Bytes() []byte

Bytes returns the underlying byte slice.

func (*Reader) Len ¶

func (r *Reader) Len() int

Len returns the length of the buffer.

func (*Reader) Read ¶

func (r *Reader) Read(b []byte) (n int, err error)

Read reads bytes into the given byte slice and returns the number of bytes read and an error if occurred.

func (*Reader) Reset ¶

func (r *Reader) Reset()

Reset resets the position of the read pointer to the beginning of the underlying byte slice.

type StreamLexer ¶

type StreamLexer struct {
	// contains filtered or unexported fields
}

StreamLexer is a buffered reader that allows peeking forward and shifting, taking an io.Reader. It keeps data in-memory until Free, taking a byte length, is called to move beyond the data.

func NewStreamLexer ¶

func NewStreamLexer(r io.Reader) *StreamLexer

NewStreamLexer returns a new StreamLexer for a given io.Reader with a 4kB estimated buffer size. If the io.Reader implements Bytes, that buffer is used instead.

func NewStreamLexerSize ¶

func NewStreamLexerSize(r io.Reader, size int) *StreamLexer

NewStreamLexerSize returns a new StreamLexer for a given io.Reader and estimated required buffer size. If the io.Reader implements Bytes, that buffer is used instead.

func (*StreamLexer) Err ¶

func (z *StreamLexer) Err() error

Err returns the error returned from io.Reader. It may still return valid bytes for a while though.

func (*StreamLexer) Free ¶

func (z *StreamLexer) Free(n int)

Free frees up bytes of length n from previously shifted tokens. Each call to Shift should at one point be followed by a call to Free with a length returned by ShiftLen.

func (*StreamLexer) Lexeme ¶

func (z *StreamLexer) Lexeme() []byte

Lexeme returns the bytes of the current selection.

func (*StreamLexer) Move ¶

func (z *StreamLexer) Move(n int)

Move advances the position.

func (*StreamLexer) Peek ¶

func (z *StreamLexer) Peek(pos int) byte

Peek returns the ith byte relative to the end position and possibly does an allocation. Peek returns zero when an error has occurred, Err returns the error. TODO: inline function

func (*StreamLexer) PeekRune ¶

func (z *StreamLexer) PeekRune(pos int) (rune, int)

PeekRune returns the rune and rune length of the ith byte relative to the end position.

func (*StreamLexer) Pos ¶

func (z *StreamLexer) Pos() int

Pos returns a mark to which can be rewinded.

func (*StreamLexer) Rewind ¶

func (z *StreamLexer) Rewind(pos int)

Rewind rewinds the position to the given position.

func (*StreamLexer) Shift ¶

func (z *StreamLexer) Shift() []byte

Shift returns the bytes of the current selection and collapses the position to the end of the selection. It also returns the number of bytes we moved since the last call to Shift. This can be used in calls to Free.

func (*StreamLexer) ShiftLen ¶

func (z *StreamLexer) ShiftLen() int

ShiftLen returns the number of bytes moved since the last call to ShiftLen. This can be used in calls to Free because it takes into account multiple Shifts or Skips.

func (*StreamLexer) Skip ¶

func (z *StreamLexer) Skip()

Skip collapses the position to the end of the selection.

type Writer ¶

type Writer struct {
	// contains filtered or unexported fields
}

Writer implements an io.Writer over a byte slice.

func NewStaticWriter ¶

func NewStaticWriter(buf []byte) *Writer

NewStaticWriter returns a new Writer for a given byte slice. It does not reallocate and expand the byte-slice.

func NewWriter ¶

func NewWriter(buf []byte) *Writer

NewWriter returns a new Writer for a given byte slice.

{
	w := NewWriter(make([]byte, 0, 11)) // initial buffer length is 11
	w.Write([]byte("Lorem ipsum"))
	fmt.Println(string(w.Bytes()))
	// Output: Lorem ipsum
}

Lorem ipsum

func (*Writer) Bytes ¶

func (w *Writer) Bytes() []byte

Bytes returns the underlying byte slice.

func (*Writer) Close ¶

func (w *Writer) Close() error

Close returns the last error.

func (*Writer) Len ¶

func (w *Writer) Len() int

Len returns the length of the underlying byte slice.

func (*Writer) Reset ¶

func (w *Writer) Reset()

Reset empties and reuses the current buffer. Subsequent writes will overwrite the buffer, so any reference to the underlying slice is invalidated after this call.

{
	w := NewWriter(make([]byte, 0, 11))                 // initial buffer length is 10
	w.Write([]byte("garbage that will be overwritten")) // does reallocation
	w.Reset()
	w.Write([]byte("Lorem ipsum"))
	fmt.Println(string(w.Bytes()))
	// Output: Lorem ipsum
}

Lorem ipsum

func (*Writer) Write ¶

func (w *Writer) Write(b []byte) (int, error)

Write writes bytes from the given byte slice and returns the number of bytes written and an error if occurred. When err != nil, n == 0.

Source Files ¶

buffer.go lexer.go reader.go streamlexer.go writer.go