package english

import "gopkg.in/neurosnap/sentences.v1/english"

Index ¶

• func NewSentenceTokenizer(s *sentences.Storage) (*sentences.DefaultSentenceTokenizer, error)
• type MultiPunctWordAnnotation
• func (a *MultiPunctWordAnnotation) Annotate(tokens []*sentences.Token) []*sentences.Token
• type WordTokenizer
• func NewWordTokenizer(p sentences.PunctStrings) *WordTokenizer
• func (e *WordTokenizer) HasSentEndChars(t *sentences.Token) bool

Functions ¶

func NewSentenceTokenizer ¶

func NewSentenceTokenizer(s *sentences.Storage) (*sentences.DefaultSentenceTokenizer, error)

English customized sentence tokenizer.

Types ¶

type MultiPunctWordAnnotation ¶

type MultiPunctWordAnnotation struct {
	*sentences.Storage
	sentences.TokenParser
	sentences.TokenGrouper
	sentences.Ortho
}

Attempts to tease out custom Abbreviations, e.g. F.B.I.

func (*MultiPunctWordAnnotation) Annotate ¶

func (a *MultiPunctWordAnnotation) Annotate(tokens []*sentences.Token) []*sentences.Token

type WordTokenizer ¶

type WordTokenizer struct {
	sentences.DefaultWordTokenizer
}

func NewWordTokenizer ¶

func NewWordTokenizer(p sentences.PunctStrings) *WordTokenizer

func (*WordTokenizer) HasSentEndChars ¶

func (e *WordTokenizer) HasSentEndChars(t *sentences.Token) bool

Find any punctuation excluding the period final

Source Files ¶

main.go