package js

import "github.com/tdewolff/parse/v2/js"

Package js is an ECMAScript5.1 lexer following the specifications at http://www.ecma-international.org/ecma-262/5.1/.

Index

Examples

Variables

var Keywords = map[string]TokenType{

	"await":      AwaitToken,
	"break":      BreakToken,
	"case":       CaseToken,
	"catch":      CatchToken,
	"class":      ClassToken,
	"const":      ConstToken,
	"continue":   ContinueToken,
	"debugger":   DebuggerToken,
	"default":    DefaultToken,
	"delete":     DeleteToken,
	"do":         DoToken,
	"else":       ElseToken,
	"enum":       EnumToken,
	"export":     ExportToken,
	"extends":    ExtendsToken,
	"false":      FalseToken,
	"finally":    FinallyToken,
	"for":        ForToken,
	"function":   FunctionToken,
	"if":         IfToken,
	"import":     ImportToken,
	"in":         InToken,
	"instanceof": InstanceofToken,
	"new":        NewToken,
	"null":       NullToken,
	"return":     ReturnToken,
	"super":      SuperToken,
	"switch":     SwitchToken,
	"this":       ThisToken,
	"throw":      ThrowToken,
	"true":       TrueToken,
	"try":        TryToken,
	"typeof":     TypeofToken,
	"var":        VarToken,
	"void":       VoidToken,
	"while":      WhileToken,
	"with":       WithToken,
	"yield":      YieldToken,

	"let":        LetToken,
	"static":     StaticToken,
	"implements": ImplementsToken,
	"interface":  InterfaceToken,
	"package":    PackageToken,
	"private":    PrivateToken,
	"protected":  ProtectedToken,
	"public":     PublicToken,

	"as":     AsToken,
	"async":  AsyncToken,
	"from":   FromToken,
	"get":    GetToken,
	"meta":   MetaToken,
	"of":     OfToken,
	"set":    SetToken,
	"target": TargetToken,
}

Functions

func AsDecimalLiteral

func AsDecimalLiteral(b []byte) bool

func AsIdentifierName

func AsIdentifierName(b []byte) bool

func IsIdentifier

func IsIdentifier(tt TokenType) bool

IsIdentifier matches Identifier, i.e. IdentifierName but not ReservedWord. Does not match yield or await.

func IsIdentifierContinue

func IsIdentifierContinue(b []byte) bool

IsIdentifierContinue returns true if the byte-slice start is a continuation of an identifier

func IsIdentifierEnd

func IsIdentifierEnd(b []byte) bool

IsIdentifierEnd returns true if the byte-slice end is a start or continuation of an identifier

func IsIdentifierName

func IsIdentifierName(tt TokenType) bool

IsIdentifierName matches IdentifierName, i.e. any identifier

func IsIdentifierStart

func IsIdentifierStart(b []byte) bool

IsIdentifierStart returns true if the byte-slice start is the start of an identifier

func IsNumeric

func IsNumeric(tt TokenType) bool

func IsOperator

func IsOperator(tt TokenType) bool

func IsPunctuator

func IsPunctuator(tt TokenType) bool

func IsReservedWord

func IsReservedWord(tt TokenType) bool

IsReservedWord matches ReservedWord

Types

type AST

type AST struct {
	Comment   []byte // first comment in file
	BlockStmt        // module
}

func Parse

func Parse(r *parse.Input) (*AST, error)

Parse returns a JS AST tree of.

func (*AST) String

func (ast *AST) String() string

type Alias

type Alias struct {
	Name    []byte // can be nil
	Binding []byte // can be nil
}

func (Alias) String

func (alias Alias) String() string

type Arguments

type Arguments struct {
	List []IExpr
	Rest IExpr // can be nil
}

func (Arguments) String

func (n Arguments) String() string

type ArrayExpr

type ArrayExpr struct {
	List []Element
}

func (ArrayExpr) String

func (n ArrayExpr) String() string

type ArrowFunc

type ArrowFunc struct {
	Async  bool
	Params Params
	Body   BlockStmt
}

func (ArrowFunc) String

func (n ArrowFunc) String() string

type BinaryExpr

type BinaryExpr struct {
	Op   TokenType
	X, Y IExpr
}

func (BinaryExpr) String

func (n BinaryExpr) String() string

type BindingArray

type BindingArray struct {
	List []BindingElement
	Rest IBinding // can be nil
}

func (BindingArray) String

func (n BindingArray) String() string

type BindingElement

type BindingElement struct {
	Binding IBinding // can be nil (in case of ellision)
	Default IExpr    // can be nil
}

func (BindingElement) String

func (n BindingElement) String() string

type BindingObject

type BindingObject struct {
	List []BindingObjectItem
	Rest *Var // can be nil
}

func (BindingObject) String

func (n BindingObject) String() string

type BindingObjectItem

type BindingObjectItem struct {
	Key   *PropertyName // can be nil
	Value BindingElement
}

type BlockStmt

type BlockStmt struct {
	List []IStmt
	Scope
}

func (BlockStmt) String

func (n BlockStmt) String() string

type BranchStmt

type BranchStmt struct {
	Type  TokenType
	Label []byte // can be nil
}

func (BranchStmt) String

func (n BranchStmt) String() string

type CallExpr

type CallExpr struct {
	X    IExpr
	Args Arguments
}

func (CallExpr) String

func (n CallExpr) String() string

type CaseClause

type CaseClause struct {
	TokenType
	Cond IExpr // can be nil
	List []IStmt
}

type ClassDecl

type ClassDecl struct {
	Name    *Var  // can be nil
	Extends IExpr // can be nil
	Methods []MethodDecl
}

func (ClassDecl) String

func (n ClassDecl) String() string

type CondExpr

type CondExpr struct {
	Cond, X, Y IExpr
}

func (CondExpr) String

func (n CondExpr) String() string

type DebuggerStmt

type DebuggerStmt struct {
}

func (DebuggerStmt) String

func (n DebuggerStmt) String() string

type DeclType

type DeclType uint16
const (
	NoDecl       DeclType = iota // undeclared variables
	VariableDecl                 // var
	FunctionDecl                 // function
	LexicalDecl                  // let, const, class
	ArgumentDecl                 // function, method, and catch statement arguments
	ExprDecl                     // function expression name or class expression name
)

func (DeclType) String

func (decl DeclType) String() string

type DoWhileStmt

type DoWhileStmt struct {
	Cond IExpr
	Body IStmt
}

func (DoWhileStmt) String

func (n DoWhileStmt) String() string

type DotExpr

type DotExpr struct {
	X    IExpr
	Y    LiteralExpr
	Prec OpPrec
}

func (DotExpr) String

func (n DotExpr) String() string

type Element

type Element struct {
	Value  IExpr // can be nil
	Spread bool
}

type EmptyStmt

type EmptyStmt struct {
}

func (EmptyStmt) String

func (n EmptyStmt) String() string

type ExportStmt

type ExportStmt struct {
	List    []Alias
	Module  []byte // can be nil
	Default bool
	Decl    IExpr
}

func (ExportStmt) String

func (n ExportStmt) String() string

type ExprStmt

type ExprStmt struct {
	Value IExpr
}

func (ExprStmt) String

func (n ExprStmt) String() string

type ForInStmt

type ForInStmt struct {
	Init  IExpr
	Value IExpr
	Body  BlockStmt
}

func (ForInStmt) String

func (n ForInStmt) String() string

type ForOfStmt

type ForOfStmt struct {
	Await bool
	Init  IExpr
	Value IExpr
	Body  BlockStmt
}

func (ForOfStmt) String

func (n ForOfStmt) String() string

type ForStmt

type ForStmt struct {
	Init IExpr // can be nil
	Cond IExpr // can be nil
	Post IExpr // can be nil
	Body BlockStmt
}

func (ForStmt) String

func (n ForStmt) String() string

type FuncDecl

type FuncDecl struct {
	Async     bool
	Generator bool
	Name      *Var // can be nil
	Params    Params
	Body      BlockStmt
}

func (FuncDecl) String

func (n FuncDecl) String() string

type GroupExpr

type GroupExpr struct {
	X IExpr
}

func (GroupExpr) String

func (n GroupExpr) String() string

type IBinding

type IBinding interface {
	String() string
	// contains filtered or unexported methods
}

type IExpr

type IExpr interface {
	String() string
	// contains filtered or unexported methods
}

type IStmt

type IStmt interface {
	String() string
	// contains filtered or unexported methods
}

type IfStmt

type IfStmt struct {
	Cond IExpr
	Body IStmt
	Else IStmt // can be nil
}

func (IfStmt) String

func (n IfStmt) String() string

type ImportMetaExpr

type ImportMetaExpr struct {
}

func (ImportMetaExpr) String

func (n ImportMetaExpr) String() string

type ImportStmt

type ImportStmt struct {
	List    []Alias
	Default []byte // can be nil
	Module  []byte
}

func (ImportStmt) String

func (n ImportStmt) String() string

type IndexExpr

type IndexExpr struct {
	X     IExpr
	Index IExpr
	Prec  OpPrec
}

func (IndexExpr) String

func (n IndexExpr) String() string

type LabelledStmt

type LabelledStmt struct {
	Label []byte
	Value IStmt
}

func (LabelledStmt) String

func (n LabelledStmt) String() string

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

Lexer is the state for the lexer.

func NewLexer

func NewLexer(r *parse.Input) *Lexer

NewLexer returns a new Lexer for a given io.Reader.

Example

Code:

{
	l := NewLexer(parse.NewInputString("var x = 'lorem ipsum';"))
	out := ""
	for {
		tt, data := l.Next()
		if tt == ErrorToken {
			break
		}
		out += string(data)
	}
	fmt.Println(out)
	// Output: var x = 'lorem ipsum';
}

Output:

var x = 'lorem ipsum';

func (*Lexer) Err

func (l *Lexer) Err() error

Err returns the error encountered during lexing, this is often io.EOF but also other errors can be returned.

func (*Lexer) Next

func (l *Lexer) Next() (TokenType, []byte)

Next returns the next Token. It returns ErrorToken when an error was encountered. Using Err() one can retrieve the error message.

func (*Lexer) RegExp

func (l *Lexer) RegExp() (TokenType, []byte)

RegExp reparses the input stream for a regular expression. It is assumed that we just received DivToken or DivEqToken with Next(). This function will go back and read that as a regular expression.

type LiteralExpr

type LiteralExpr struct {
	TokenType
	Data []byte
}

func (LiteralExpr) String

func (n LiteralExpr) String() string

type MethodDecl

type MethodDecl struct {
	Static    bool
	Async     bool
	Generator bool
	Get       bool
	Set       bool
	Name      PropertyName
	Params    Params
	Body      BlockStmt
}

func (MethodDecl) String

func (n MethodDecl) String() string

type NewExpr

type NewExpr struct {
	X    IExpr
	Args *Arguments // can be nil
}

func (NewExpr) String

func (n NewExpr) String() string

type NewTargetExpr

type NewTargetExpr struct {
}

func (NewTargetExpr) String

func (n NewTargetExpr) String() string

type ObjectExpr

type ObjectExpr struct {
	List []Property
}

func (ObjectExpr) String

func (n ObjectExpr) String() string

type OpPrec

type OpPrec int
const (
	OpExpr     OpPrec = iota // a,b
	OpAssign                 // a?b:c, yield x, ()=>x, async ()=>x, a=b, a+=b, ...
	OpCoalesce               // a??b
	OpOr                     // a||b
	OpAnd                    // a&&b
	OpBitOr                  // a|b
	OpBitXor                 // a^b
	OpBitAnd                 // a&b
	OpEquals                 // a==b, a!=b, a===b, a!==b
	OpCompare                // a<b, a>b, a<=b, a>=b, a instanceof b, a in b
	OpShift                  // a<<b, a>>b, a>>>b
	OpAdd                    // a+b, a-b
	OpMul                    // a*b, a/b, a%b
	OpExp                    // a**b
	OpUnary                  // ++x, --x, delete x, void x, typeof x, +x, -x, ~x, !x, await x
	OpUpdate                 // x++, x--
	OpLHS                    // CallExpr/OptChainExpr or NewExpr
	OpCall                   // a?.b, a(b), super(a), import(a)
	OpNew                    // new a
	OpMember                 // a[b], a.b, a`b`, super[x], super.x, new.target, import.meta, new a(b)
	OpPrimary                // literal, function, class, parenthesized
)

func (OpPrec) String

func (prec OpPrec) String() string

type OptChainExpr

type OptChainExpr struct {
	X IExpr
	Y IExpr // can be CallExpr, IndexExpr, LiteralExpr, or TemplateExpr
}

func (OptChainExpr) String

func (n OptChainExpr) String() string

type Params

type Params struct {
	List []BindingElement
	Rest IBinding // can be nil
}

func (Params) String

func (n Params) String() string

type Parser

type Parser struct {
	// contains filtered or unexported fields
}

Parser is the state for the parser.

type Property

type Property struct {
	// either Name or Spread are set. When Spread is set then Value is AssignmentExpression
	// if Init is set then Value is IdentifierReference, otherwise it can also be MethodDefinition
	Name   *PropertyName // can be nil
	Spread bool
	Value  IExpr
	Init   IExpr // can be nil
}

func (Property) String

func (n Property) String() string

type PropertyName

type PropertyName struct {
	Literal  LiteralExpr
	Computed IExpr // can be nil
}

func (PropertyName) IsComputed

func (n PropertyName) IsComputed() bool

func (PropertyName) IsIdent

func (n PropertyName) IsIdent(data []byte) bool

func (PropertyName) IsSet

func (n PropertyName) IsSet() bool

func (PropertyName) String

func (n PropertyName) String() string

type ReturnStmt

type ReturnStmt struct {
	Value IExpr // can be nil
}

func (ReturnStmt) String

func (n ReturnStmt) String() string

type Scope

type Scope struct {
	Parent, Func   *Scope   // Parent is nil for global scope, Parent equals Func for function scope
	Declared       VarArray // Link in Var are always nil
	Undeclared     VarArray
	NumVarDecls    uint16 // number of variable declaration statements in a function scope
	NumArguments   uint16 // offset into Undeclared to mark variables used in arguments
	IsGlobalOrFunc bool
	HasWith        bool
}

Scope is a function or block scope with a list of variables declared and used

func (*Scope) Declare

func (s *Scope) Declare(decl DeclType, name []byte) (*Var, bool)

Declare a new variable

func (*Scope) HoistUndeclared

func (s *Scope) HoistUndeclared()

func (*Scope) MarkArguments

func (s *Scope) MarkArguments()

func (Scope) String

func (s Scope) String() string

func (*Scope) UndeclareScope

func (s *Scope) UndeclareScope()

func (*Scope) Use

func (s *Scope) Use(name []byte) *Var

Use a variable

type SwitchStmt

type SwitchStmt struct {
	Init IExpr
	List []CaseClause
}

func (SwitchStmt) String

func (n SwitchStmt) String() string

type TemplateExpr

type TemplateExpr struct {
	Tag  IExpr // can be nil
	List []TemplatePart
	Tail []byte
	Prec OpPrec
}

func (TemplateExpr) String

func (n TemplateExpr) String() string

type TemplatePart

type TemplatePart struct {
	Value []byte
	Expr  IExpr
}

type ThrowStmt

type ThrowStmt struct {
	Value IExpr
}

func (ThrowStmt) String

func (n ThrowStmt) String() string

type TokenType

type TokenType uint16 // from LSB to MSB: 8 bits for tokens per category, 1 bit for numeric, 1 bit for punctuator, 1 bit for operator, 1 bit for identifier, 4 bits unused

TokenType determines the type of token, eg. a number or a semicolon.

const (
	ErrorToken TokenType = iota // extra token when errors occur
	WhitespaceToken
	LineTerminatorToken // \r \n \r\n
	CommentToken
	CommentLineTerminatorToken
	StringToken
	TemplateToken
	TemplateStartToken
	TemplateMiddleToken
	TemplateEndToken
	RegExpToken
)

TokenType values.

const (
	NumericToken TokenType = 0x0100 + iota
	DecimalToken
	BinaryToken
	OctalToken
	HexadecimalToken
	BigIntToken
)
const (
	PunctuatorToken   TokenType = 0x0200 + iota
	OpenBraceToken              // {
	CloseBraceToken             // }
	OpenParenToken              // (
	CloseParenToken             // )
	OpenBracketToken            // [
	CloseBracketToken           // ]
	DotToken                    // .
	SemicolonToken              // ;
	CommaToken                  // ,
	QuestionToken               // ?
	ColonToken                  // :
	ArrowToken                  // =>
	EllipsisToken               // ...
)
const (
	OperatorToken TokenType = 0x0600 + iota
	EqToken                 // =
	EqEqToken               // ==
	EqEqEqToken             // ===
	NotToken                // !
	NotEqToken              // !=
	NotEqEqToken            // !==
	LtToken                 // <
	LtEqToken               // <=
	LtLtToken               // <<
	LtLtEqToken             // <<=
	GtToken                 // >
	GtEqToken               // >=
	GtGtToken               // >>
	GtGtEqToken             // >>=
	GtGtGtToken             // >>>
	GtGtGtEqToken           // >>>=
	AddToken                // +
	AddEqToken              // +=
	IncrToken               // ++
	SubToken                // -
	SubEqToken              // -=
	DecrToken               // --
	MulToken                // *
	MulEqToken              // *=
	ExpToken                // **
	ExpEqToken              // **=
	DivToken                // /
	DivEqToken              // /=
	ModToken                // %
	ModEqToken              // %=
	BitAndToken             // &
	BitOrToken              // |
	BitXorToken             // ^
	BitNotToken             // ~
	BitAndEqToken           // &=
	BitOrEqToken            // |=
	BitXorEqToken           // ^=
	AndToken                // &&
	OrToken                 // ||
	NullishToken            // ??
	OptChainToken           // ?.

	// unused in lexer
	PosToken      // +a
	NegToken      // -a
	PreIncrToken  // ++a
	PreDecrToken  // --a
	PostIncrToken // a++
	PostDecrToken // a--
)
const (
	ReservedToken TokenType = 0x0800 + iota
	AwaitToken
	BreakToken
	CaseToken
	CatchToken
	ClassToken
	ConstToken
	ContinueToken
	DebuggerToken
	DefaultToken
	DeleteToken
	DoToken
	ElseToken
	EnumToken
	ExportToken
	ExtendsToken
	FalseToken
	FinallyToken
	ForToken
	FunctionToken
	IfToken
	ImportToken
	InToken
	InstanceofToken
	NewToken
	NullToken
	ReturnToken
	SuperToken
	SwitchToken
	ThisToken
	ThrowToken
	TrueToken
	TryToken
	TypeofToken
	YieldToken
	VarToken
	VoidToken
	WhileToken
	WithToken
)
const (
	IdentifierToken TokenType = 0x1000 + iota
	AsToken
	AsyncToken
	FromToken
	GetToken
	ImplementsToken
	InterfaceToken
	LetToken
	MetaToken
	OfToken
	PackageToken
	PrivateToken
	ProtectedToken
	PublicToken
	SetToken
	StaticToken
	TargetToken
)

func (TokenType) Bytes

func (tt TokenType) Bytes() []byte

Bytes returns the string representation of a TokenType.

func (TokenType) String

func (tt TokenType) String() string

String returns the string representation of a TokenType.

type TryStmt

type TryStmt struct {
	Body    BlockStmt
	Binding IBinding   // can be nil
	Catch   *BlockStmt // can be nil
	Finally *BlockStmt // can be nil
}

func (TryStmt) String

func (n TryStmt) String() string

type UnaryExpr

type UnaryExpr struct {
	Op TokenType
	X  IExpr
}

func (UnaryExpr) String

func (n UnaryExpr) String() string

type Var

type Var struct {
	Decl DeclType
	Data []byte
	Link *Var // is set when merging variable uses, as in:  {a} {var a}  where the first lins to the second
	Uses uint16
}

Var is a variable, where Decl is the type of declaration and can be var|function for function scoped variables, let|const|class for block scoped variables

func (*Var) Name

func (v *Var) Name() []byte

func (*Var) String

func (v *Var) String() string

type VarArray

type VarArray []*Var

func (VarArray) String

func (vs VarArray) String() string

type VarDecl

type VarDecl struct {
	TokenType
	List []BindingElement
}

func (VarDecl) String

func (n VarDecl) String() string

type VarsByUses

type VarsByUses VarArray

VarsByUses is sortable by uses in descending order

func (VarsByUses) Len

func (vs VarsByUses) Len() int

func (VarsByUses) Less

func (vs VarsByUses) Less(i, j int) bool

func (VarsByUses) Swap

func (vs VarsByUses) Swap(i, j int)

type WhileStmt

type WhileStmt struct {
	Cond IExpr
	Body IStmt
}

func (WhileStmt) String

func (n WhileStmt) String() string

type WithStmt

type WithStmt struct {
	Cond IExpr
	Body IStmt
}

func (WithStmt) String

func (n WithStmt) String() string

type YieldExpr

type YieldExpr struct {
	Generator bool
	X         IExpr // can be nil
}

func (YieldExpr) String

func (n YieldExpr) String() string

Source Files

ast.go lex.go parse.go table.go tokentype.go util.go

Version
v2.5.0
Published
Aug 26, 2020
Platform
js/wasm
Imports
8 packages
Last checked
2 hours ago

Tools for package owners.