class CLTK::Scanner::Environment

Overview

Used to keep track of the Lexing State. Used as context for token callback yields and gets returned by the lexer

cltk/scanner.cr

#add_token(token : Token, size : Int32)
add a token at the current position doesn't advance the @offset index
#advance_position(count : Int32)
advance the offset by count
#map(&block)
yield the given block for each token and it's position and returns the results in an array
#offset : Int32
current lexing position
#pop_state
pop the last state from the stack
#positions : Array({position: Int32, size: Int32})
positions for lexed tokens
#push_state(state : Symbol)
push a new lexing state to the stack
#state
get the currently active lexing state
#tokens : Array({Symbol} | {Symbol, String?})
the lexed tokens
#tokens_with_positions
returns an array of all lexed tokens and their positions

def add_token(token : Token, size : Int32) #

add a token at the current position doesn't advance the @offset index

[View source]

def advance_position(count : Int32) #

advance the offset by count

[View source]

def map(&block) #

yield the given block for each token and it's position and returns the results in an array

[View source]

def offset : Int32 #

current lexing position

[View source]

def pop_state #

pop the last state from the stack

[View source]

def positions : Array({position: Int32, size: Int32}) #

positions for lexed tokens

[View source]

def push_state(state : Symbol) #

push a new lexing state to the stack

[View source]

def state #

get the currently active lexing state

[View source]

def tokens : Array({Symbol} | {Symbol, String?}) #

the lexed tokens

[View source]

def tokens_with_positions #

returns an array of all lexed tokens and their positions

[View source]