Class: Ibex::BisonImport::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/ibex/bison_import/tokenizer.rb

Overview

Iterative scanner for the punctuation needed to recover Bison rules.

Defined Under Namespace

Classes: Token

Constant Summary collapse

IDENTIFIER_START =

: Regexp

/[A-Za-z_$.]/
IDENTIFIER_CONTINUE =

: Regexp

/[A-Za-z0-9_$.-]/

Instance Method Summary collapse

Constructor Details

#initialize(source, start_line:, max_tokens:) ⇒ Tokenizer

Returns a new instance of Tokenizer.



29
30
31
32
33
34
35
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 29

def initialize(source, start_line:, max_tokens:)
  @source = source
  @index = 0
  @line = start_line
  @column = 1
  @max_tokens = max_tokens
end

Instance Method Details

#tokenize ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 38

def tokenize
  tokens = [] #: Array[Token]
  until eof?
    skip_ignored
    break if eof?

    tokens << scan_token
    check_token_budget(tokens.length)
  end
  tokens.freeze
end