Class: Ibex::Frontend::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/ibex/frontend/lexer.rb,
lib/ibex/frontend/lexer_recovery.rb

Overview

Tokenizes a racc-compatible grammar while preserving source locations. rubocop:disable Metrics/ClassLength -- token scanners stay together to preserve cursor invariants.

Constant Summary collapse

PUNCTUATION = 
%w[: | ; = < > ? * + , ( )].to_h do |character|
  [character, character.to_sym]
end.freeze
USER_CODE =

: Hash[String, Symbol]

/\A----[ \t]+(header|inner|footer)[ \t]*(?:\r?\n|\z)/
NODE_DIRECTIVE =

: Regexp

/\A@node(?=\s|\z)/
PERCENT_DIRECTIVES = 
{
  "%expect-rr" => [:identifier, "expect_rr"],
  "%precedence" => [:identifier, "precedence"],
  "%param" => [:identifier, "param"], "%printer" => [:identifier, "printer"],
  "%recover" => [:identifier, "recover"], "%on_error_reduce" => [:identifier, "on_error_reduce"],
  "%test" => [:identifier, "test"], "%inline" => [:inline, "%inline"],
  "%empty" => [:empty, "%empty"]
}.freeze

Instance Method Summary collapse

Constructor Details

#initialize(source, file: "(grammar)") ⇒ Lexer

Returns a new instance of Lexer.



29
30
31
32
33
 
# File 'lib/ibex/frontend/lexer.rb', line 29

def initialize(source, file: "(grammar)")
  @cursor = SourceCursor.new(source, file)
  @segments = [] #: Array[Segment]
  @emitted_tokens = [] #: Array[Token]
end

Instance Method Details

#next_token ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/ibex/frontend/lexer.rb', line 53

def next_token
  eof_token = @eof_token
  return eof_token if eof_token

  @active_token_start = nil
  skip_ignored
  start = @cursor.position
  @active_token_start = start
  scanned = if @cursor.eof?
              token(:eof, nil)
            else
              scan_from_cursor
            end
  emit_token(scanned, start) unless scanned.span
  @emitted_tokens << scanned
  @eof_token = scanned if scanned.type == :eof
  @active_token_start = nil
  scanned
end

#tokenize ⇒ Object 



36
37
38
39
40
41
42
43
 
# File 'lib/ibex/frontend/lexer.rb', line 36

def tokenize
  tokens = [] #: Array[Token]
  loop do
    token = next_token
    tokens << token
    return tokens if token.type == :eof
  end
end

#tokenize_document ⇒ Object 



46
47
48
49
50
 
# File 'lib/ibex/frontend/lexer.rb', line 46

def tokenize_document
  tokenize unless @eof_token
  cst = CST::Document.new(@segments)
  SourceDocument.new(source: @cursor.source, file: @cursor.file, tokens: @emitted_tokens, cst: cst)
end

#tokenize_document_recovering(max_diagnostics: 20) ⇒ Object

Recover lexical failures while retaining every source byte.



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/ibex/frontend/lexer_recovery.rb', line 8

def tokenize_document_recovering(max_diagnostics: 20)
  unless max_diagnostics.is_a?(Integer) && max_diagnostics.positive?
    raise ArgumentError, "max_diagnostics must be a positive integer"
  end

  diagnostics = [] #: Array[Diagnostic]
  until @eof_token
    begin
      next_token
    rescue Ibex::Error => e
      diagnostic = recover_lexical_error(e, capture: diagnostics.length < max_diagnostics)
      diagnostics << diagnostic if diagnostic
    end
  end
  cst = CST::Document.new(@segments)
  document = SourceDocument.new(source: @cursor.source, file: @cursor.file,
                                tokens: @emitted_tokens, cst: cst)
  [document, diagnostics.freeze]
end