Module: Ibex::Runtime::GeneratedLexer

Defined in:
lib/ibex/runtime/generated_lexer.rb

Overview

Runtime contract mixed into parser classes that declare a lexer. rubocop:disable Metrics/ModuleLength -- matching, actions, state, and positions share one session invariant.

Constant Summary collapse

NO_EMISSION =

: Object

Object.new.freeze
EMPTY_GREEN_TRIVIA =

: Array

empty_green_trivia.freeze

Instance Method Summary collapse

Instance Method Details

#lex(source, file: "(input)") ⇒ Object

Reset the generated lexer to the beginning of an input.



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 33

def lex(source, file: "(input)")
  @lexer_input = LexerInput.new(source)
  @lexer_file = file
  @lexer_states = ["INITIAL"]
  @lexer_line = 1
  @lexer_column = 1
  @lexer_byte_column = 1
  @lexer_byte_offset = 0
  @lexer_lexeme = nil
  @lexer_emission = NO_EMISSION
  @lexer_skip_requested = false
  @lexer_pending_green_trivia = []
  configure_lexer_cst
  @lexer_has_token = false
  self
end

#lexer_state ⇒ Object

Return the current named lexer state.



87
88
89
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 87

def lexer_state
  (@lexer_states || ["INITIAL"]).last.to_sym
end

#lexer_state=(state) ⇒ Object

Replace the current named lexer state.



93
94
95
96
97
98
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 93

def lexer_state=(state)
  name = validate_lexer_state(state)
  states = @lexer_states ||= ["INITIAL"]
  states[-1] = name
  name.to_sym
end

#next_token ⇒ Object

Pull one token using longest match and declaration-order tie breaking.

Raises:



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 102

def next_token
  input = @lexer_input
  raise ParseError, "(lexer):1:1: call parse or lex before next_token" unless input

  loop do
    unless ensure_lexer_data?(input)
      location = lexer_zero_width_location
      location[:ibex_lexer_start_state] = lexer_state
      location = attach_cst_trivia(location) if cst_enabled?
      return [nil, nil, location.freeze]
    end

    rule, lexeme = select_lexer_match(input)
    raise_lexer_no_match(input) unless rule && lexeme

    location = consume_lexer_match(input, lexeme)
    location[:ibex_lexer_start_state] = lexer_state
    emitted = apply_lexer_rule(rule, lexeme, location)
    return emitted if emitted
  end
end

#parse(source, file: "(input)") ⇒ Object

Lex and parse one String, IO, or Fiber source.



52
53
54
55
56
57
58
59
60
61
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 52

def parse(source, file: "(input)")
  lex(source, file: file)
  parser = self #: Parser
  parser.do_parse
rescue ParseError => e
  raise unless parser_tables[:cst].is_a?(Hash)

  parser = self #: Parser
  parser.__send__(:cst_lexical_failure, e)
end

#parse_syntax(source, file: "(input)") ⇒ Object

Parse without executing parser production actions. Lexer actions still run because they define tokenization and lexer state.



81
82
83
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 81

def parse_syntax(source, file: "(input)")
  parse_syntax_with_cache(CST::SourceText.new(source, file: file), CST::NodeCache.new)
end

#parse_with_syntax(source, file: "(input)") ⇒ Object

Parse one generated-lexer source and return its semantic and syntax results.



65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 65

def parse_with_syntax(source, file: "(input)")
  lex(source, file: file)
  parser = self #: Parser
  value = parser.do_parse
  parser.__send__(:syntax_parse_result, value)
rescue ParseError => e
  raise unless parser_tables[:cst].is_a?(Hash)

  parser = self #: Parser
  value = parser.__send__(:cst_lexical_failure, e)
  parser.__send__(:syntax_parse_result, value)
end