def scan_tokens encoder, options
state = :initial
last_token = ''
until eos?
if state == :initial
if match = scan(%r \s+ /)
encoder.text_token match, :space
next
elsif match = scan(%r \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !x)
encoder.text_token match, :preprocessor
next
elsif match = scan(%r // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !x)
encoder.text_token match, :comment
next
elsif match = scan(%r <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /)
encoder.text_token match, :operator
elsif match = scan(%r\./)
encoder.text_token match, :operator
next if last_token == 'end'
elsif match = scan(%r [A-Za-z_][A-Za-z_0-9]* /)
encoder.text_token match, NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
elsif match = skip(%r ' ( [^\n']|'' ) (?:'|$) /)
encoder.begin_group :char
encoder.text_token "'", :delimiter
encoder.text_token self[1], :content
encoder.text_token "'", :delimiter
encoder.end_group :char
next
elsif match = scan(%r ' /)
encoder.begin_group :string
encoder.text_token match, :delimiter
state = :string
elsif match = scan(%r \# (?: \d+ | \$[0-9A-Fa-f]+ ) /)
encoder.text_token match, :char
elsif match = scan(%r \$ [0-9A-Fa-f]+ /)
encoder.text_token match, :hex
elsif match = scan(%r (?: \d+ ) (?![eE]|\.[^.]) /)
encoder.text_token match, :integer
elsif match = scan(%r \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /)
encoder.text_token match, :float
else
encoder.text_token getch, :error
next
end
elsif state == :string
if match = scan(%r[^\n']+/)
encoder.text_token match, :content
elsif match = scan(%r''/)
encoder.text_token match, :char
elsif match = scan(%r'/)
encoder.text_token match, :delimiter
encoder.end_group :string
state = :initial
next
elsif match = scan(%r\n/)
encoder.end_group :string
encoder.text_token match, :space
state = :initial
else
raise "else case \' reached; %p not handled." % peek(1), encoder
end
else
raise 'else-case reached', encoder
end
last_token = match
end
if state == :string
encoder.end_group state
end
encoder
end