def scan_tokens encoder, options
match = nil
code = ''
until eos?
if bol?
if match = scan(%r!!!.*/)
encoder.text_token match, :doctype
next
end
if match = scan(%r(?>( *)(\/(?!\[if)|-\#|:javascript|:ruby|:\w+) *)(?=\n)/)
encoder.text_token match, :comment
code = self[2]
if match = scan(%r(?:\n+#{self[1]} .*)+/)
case code
when '/', '-#'
encoder.text_token match, :comment
when ':javascript'
@java_script_scanner ||= CodeRay.scanner :java_script, :tokens => @tokens, :keep_tokens => true
@java_script_scanner.tokenize match, :tokens => encoder
when ':ruby'
@ruby_scanner.tokenize match, :tokens => encoder
when %r:\w+/
encoder.text_token match, :comment
else
raise 'else-case reached: %p' % [code]
end
end
end
if match = scan(%r +/)
encoder.text_token match, :space
end
if match = scan(%r\/.*/)
encoder.text_token match, :comment
next
end
if match = scan(%r\\/)
encoder.text_token match, :plain
if match = scan(%r.+/)
@html_scanner.tokenize match, :tokens => encoder
end
next
end
tag = false
if match = scan(%r%[\w:]+\/?/)
encoder.text_token match, :tag
tag = true
end
while match = scan(%r([.#])[-\w]*\w/)
encoder.text_token match, self[1] == '#' ? :constant : :class
tag = true
end
if tag && match = scan(%r(\()([^)]+)?(\))?/)
encoder.text_token self[1], :plain
@html_scanner.tokenize self[2], :tokens => encoder, :state => :attribute if self[2]
encoder.text_token self[3], :plain if self[3]
end
if tag && match = scan(%r\{/)
encoder.text_token match, :plain
code = ''
level = 1
while true
code << scan(%r([^\{\},\n]|, *\n?)*/)
case match = getch
when '{'
level += 1
code << match
when '}'
level -= 1
if level > 0
code << match
else
break
end
when "\n", ",", nil
break
end
end
@ruby_scanner.tokenize code, :tokens => encoder unless code.empty?
encoder.text_token match, :plain if match
end
if tag && match = scan(%r(\[)([^\]\n]+)?(\])?/)
encoder.text_token self[1], :plain
@ruby_scanner.tokenize self[2], :tokens => encoder if self[2]
encoder.text_token self[3], :plain if self[3]
end
if tag && match = scan(%r\//)
encoder.text_token match, :tag
end
if scan(%r(>?<?[-=]|[&!]=|(& |!)|~)( *)([^,\n\|]+(?:(, *|\|(?=.|\n.*\|$))\n?[^,\n\|]*)*)?/)
encoder.text_token self[1] + self[3], :plain
if self[4]
if self[2]
@embedded_ruby_scanner.tokenize self[4], :tokens => encoder
else
@ruby_scanner.tokenize self[4], :tokens => encoder
end
end
elsif match = scan(%r((?:<|><?)(?![!?\/\w]))?(.+)?/)
encoder.text_token self[1], :plain if self[1]
@html_scanner.tokenize self[2], :tokens => encoder if self[2]
end
elsif match = scan(%r.+/)
@html_scanner.tokenize match, :tokens => encoder
end
if match = scan(%r\n/)
encoder.text_token match, :space
end
end
encoder
end