def scan_tokens encoder, options
state = :initial
key_indent = string_indent = 0
until eos?
key_indent = nil if bol?
if match = scan(%r +[\t ]*/)
encoder.text_token match, :space
elsif match = scan(%r\n+/)
encoder.text_token match, :space
state = :initial if match.index(\n\)
elsif match = scan(%r#.*/)
encoder.text_token match, :comment
elsif bol? and case
when match = scan(%r---|\.\.\./)
encoder.begin_group :head
encoder.text_token match, :head
encoder.end_group :head
next
when match = scan(%r%.*/)
encoder.text_token match, :doctype
next
end
elsif state == :value and case
when !check(%r(?:"[^"]*")(?=: |:$)/) && match = scan(%r"/)
encoder.begin_group :string
encoder.text_token match, :delimiter
encoder.text_token match, :content if match = scan(%r [^"\\]* (?: \\. [^"\\]* )* /x)
encoder.text_token match, :delimiter if match = scan(%r"/)
encoder.end_group :string
next
when match = scan(%r[|>][-+]?/)
encoder.begin_group :string
encoder.text_token match, :delimiter
string_indent = key_indent || column(pos - match.size) - 1
encoder.text_token matched, :content if scan(%r(?:\n+ {#{string_indent + 1}}.*)+/)
encoder.end_group :string
next
when match = scan(%r(?![!"*&]).+?(?=$|\s+#)/)
encoder.begin_group :string
encoder.text_token match, :content
string_indent = key_indent || column(pos - match.size) - 1
encoder.text_token matched, :content if scan(%r(?:\n+ {#{string_indent + 1}}.*)+/)
encoder.end_group :string
next
end
elsif case
when match = scan(%r[-:](?= |$)/)
state = :value if state == :colon && (match == ':' || match == '-')
state = :value if state == :initial && match == '-'
encoder.text_token match, :operator
next
when match = scan(%r[,{}\[\]]/)
encoder.text_token match, :operator
next
when state == :initial && match = scan(%r[-\w.()\/ ]*\S(?= *:(?: |$))/)
encoder.text_token match, :key
key_indent = column(pos - match.size) - 1
state = :colon
next
when match = scan(%r(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/)
encoder.begin_group :key
encoder.text_token match[0,1], :delimiter
encoder.text_token match[1..-2], :content
encoder.text_token match[-1,1], :delimiter
encoder.end_group :key
key_indent = column(pos - match.size) - 1
state = :colon
next
when match = scan(%r(![\w\/]+)(:([\w:]+))?/)
encoder.text_token self[1], :type
if self[2]
encoder.text_token ':', :operator
encoder.text_token self[3], :class
end
next
when match = scan(%r&\S+/)
encoder.text_token match, :variable
next
when match = scan(%r\*\w+/)
encoder.text_token match, :global_variable
next
when match = scan(%r<</)
encoder.text_token match, :class_variable
next
when match = scan(%r\d\d:\d\d:\d\d/)
encoder.text_token match, :octal
next
when match = scan(%r\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
encoder.text_token match, :octal
next
when match = scan(%r:\w+/)
encoder.text_token match, :symbol
next
when match = scan(%r[^:\s]+(:(?! |$)[^:\s]*)* .*/)
encoder.text_token match, :error
next
when match = scan(%r[^:\s]+(:(?! |$)[^:\s]*)*/)
encoder.text_token match, :error
next
end
else
raise if eos?
encoder.text_token getch, :error
end
end
encoder
end