def scan_tokens encoder, options
states = Array(options[:state] || @state)
value_expected = @value_expected
until eos?
if match = scan(%r\s+/)
encoder.text_token match, :space
elsif case states.last
when :initial, :media
if match = scan(%r(?>#{RE::Ident})(?!\()|\*/x)
encoder.text_token match, :type
next
elsif match = scan(RE::Class)
encoder.text_token match, :class
next
elsif match = scan(RE::Id)
encoder.text_token match, :constant
next
elsif match = scan(RE::PseudoClass)
encoder.text_token match, :pseudo_class
next
elsif match = scan(RE::AttributeSelector)
encoder.text_token match[0,1], :operator
encoder.text_token match[1..-2], :attribute_name if match.size > 2
encoder.text_token match[-1,1], :operator if match[-1] == ]]
next
elsif match = scan(%r@media/)
encoder.text_token match, :directive
states.push :media_before_name
next
end
when :block
if match = scan(%r(?>#{RE::Ident})(?!\()/x)
if value_expected
encoder.text_token match, :value
else
encoder.text_token match, :key
end
next
end
when :media_before_name
if match = scan(RE::Ident)
encoder.text_token match, :type
states[-1] = :media_after_name
next
end
when :media_after_name
if match = scan(%r\{/)
encoder.text_token match, :operator
states[-1] = :media
next
end
else
raise_inspect 'Unknown state', encoder
end
elsif match = scan(%r\/\*(?:.*?\*\/|\z)/)
encoder.text_token match, :comment
elsif match = scan(%r\{/)
value_expected = false
encoder.text_token match, :operator
states.push :block
elsif match = scan(%r\}/)
value_expected = false
encoder.text_token match, :operator
if states.last == :block || states.last == :media
states.pop
end
elsif match = scan(%r#{RE::String}/)
encoder.begin_group :string
encoder.text_token match[0, 1], :delimiter
encoder.text_token match[1..-2], :content if match.size > 2
encoder.text_token match[-1, 1], :delimiter if match.size >= 2
encoder.end_group :string
elsif match = scan(%r#{RE::Function}/)
encoder.begin_group :function
start = match[%r^\w+\(/]
encoder.text_token start, :delimiter
if match[-1] == ))
encoder.text_token match[start.size..-2], :content
encoder.text_token ')', :delimiter
else
encoder.text_token match[start.size..-1], :content
end
encoder.end_group :function
elsif match = scan(%r(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/x)
encoder.text_token match, :float
elsif match = scan(%r#{RE::Color}/)
encoder.text_token match, :color
elsif match = scan(%r! *important/)
encoder.text_token match, :important
elsif match = scan(%r(?:rgb|hsl)a?\([^()\n]*\)?/)
encoder.text_token match, :color
elsif match = scan(RE::AtKeyword)
encoder.text_token match, :directive
elsif match = scan(%r [+>:;,.=()\/] /)
if match == ':'
value_expected = true
elsif match == ';'
value_expected = false
end
encoder.text_token match, :operator
else
encoder.text_token getch, :error
end
end
if options[:keep_state]
@state = states
@value_expected = value_expected
end
encoder
end