Scanner

The base class for all Scanners.

It is a subclass of Ruby’s great StringScanner, which makes it easy to access the scanning methods inside.

It is also Enumerable, so you can use it like an Array of Tokens:

  require 'coderay'

  c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"

  for text, kind in c_scanner
    puts text if kind == :operator
  end

  # prints: (*==)++;

OK, this is a very simple example :) You can also use map, +any?+, find and even sort_by, if you want.

Methods

code=
column
each
file_extension
lang
line
marshal_dump
marshal_load
new
normify
reset
streamable?
streaming?
string=
tokenize
tokens

Included Modules

Enumerable

Constants

ScanError	=	Class.new(Exception)
	Raised if a Scanner fails while scanning
DEFAULT_OPTIONS	=	{ :stream => false }
	The default options for all scanner classes. Define @default_options for subclasses.
KINDS_NOT_LOC	=	[:comment, :doctype]

Public Class methods

file_extension(extension = nil)

→ show source

# File lib/coderay/scanner.rb, line 85
        def file_extension extension = nil
          if extension
            @file_extension = extension.to_s
          else
            @file_extension ||= plugin_id.to_s
          end
        end

new(code='', options = {})

Create a new Scanner.

code is the input String and is handled by the superclass StringScanner.
options is a Hash with Symbols as keys. It is merged with the default options of the class (you can overwrite default options here.)
block is the callback for streamed highlighting.

If you set :stream to true in the options, the Scanner uses a TokenStream with the block as callback to handle the tokens.

Else, a Tokens object is used.

→ show source

# File lib/coderay/scanner.rb, line 119
      def initialize code='', options = {}, &block
        raise "I am only the basic Scanner class. I can't scan "\
          "anything. :( Use my subclasses." if self.class == Scanner
        
        @options = self.class::DEFAULT_OPTIONS.merge options

        super Scanner.normify(code)

        @tokens = options[:tokens]
        if @options[:stream]
          warn "warning in CodeRay::Scanner.new: :stream is set, "\
            "but no block was given" unless block_given?
          raise NotStreamableError, self unless kind_of? Streamable
          @tokens ||= TokenStream.new(&block)
        else
          warn "warning in CodeRay::Scanner.new: Block given, "\
            "but :stream is #{@options[:stream]}" if block_given?
          @tokens ||= Tokens.new
        end
        @tokens.scanner = self

        setup
      end

normify(code)

→ show source

# File lib/coderay/scanner.rb, line 69
        def normify code
          code = code.to_s
          if code.respond_to? :force_encoding
            debug, $DEBUG = $DEBUG, false
            begin
              code.force_encoding 'utf-8'
              code[/\z/]  # raises an ArgumentError when code contains a non-UTF-8 char
            rescue ArgumentError
              code.force_encoding 'binary'
            ensure
              $DEBUG = debug
            end
          end
          code.to_unix
        end

streamable?()

Returns if the Scanner can be used in streaming mode.

→ show source

# File lib/coderay/scanner.rb, line 65
        def streamable?
          is_a? Streamable
        end

Public Instance methods

code=(code)

Alias for string=

column(pos = self.pos)

→ show source

# File lib/coderay/scanner.rb, line 202
      def column pos = self.pos
        return 0 if pos <= 0
        string = string()
        if string.respond_to?(:bytesize) && (defined?(@bin_string) || string.bytesize != string.size)
          @bin_string ||= string.dup.force_encoding('binary')
          string = @bin_string
        end
        pos - (string.rindex(?\n, pos) || 0)
      end

each(&block)

Traverses the tokens.

→ show source

# File lib/coderay/scanner.rb, line 187
      def each &block
        raise ArgumentError,
          'Cannot traverse TokenStream.' if @options[:stream]
        tokens.each(&block)
      end

lang()

Returns the Plugin ID for this scanner.

→ show source

# File lib/coderay/scanner.rb, line 159
      def lang
        self.class.plugin_id
      end

line()

The current line position of the scanner.

Beware, this is implemented inefficiently. It should be used for debugging only.

→ show source

# File lib/coderay/scanner.rb, line 198
      def line
        string[0..pos].count("\n") + 1
      end

marshal_dump()

→ show source

# File lib/coderay/scanner.rb, line 212
      def marshal_dump
        @options
      end

marshal_load(options)

→ show source

# File lib/coderay/scanner.rb, line 216
      def marshal_load options
        @options = options
      end

reset()

→ show source

# File lib/coderay/scanner.rb, line 143
      def reset
        super
        reset_instance
      end

streaming?()

Whether the scanner is in streaming mode.

→ show source

# File lib/coderay/scanner.rb, line 182
      def streaming?
        !!@options[:stream]
      end

string=(code)

--- This method is also aliased as code= ---

→ show source

# File lib/coderay/scanner.rb, line 148
      def string= code
        code = Scanner.normify(code)
        super code
        reset_instance
      end

tokenize(new_string=nil, options = {})

Scans the code and returns all tokens in a Tokens object.

→ show source

# File lib/coderay/scanner.rb, line 164
      def tokenize new_string=nil, options = {}
        options = @options.merge(options)
        self.string = new_string if new_string
        @cached_tokens =
          if @options[:stream]  # :stream must have been set already
            reset unless new_string
            scan_tokens @tokens, options
            @tokens
          else
            scan_tokens @tokens, options
          end
      end

tokens()

→ show source

# File lib/coderay/scanner.rb, line 177
      def tokens
        @cached_tokens ||= tokenize
      end