# File lib/parser/base.rb, line 204
    def tokenize(source_buffer, recover=false)
      @lexer.tokens = []
      @lexer.comments = []

      begin
        ast = parse(source_buffer)
      rescue Parser::SyntaxError
        raise if !recover
      end

      [ ast, @lexer.comments, @lexer.tokens ]
    ensure
      @lexer.tokens = nil
      @lexer.comments = nil
    end