/* 
 * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; version 2 of the
 * License.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301  USA
 */

#include <sstream>
#include <algorithm>
#include <string>

#include "base/log.h"
#include "base/string_utilities.h"

#include "mysql-parser.h"

DEFAULT_LOG_DOMAIN("MySQL parser")

extern "C" {
  
  /**
   * Error report function which is set in the parser (see MySQL.g where this is done). The default
   * reports to stderr, while want it here in our recognizer (and customized).
   */
  void on_parse_error(struct ANTLR3_BASE_RECOGNIZER_struct *recognizer, pANTLR3_UINT8 *tokenNames)
  {
    std::ostringstream error;
  
	  pANTLR3_PARSER parser;
	  pANTLR3_TREE_PARSER tparser;
	  pANTLR3_INT_STREAM is;
	  pANTLR3_STRING ttext;
	  pANTLR3_STRING ftext;
	  pANTLR3_EXCEPTION ex;
	  pANTLR3_COMMON_TOKEN theToken;
	  pANTLR3_BASE_TREE theBaseTree;
	  pANTLR3_COMMON_TREE theCommonTree;

    // TODO: the generated message can be simplified, now that we also return token info.
  
	  // Retrieve some info for easy reading.
	  ex = recognizer->state->exception;
	  ttext = NULL;

	  // See if there is a 'filename' we can use
	  //
	  if	(ex->streamName == NULL)
	  {
		  if	(((pANTLR3_COMMON_TOKEN)(ex->token))->type == ANTLR3_TOKEN_EOF)
			  error << "-end of input-(";
		  else
			  error << "-unknown source-(";
	  }
	  else
	  {
		  ftext = ex->streamName->to8(ex->streamName);
		  error << ftext->chars << "(";
	  }

	  // Next comes the line number.
	  error << recognizer->state->exception->line << ") ";
	  error << " : error " << recognizer->state->exception->type << " : " <<	(pANTLR3_UINT8)(recognizer->state->exception->message);

	  // How we determine the next piece is dependent on which thing raised the error.
	  switch	(recognizer->type)
	  {
	  case	ANTLR3_TYPE_PARSER:
		  // Prepare the knowledge we know we have.
		  parser = (pANTLR3_PARSER) (recognizer->super);
		  tparser = NULL;
		  is = parser->tstream->istream;
		  theToken = (pANTLR3_COMMON_TOKEN)(recognizer->state->exception->token);
		  ttext = theToken->toString(theToken);

		  error << ", at offset " << recognizer->state->exception->charPositionInLine;
		  if  (theToken != NULL)
		  {
			  if (theToken->type == ANTLR3_TOKEN_EOF)
				  error << ", at <EOF>";
			  else
				  error << "\n    near " << (ttext == NULL ? (pANTLR3_UINT8)"<no text for the token>" : ttext->chars) << "\n    ";
		  }
		  break;

	  case	ANTLR3_TYPE_TREE_PARSER:
		  tparser		= (pANTLR3_TREE_PARSER) (recognizer->super);
		  parser		= NULL;
		  is			= tparser->ctnstream->tnstream->istream;
		  theBaseTree	= (pANTLR3_BASE_TREE)(recognizer->state->exception->token);
		  ttext		= theBaseTree->toStringTree(theBaseTree);

		  if  (theBaseTree != NULL)
		  {
			  theCommonTree	= (pANTLR3_COMMON_TREE)	    theBaseTree->super;

			  if	(theCommonTree != NULL)
				  theToken	= (pANTLR3_COMMON_TOKEN)theBaseTree->getToken(theBaseTree);
			  error << ", at offset " << theBaseTree->getCharPositionInLine(theBaseTree) << ", near " << ttext->chars;
		  }
		  break;

	  default:
		  error << "Internal error: unknown recognizer type appeared in error reporting.\n";
		  return;
		  break;
	  }

	  // Although this function should generally be provided by the implementation, this one
	  // should be as helpful as possible for grammar developers and serve as an example
	  // of what you can do with each exception type. In general, when you make up your
	  // 'real' handler, you should debug the routine with all possible errors you expect
	  // which will then let you be as specific as possible about all circumstances.
	  //
	  // Note that in the general case, errors thrown by tree parsers indicate a problem
	  // with the output of the parser or with the tree grammar itself. The job of the parser
	  // is to produce a perfect (in traversal terms) syntactically correct tree, so errors
	  // at that stage should really be semantic errors that your own code determines and handles
	  // in whatever way is appropriate.
	  switch  (ex->type)
	  {
	  case	ANTLR3_UNWANTED_TOKEN_EXCEPTION:

		  // Indicates that the recognizer was fed a token which seesm to be
		  // spurious input. We can detect this when the token that follows
		  // this unwanted token would normally be part of the syntactically
		  // correct stream. Then we can see that the token we are looking at
		  // is just something that should not be there and throw this exception.
		  //
		  if	(tokenNames == NULL)
			  error << " : Extraneous input...";
		  else
		  {
			  if	(ex->expecting == ANTLR3_TOKEN_EOF)
				  error << " : Extraneous input - expected <EOF>\n";
			  else
				  error << " : Extraneous input - expected " << tokenNames[ex->expecting] <<  "...\n";
		  }
		  break;

	  case	ANTLR3_MISSING_TOKEN_EXCEPTION:

		  // Indicates that the recognizer detected that the token we just
		  // hit would be valid syntactically if preceeded by a particular 
		  // token. Perhaps a missing ';' at line end or a missing ',' in an
		  // expression list, and such like.
		  //
		  if	(tokenNames == NULL)
		  {
			  error << " : Missing token (" << ex->expecting << ")...\n";
		  }
		  else
		  {
			  if	(ex->expecting == ANTLR3_TOKEN_EOF)
				  error << " : Missing <EOF>\n";
			  else
				  error << " : Missing " << tokenNames[ex->expecting] << " \n";
		  }
		  break;

	  case	ANTLR3_RECOGNITION_EXCEPTION:

		  // Indicates that the recognizer received a token
		  // in the input that was not predicted. This is the basic exception type 
		  // from which all others are derived. So we assume it was a syntax error.
		  // You may get this if there are not more tokens and more are needed
		  // to complete a parse for instance.
		  error << " : syntax error...\n";    
		  break;

	  case    ANTLR3_MISMATCHED_TOKEN_EXCEPTION:

		  // We were expecting to see one thing and got another. This is the
		  // most common error if we coudl not detect a missing or unwanted token.
		  // Here you can spend your efforts to
		  // derive more useful error messages based on the expected
		  // token set and the last token and so on. The error following
		  // bitmaps do a good job of reducing the set that we were looking
		  // for down to something small. Knowing what you are parsing may be
		  // able to allow you to be even more specific about an error.
		  if	(tokenNames == NULL)
			  error << " : syntax error...\n";
		  else
		  {
			  if	(ex->expecting == ANTLR3_TOKEN_EOF)
				  error << " : expected <EOF>\n";
			  else
				  error << " : expected " << tokenNames[ex->expecting] << " ...\n";
		  }
		  break;

	  case	ANTLR3_NO_VIABLE_ALT_EXCEPTION:

		  // We could not pick any alt decision from the input given
		  // so god knows what happened - however when you examine your grammar,
		  // you should. It means that at the point where the current token occurred
		  // that the DFA indicates nowhere to go from here.
		  error << " : cannot match to any predicted input...\n";

		  break;

	  case	ANTLR3_MISMATCHED_SET_EXCEPTION:

		  {
			  ANTLR3_UINT32	  count;
			  ANTLR3_UINT32	  bit;
			  ANTLR3_UINT32	  size;
			  ANTLR3_UINT32	  numbits;
			  pANTLR3_BITSET	errBits;

			  // This means we were able to deal with one of a set of
			  // possible tokens at this point, but we did not see any
			  // member of that set.
			  error << " : unexpected input...\n  expected one of : ";

			  // What tokens could we have accepted at this point in the parse?
			  count   = 0;
			  errBits = antlr3BitsetLoad(ex->expectingSet);
			  numbits = errBits->numBits(errBits);
			  size    = errBits->size(errBits);

			  if  (size > 0)
			  {
				  // However many tokens we could have dealt with here, it is usually
				  // not useful to print ALL of the set here. I arbitrarily chose 8
				  // here, but you should do whatever makes sense for you of course.
				  // No token number 0, so look for bit 1 and on.
				  for	(bit = 1; bit < numbits && count < 8 && count < size; bit++)
				  {
					  // TODO: This doesn;t look right - should be asking if the bit is set!!
					  if  (tokenNames[bit])
					  {
						  error << (count > 0 ? ", " : "") << tokenNames[bit]; 
						  count++;
					  }
				  }
				  error << "\n";
			  }
			  else
			  {
				  error << "Actually dude, we didn't seem to be expecting anything here, or at least\n";
				  error << "I could not work out what I was expecting, like so many of us these days!\n";
			  }
		  }
		  break;

	  case	ANTLR3_EARLY_EXIT_EXCEPTION:

		  // We entered a loop requiring a number of token sequences
		  // but found a token that ended that sequence earlier than
		  // we should have done.
		  error << " : missing elements...\n";
		  break;

	  default:

		  // We don't handle any other exceptions here, but you can
		  // if you wish. If we get an exception that hits this point
		  // then we are just going to report what we know about the
		  // token.
		  error << " : syntax not recognized...\n";
		  break;
	  }

    int length = 0;
    if (theToken != NULL)
      length = (int)theToken->stop - (int)theToken->start + 1;
    MySQLRecognizer *recognizer_class = (MySQLRecognizer*)((RecognitionContext*)recognizer->state->userp)->payload;
    if (ex == NULL)
      recognizer_class->add_error(error.str(), 0, 0, 0, 0, 0);
    else
      recognizer_class->add_error(error.str(), ex->type, (theToken == NULL) ? 0 : theToken->type,
                                  ex->line, ex->charPositionInLine, length);
  }

  /**
   * Checks the given identifier whether it is an actually defined charset, which directs
   * the lexer to either classify it appropriately.
   */
  ANTLR3_UINT32 check_charset(void *payload, pANTLR3_STRING text)
  {
    // Get the actual token text and skip the initial underscore char.
    // There's an additional char at the end of the input for some reason (maybe a lexer bug),
    // so we also ignore the last char.
    MySQLRecognizer *recognizer = (MySQLRecognizer*)payload;

    std::string token_text((const char*)text->chars + 1, text->len - 2);
    if (recognizer->is_charset(base::tolower(token_text)))
      return UNDERSCORE_CHARSET;
    return IDENTIFIER;
  }
  
  /**
   * Checks the given text if it is equal to "\N" (w/o quotes and in uppercase). We need this extra
   * check as our lexer is case insensitive.
   */
  ANTLR3_UINT32 check_null(pANTLR3_STRING text)
  {
    std::string token_text((const char*)text->chars, text->len - 1);
    if (token_text == "\\N")
      return NULL2_SYMBOL;
    return ANTLR3_TOKEN_INVALID;
  }

} // extern "C"

//----------------- MySQLTreeWalker ----------------------------------------------------------------

struct compare_token_index
{
  inline bool operator() (const pANTLR3_BASE_TREE left, const pANTLR3_BASE_TREE right)
  {
    pANTLR3_COMMON_TOKEN t1 = left->getToken(left);
    pANTLR3_COMMON_TOKEN t2 = right->getToken(right);
    return t1->index < t2->index;
  }
};

MySQLRecognizerTreeWalker::MySQLRecognizerTreeWalker(MySQLRecognizer *recognizer, pANTLR3_BASE_TREE tree)
{
  _recognizer = recognizer;
  _tree = tree;
  _origin = tree;

  // Fill the list of tokens for quick lookup by type or position in the correct order.
  pANTLR3_BASE_TREE run = _tree;
  while (run != NULL)
  {
    // Add only entries that carry useful information for position search.
    pANTLR3_COMMON_TOKEN token = run->getToken(run);
    if (token != NULL && token->index > 0)
      _token_list.push_back(run);
    run = get_next(run, true);
  }

  // Sort token list by token index, which puts them in appearance order.
  if (_token_list.size() > 2)
    std::sort(_token_list.begin(), _token_list.end(), compare_token_index());
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns the next node after the given one without changing any internal state or NULL if there's
 * no next node. The recurse flag determines if we can change tree levels or stay in the one we are in
 * currently.
 */
pANTLR3_BASE_TREE MySQLRecognizerTreeWalker::get_next(pANTLR3_BASE_TREE node, bool recurse)
{
  if (recurse)
  {
    // If there are child take the first one.
    if (node->getChildCount(node) > 0)
      return (pANTLR3_BASE_TREE)_tree->getChild(node, 0);
  }
  
  // No child nodes (or no recursion), so see if there is another sibling of this node or one of its parents.
  while (true)
  { 
    pANTLR3_BASE_TREE parent = node->getParent(node);
    if (parent == NULL)
      return NULL;
  
    int index = parent->getChildIndex(node) + 1;
    if ((int)parent->getChildCount(parent) > index)
      return (pANTLR3_BASE_TREE)parent->getChild(parent, index);
    
    if (!recurse)
      return NULL;
    
    // No sibling either - go up one level and try the next sibling of the parent.
    node = parent;
  }
  return NULL;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns the previous node before the given one without changing any internal state or NULL if there's
 * no previous node. The meaning of the recurse flag is the same as for get_next().
 */
pANTLR3_BASE_TREE MySQLRecognizerTreeWalker::get_previous(pANTLR3_BASE_TREE node, bool recurse)
{
  pANTLR3_BASE_TREE parent = _tree->getParent(_tree);
  if (parent == NULL)
    return NULL;
  
  int index = parent->getChildIndex(_tree) - 1;
  if (index < 0)
  {
    if (!recurse)
      return NULL;
    return parent;
  }
  
  pANTLR3_BASE_TREE last_node = (pANTLR3_BASE_TREE)parent->getChild(parent, index);
  if (recurse)
  {
    while (last_node->getChildCount(last_node) > 0)
    {
      // Walk down the entire tree hierarchy to the last sub node of the previous sibling.
      index = last_node->getChildCount(last_node) - 1;
      last_node = (pANTLR3_BASE_TREE)last_node->getChild(last_node, index);
    }
  }
  
  return last_node;
}

//--------------------------------------------------------------------------------------------------

void MySQLRecognizerTreeWalker::print_token(pANTLR3_BASE_TREE tree)
{
  pANTLR3_STRING token_text = tree->getText(tree);
  printf("Token: %s\n", (token_text == NULL) ? "nil" : (char*)token_text->chars);
}

//--------------------------------------------------------------------------------------------------

/**
 * Advances to the next token. If this is a subtree the next token is the first child node,
 * otherwise the next sibling node is used. If there is no sibling node then the next sibling of
 * the parent is used, if there's one, and so on.
 *
 * @return True if there was a next node, false otherwise. No change in the state is performed if
 *         there was no next node.
 */
bool MySQLRecognizerTreeWalker::next()
{
  pANTLR3_BASE_TREE node = get_next(_tree, true);
  if (node != NULL)
  {
    _tree = node;
    return true;
  }
  return false;
}

//--------------------------------------------------------------------------------------------------

/**
 * Advances to the next sibling token, ignoring any child nodes (if there are any).
 *
 * @return True if there was a next sibling node, false otherwise (no change performed then).
 */
bool MySQLRecognizerTreeWalker::next_sibling()
{
  pANTLR3_BASE_TREE node = get_next(_tree, false);
  if (node != NULL)
  {
    _tree = node;
    return true;
  }
  return false;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns to the previous token. If this is the first child in a sub tree then the parent node
 * is used, otherwise the previous sibling. If the previous sibling has children then the last child
 * of it is used instead. If this also has children the last grand child is used and so on.
 *
 * @return True if there was a previous node, false otherwise (no change performed then).
 */
bool MySQLRecognizerTreeWalker::previous()
{
  pANTLR3_BASE_TREE node = get_previous(_tree, true);
  if (node == NULL)
    return false;
  
  _tree = node;
  
  return true;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns to the previous token in index order, which might be different to the tree order.
 * The index order is the order by which tokens appear in the text. Index ordering is not continuous
 * as some tokens are hidden.
 *
 * @return True if there is a previous node, false otherwise (no change performed then).
 */
bool MySQLRecognizerTreeWalker::previous_by_index()
{
  if (_tree == NULL)
    return false;

  pANTLR3_BASE_TREE run = _tree;
  do 
  {
    pANTLR3_COMMON_TOKEN token = run->getToken(run);
    if (token->index > 0)
      break;
    run = get_previous(run, true);
  } while (true);
  
  std::vector<pANTLR3_BASE_TREE>::const_iterator iterator =
    lower_bound(_token_list.begin(), _token_list.end(), run, compare_token_index());
  if (iterator == _token_list.end())
    return false;

  if (iterator == _token_list.begin())
    return false;

  _tree = *(--iterator);
  
  return true;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns to the previous sibling token, ignoring any child nodes of that sibling. If the current
 * node is the first child in a subtree then the function fails.
 *
 * @return True if there was a previous node, false otherwise (no change performed then).
 */
bool MySQLRecognizerTreeWalker::previous_sibling()
{
  pANTLR3_BASE_TREE node = get_previous(_tree, false);
  if (node == NULL)
    return false;
  
  _tree = node;
  
  return true;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns to the parent token if the current one is a child.
 *
 * @return True if there was a parent node, false otherwise (no change performed then).
 */
bool MySQLRecognizerTreeWalker::up()
{
  pANTLR3_BASE_TREE parent = _tree->getParent(_tree);
  if (parent == NULL)
    return false;
  
  _tree = parent;
  
  return true;
}

//--------------------------------------------------------------------------------------------------

/**
 * Advances to the token that covers the given line and char offset. The line number is one-based
 * while the character offset is zero-based. This search only considers real tokens.
 *
 * Note: if the given position is between two tokens then the first one is used.
 *
 * @return True if such a node exists, false otherwise (no change performed then).
 */
bool MySQLRecognizerTreeWalker::advance_to_position(int line, int offset)
{
  if (_token_list.size() == 0)
    return false;

  pANTLR3_BASE_TREE current = _tree;
  for (size_t i = 0; i < _token_list.size(); i++)
  {
    pANTLR3_BASE_TREE run = _token_list[i];
    ANTLR3_UINT32 token_line = run->getLine(run);
    if ((int)token_line >= line)
    {
      ANTLR3_UINT32 token_offset = run->getCharPositionInLine(run);
      if ((int)token_line > line || (int)token_offset > offset)
      {
        // We reached a token after the current offset. Take the previous one as result then.
        _tree = (i > 0) ? _token_list[i - 1] : _origin;
        break;
      }
    }
  }

  // If we are still at the same token then see if the given position is beyond the last position.
  if (current == _tree)
  {
    pANTLR3_BASE_TREE run = _token_list[_token_list.size() - 1];
    ANTLR3_UINT32 token_line = run->getLine(run);
    if (line > (int)token_line)
      _tree = run;
    else
    {
      ANTLR3_UINT32 token_offset = run->getCharPositionInLine(run);
      pANTLR3_COMMON_TOKEN token = run->getToken(run);
      if (line < (int)token_line || offset >= (int)token_offset + ((int) token->stop - (int) token->start + 1))
        _tree = run;
    }
  }

  return (current != _tree) && !_tree->isNilNode(_tree);
}

//--------------------------------------------------------------------------------------------------

/**
 * Advances to the next token with the given type. The parameter type can be any token id listed in
 * MySQLParser.h (search for "Symbolic definitions"). It can be either a simple lexical token
 * like DIV_OPERATOR or SELECT_SYMBOL, a more complex lexical token like IDENTIFIER or a parser token like 
 * GROUP_BY or MISSING_ID. Tokens ending with _SYMBOL are text literals (e.g. "SELECT", "DIV" etc.)
 *
 * @param type The token type to search.
 * @param recurse If false search only siblings, otherwise any node in any tree level.
 * @return True if such a node exists, false otherwise (no change performed then).
 */
bool MySQLRecognizerTreeWalker::advance_to_type(unsigned int type, bool recurse)
{
  pANTLR3_BASE_TREE run = _tree;
  while (true)
  {
    run = get_next(run, recurse);
    if (run == NULL)
      return false;
    
    if (run->getType(run) == type)
    {
      _tree = run;
      return true;
    }
  }
  
  return false;
}

//--------------------------------------------------------------------------------------------------

/**
 * Steps over a number of tokens and positions the walker at the first token after the last one.
 * The tokens must all be siblings of the current token and are traversed in exactly the given order
 * without intermediate tokens. The current token must be start_token.
 *
 * Note: the list must be terminated by ANTLR3_TOKEN_INVALID (or 0).
 *
 * @return True if all the given tokens were found and there is another sibling after the last token
 *         in the list, false otherwise. If the token sequence could not be found or there is no more
 *         token the internal state is undefined.
 */
bool MySQLRecognizerTreeWalker::skip_token_sequence(unsigned int start_token, ...)
{
  bool result = false;
  
  unsigned int token = start_token;
  va_list tokens;
  va_start(tokens, start_token);
  while (true)
  {
    if (token_type() != token)
      break;
    
    if (!next_sibling())
      break;

    token = va_arg(tokens, unsigned int);
    if (token == ANTLR3_TOKEN_INVALID)
    {
      result = true;
      break;
    }
  }
  va_end(tokens);
  
  return result;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns the type of the next sibling (if recursive is false) or the next token (including child
 * nodes) without changing the internal state.
 */
unsigned int MySQLRecognizerTreeWalker::look_ahead(bool recursive)
{
  pANTLR3_BASE_TREE next = get_next(_tree, recursive);
  if (next == NULL)
    return ANTLR3_TOKEN_INVALID;
  return next->getType(next);
}

//--------------------------------------------------------------------------------------------------

/**
 * Resets the walker to be at the original location.
 */
void MySQLRecognizerTreeWalker::reset()
{
  _tree = _origin;
  while (!_token_stack.empty())
    _token_stack.pop();
}

//--------------------------------------------------------------------------------------------------

/**
 * Store the current node on the stack, so we can easily come back when needed.
 */
void MySQLRecognizerTreeWalker::push()
{
  _token_stack.push(_tree);
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns to the location at the top of the token stack (if any).
 */
bool MySQLRecognizerTreeWalker::pop()
{
  if (_token_stack.empty())
    return false;
  
  _tree = _token_stack.top();
  _token_stack.pop();
  return true;
}

//--------------------------------------------------------------------------------------------------

/**
 * Removes the current top of stack entry without restoring the internal state.
 * Does nothing if the stack is empty.
 */
void MySQLRecognizerTreeWalker::remove_tos()
{
  if (!_token_stack.empty())
    _token_stack.pop();
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns true if the current token is empty, false otherwise.
 */
bool MySQLRecognizerTreeWalker::is_nil()
{
  return _tree->isNilNode(_tree) == ANTLR3_TRUE;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns true if the current token is the root of a subtree (i.e. has child nodes).
 */
bool MySQLRecognizerTreeWalker::is_subtree()
{
  return _tree->getChildCount(_tree) > 0;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns true if the current token is an identifier.
 */
bool MySQLRecognizerTreeWalker::is_identifier()
{
  return is_identifier(_tree->getType(_tree));
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns true if the given token is an identifier.
 */
bool MySQLRecognizerTreeWalker::is_identifier(unsigned int type)
{
  bool result = (type == IDENTIFIER) || (type == BACK_TICK_QUOTED_ID);
  if (!result)
  {
    // Double quoted text represents identifiers only if the ANSI QUOTES sql mode is active.
    result = ((_recognizer->sql_mode() & SQL_MODE_ANSI_QUOTES) != 0) && (type == DOUBLE_QUOTED_TEXT);

  }
  return result;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns true if the current token is an identifier.
 */
bool MySQLRecognizerTreeWalker::is_keyword()
{
  return is_keyword(_tree->getType(_tree));
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns true if the given token is an identifier.
 */
bool MySQLRecognizerTreeWalker::is_keyword(unsigned int type)
{
  if (ACCESSIBLE_SYMBOL <= type && type <= ZEROFILL_SYMBOL)
  {
    switch (type)
    {
    case AT_SIGN_SYMBOL:
    case AT_AT_SIGN_SYMBOL:
    case BACK_TICK:
    case BACK_TICK_QUOTED_ID:
    case BITNUMBER:
    case BITSTRING:
    case BITWISE_AND_OPERATOR:
    case BITWISE_NOT_OPERATOR:
    case BITWISE_OR_OPERATOR:
    case BITWISE_XOR_OPERATOR:
    case CLOSE_PAR_SYMBOL:
    case COLON_SYMBOL:
    case COMMA_SYMBOL:
    case DASHDASH_COMMENT:
    case DIGIT:
    case DIGITS:
    case DIV_OPERATOR:
    case DOT_SYMBOL:
    case DOUBLE_QUOTE:
    case DOUBLE_QUOTED_TEXT:
    case EQUAL_OPERATOR:
    case ESCAPE_OPERATOR:
    case EXPRESSION:
    case FIELD_REF_ID:
    case FLOAT:
    case FUNCTION_CALL:
    case GREATER_OR_EQUAL_OPERATOR:
    case GREATER_THAN_OPERATOR:
    case HEXDIGIT:
    case HEXNUMBER:
    case HEXSTRING:
    case IDENTIFIER:
    case INDEX_HINT_LIST:
    case INTEGER:
    case JOIN:
    case LESS_OR_EQUAL_OPERATOR:
    case LESS_THAN_OPERATOR:
    case LETTER_WHEN_UNQUOTED:
    case LOGICAL_AND_OPERATOR:
    case LOGICAL_NOT_OPERATOR:
    case LOGICAL_OR_OPERATOR:
    case MINUS_OPERATOR:
    case ML_COMMENT_END:
    case ML_COMMENT_HEAD:
    case MOD_OPERATOR:
    case MULT_OPERATOR:
    case NCHAR_TEXT:
    case NOT_EQUAL2_OPERATOR:
    case NOT_EQUAL_OPERATOR:
    case NULL2_SYMBOL:
    case NULL_SAFE_EQUAL_OPERATOR:
    case OPEN_PAR_SYMBOL:
    case PARAM_MARKER:
    case PAR_EXPRESSION:
    case PLUS_OPERATOR:
    case POUND_COMMENT:
    case SEMICOLON_SYMBOL:
    case SHIFT_LEFT_OPERATOR:
    case SHIFT_RIGHT_OPERATOR:
    case SINGLE_QUOTE:
    case SINGLE_QUOTED_TEXT:
    case STRING:
    case SUBQUERY:
    case TABLE_REFERENCE:
    case TABLE_REF_ID:
    case UNDERLINE_SYMBOL:
    case UNDERSCORE_CHARSET:
    case VERSION_COMMENT:
    case VERSION_COMMENT_END:
    case VERSION_COMMENT_INTRODUCER:
    case VERSION_COMMENT_START:
    case VERSION_COMMENT_TAIL:
    case WS:
    case XA_ID:
      return false;

    default:
      return true;
    }
  }
  return false;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns true if the current token is a relation specifier (operator, math symbol etc).
 */
bool MySQLRecognizerTreeWalker::is_relation()
{
  return is_relation(_tree->getType(_tree));
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns true if the given token is an identifier.
 */
bool MySQLRecognizerTreeWalker::is_relation(unsigned int type)
{
  switch (type)
  {
    case EQUAL_OPERATOR:
    case ASSIGN_OPERATOR:
    case NULL_SAFE_EQUAL_OPERATOR:
    case GREATER_OR_EQUAL_OPERATOR:
    case GREATER_THAN_OPERATOR:
    case LESS_OR_EQUAL_OPERATOR:
    case LESS_THAN_OPERATOR:
    case NOT_EQUAL_OPERATOR:
    case NOT_EQUAL2_OPERATOR:
    case PLUS_OPERATOR:
    case MINUS_OPERATOR:
    case MULT_OPERATOR:
    case DIV_OPERATOR:
    case MOD_OPERATOR:
    case LOGICAL_NOT_OPERATOR:
    case BITWISE_NOT_OPERATOR:
    case SHIFT_LEFT_OPERATOR:
    case SHIFT_RIGHT_OPERATOR:
    case LOGICAL_AND_OPERATOR:
    case BITWISE_AND_OPERATOR:
    case BITWISE_XOR_OPERATOR:
    case LOGICAL_OR_OPERATOR:
    case BITWISE_OR_OPERATOR:

    case OR_SYMBOL:
    case XOR_SYMBOL:
    case AND_SYMBOL:
    case IS_SYMBOL:
    case BETWEEN_SYMBOL:
    case LIKE_SYMBOL:
    case REGEXP_SYMBOL:
    case IN_SYMBOL:
    case SOUNDS_SYMBOL:
    case NOT_SYMBOL:

    return true;

  default:
    return false;
  }
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns the textual expression of the token. This will either return the actual text in the 
 * parsed query (if it is a lexer symbol) or the textual expression of the constant name for abstract
 * tokens.
 */
std::string MySQLRecognizerTreeWalker::token_text()
{
  return _recognizer->token_text(_tree);
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns the type of the current token. Same as the type you can specify in advance_to().
 */
unsigned int MySQLRecognizerTreeWalker::token_type()
{
  return _tree->getType(_tree);
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns the (one-base) line number of the token.
 */
unsigned int MySQLRecognizerTreeWalker::token_line()
{
  return _tree->getLine(_tree);
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns the (zero-based) offset of the token on its line.
 */
unsigned int MySQLRecognizerTreeWalker::token_start()
{
  return _tree->getCharPositionInLine(_tree);
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns the length of the token in bytes.
 */
int MySQLRecognizerTreeWalker::token_length()
{
  pANTLR3_COMMON_TOKEN token = _tree->getToken(_tree);
  if (token == NULL)
    return 0;

  // Start and stop are actually pointers into the input stream.
  return (int) token->stop - (int) token->start + 1;
}

//----------------- MySQLRecognizer ----------------------------------------------------------------

MySQLRecognizer::MySQLRecognizer(const char *text, int length, bool is_utf8, long server_version,
  const std::string &sql_mode, const std::set<std::string> &charsets)
{
  _text = text;
  _text_length = length;
  _context.version = server_version;
  _context.payload = this;
  _context.sql_mode = parse_sql_mode(sql_mode);

  _charsets = charsets;

  // If the text is not using utf-8 (which it should) then we interpret as 8bit encoding
  // (everything requiring only one byte per char as Latin1, ASCII and similar).
  _input_encoding = is_utf8 ? ANTLR3_ENC_UTF8 : ANTLR3_ENC_8BIT;
  parse();
}

//--------------------------------------------------------------------------------------------------

MySQLRecognizer::~MySQLRecognizer()
{
  _parser->free(_parser);
  _tokens ->free(_tokens);
  _lexer->free(_lexer);
  _input->close(_input);
}

//--------------------------------------------------------------------------------------------------

std::string MySQLRecognizer::dump_tree()
{
  return dump_tree(_ast.tree, "");
}

//--------------------------------------------------------------------------------------------------

std::string MySQLRecognizer::dump_tree(pANTLR3_BASE_TREE tree, const std::string &indentation)
{
  log_debug2("Dumping parse tree\n");
  
  std::string result;
  
  pANTLR3_RECOGNIZER_SHARED_STATE state = _parser->pParser->rec->state;
  ANTLR3_UINT32 char_pos = tree->getCharPositionInLine(tree);
  ANTLR3_UINT32 line = tree->getLine(tree) - 1; // Lines are one-based.
  pANTLR3_STRING token_text = tree->getText(tree);

  pANTLR3_COMMON_TOKEN token = tree->getToken(tree);
  const char* utf8 = (const char*)token_text->chars;
  if (token != NULL)
  {
    ANTLR3_UINT32 token_type = token->getType(token);

    pANTLR3_UINT8 token_name = state->tokenNames[token_type];
    if (token_name == (pANTLR3_UINT8)EOF)
      token_name = (pANTLR3_UINT8)"EOF";

    result = base::strfmt("%s(%i, %i, %s)    %s\n", indentation.c_str(), line, char_pos, token_name, utf8);
  }
  else
  {
    result = base::strfmt("%s(%i, %i, nil)    %s\n", indentation.c_str(), line, char_pos, utf8);
  }
  
  for (ANTLR3_UINT32 index = 0; index < tree->getChildCount(tree); index++)
  {
    pANTLR3_BASE_TREE child = (pANTLR3_BASE_TREE)tree->getChild(tree, index);
    std::string child_text = dump_tree(child, indentation + "\t");
    result += child_text;
  }
  return result;
}

//--------------------------------------------------------------------------------------------------

void MySQLRecognizer::add_error(const std::string &text, ANTLR3_UINT32 error, ANTLR3_UINT32 token,
  ANTLR3_UINT32 line, ANTLR3_UINT32 offset, ANTLR3_UINT32 length)
{
  MySQLParserErrorInfo info = {text, error, token, length, line, offset};
  _error_info.push_back(info);
};

//--------------------------------------------------------------------------------------------------

void MySQLRecognizer::parse()
{
  log_debug2("Start parsing\n");
  
  _error_info.clear();
  
  _input = antlr3StringStreamNew((pANTLR3_UINT8)_text, _input_encoding, _text_length, (pANTLR3_UINT8)"mysql-script");
  _input->setUcaseLA(_input, ANTLR3_TRUE); // Make input case-insensitive. String literals must all be upper case in the grammar!
  
  _lexer = MySQLLexerNew(_input);
  _lexer->pLexer->rec->state->userp = &_context;
  _tokens = antlr3CommonTokenStreamSourceNew(ANTLR3_SIZE_HINT, TOKENSOURCE(_lexer));
  _parser = MySQLParserNew(_tokens);
  _parser->pParser->rec->state->userp = &_context;

  _ast = _parser->query(_parser);
  
  ANTLR3_UINT32 error_count = _parser->pParser->rec->getNumberOfSyntaxErrors(_parser->pParser->rec);
  if (error_count > 0)
    log_debug3("%i errors found\n", error_count);
  log_debug2("Parsing ended\n");
}

//--------------------------------------------------------------------------------------------------

unsigned MySQLRecognizer::parse_sql_mode(const std::string &sql_mode)
{
  unsigned result = 0;

  std::string sql_mode_string= base::toupper(sql_mode);
  std::istringstream iss(sql_mode_string);
  std::string mode;
  while (std::getline(iss, mode, ','))
  {
    mode = base::trim(mode);
    if (mode == "ANSI" || mode == "DB2" || mode == "MAXDB" || mode == "MSSQL" || mode == "ORACLE" ||
      mode == "POSTGRESQL")
      result |= SQL_MODE_ANSI_QUOTES | SQL_MODE_PIPES_AS_CONCAT | SQL_MODE_IGNORE_SPACE;
    else if (mode == "ANSI_QUOTES")
      result |= SQL_MODE_ANSI_QUOTES;
    else if (mode == "PIPES_AS_CONCAT")
      result |= SQL_MODE_PIPES_AS_CONCAT;
    else if (mode == "NO_BACKSLASH_ESCAPES")
      result |= SQL_MODE_NO_BACKSLASH_ESCAPES;
    else if (mode == "IGNORE_SPACE")
      result |= SQL_MODE_IGNORE_SPACE;
    else if (mode == "HIGH_NOT_PRECEDENCE" || mode == "MYSQL323" || mode == "MYSQL40")
      result |= SQL_MODE_HIGH_NOT_PRECEDENCE;
  }
  return result;
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns a tree walker for the current AST.
 */
MySQLRecognizerTreeWalker MySQLRecognizer::tree_walker()
{
  return MySQLRecognizerTreeWalker(this, _ast.tree);
}

//--------------------------------------------------------------------------------------------------

unsigned MySQLRecognizer::sql_mode()
{
  return _context.sql_mode;
}

//--------------------------------------------------------------------------------------------------

/**
 * Determines if the given string is one of the supported charsets.
 */
bool MySQLRecognizer::is_charset(const std::string &s)
{
  return _charsets.find(s) != _charsets.end();
}

//--------------------------------------------------------------------------------------------------

/**
 * Returns the text of the given node. The result depends on the type of the node. If it represents
 * a quoted text entity then two consecutive quote chars are replaced by a single one and if 
 * escape sequence parsing is not switched off (as per sql mode) then such sequences are handled too.
 */
std::string MySQLRecognizer::token_text(pANTLR3_BASE_TREE node)
{
  pANTLR3_STRING text = node->getText(node);
  if (text == NULL)
    return "";

  std::string chars;
  pANTLR3_COMMON_TOKEN token = node->getToken(node);
  ANTLR3_UINT32 type = (token != NULL) ? token->type : ANTLR3_TOKEN_INVALID;

  if (type == STRING)
  {
    // STRING is the grouping subtree for multiple consecutive string literals, which
    // must be concatenated.
    for (ANTLR3_UINT32 index = 0; index < node->getChildCount(node); index++)
    {
      pANTLR3_BASE_TREE child = (pANTLR3_BASE_TREE)node->getChild(node, index);
      chars += token_text(child);
    }

    return chars;
  }

  std::string quote_char = (type == BACK_TICK_QUOTED_ID) ? "`" : (type == SINGLE_QUOTED_TEXT) ? "'" : "\"";
  std::string double_quotes = quote_char + quote_char;
  chars = (const char*)text->chars;

  if ((_context.sql_mode & SQL_MODE_NO_BACKSLASH_ESCAPES) == 0)
    return base::unescape_sql_string(chars, quote_char[0]);
  else
    if (token->user1 > 0)
    {
      // The field user1 is set by the parser to the number of quote char pairs it found.
      // So we can use it to shortcut our handling here.
      base::replace(chars, double_quotes, quote_char);
      return chars;
    }

  return (const char*)text->chars;
}

//--------------------------------------------------------------------------------------------------

