/*
 * Copyright (c) 2008, Aaron Digulla
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 * 
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Aaron Digulla nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */
package de.pdark.decentxml;

import de.pdark.decentxml.validation.CharValidator;

/**
 * This class allows to chop an XMLSource into tokens.
 * 
 * <p>You can use it to parse XML yourself or use the XMLParser to
 * let it parse XML into a Document.
 * 
 * @author digulla
 * @see de.pdark.decentxml.XMLSource
 * @see de.pdark.decentxml.XMLParser
 * @see de.pdark.decentxml.Document
 */
public class XMLTokenizer
{
    /** Types of tokens the tokenizer can return */
    public static enum Type {
        /** A piece of text with the entities still intact */
        TEXT,
        /** A CDATA segment (including the CDATA marker) */
        CDATA,
        /** Whitespace in a doctype */
        DTD_WHITESPACE,
        /** A processing instruction with the begin and end tag */
        PROCESSING_INSTRUCTION,
        /** A comment (including the begin and end tag */
        COMMENT,
        /** This is the name part of the start tag */
        BEGIN_ELEMENT,
        /** An attribute in the start tag */
        ATTRIBUTE,
        /** A custom attribute. Use this if you extended Attribute and need to distinguish
         *  the nodes from common attributes. 
         */
        CUSTOM_ATTRIBUTE,
        /** The token which terminates the start tag. It's value is either '>' or '/>' if it's an empty element */
        BEGIN_ELEMENT_END,
        /** The end element */
        END_ELEMENT,
        /** A document node */
        DOCUMENT,
        /** Node-type for elements after they have been parsed */
        ELEMENT,
        /** A custom element. Use this if you extended Element and need to distinguish the nodes
         *  from common elements.
         */
        CUSTOM_ELEMENT,
        /** An entity */
        ENTITY,
        /** <!DOCTYPE */
        DOCTYPE,
        /** "SYSTEM" */
        DOCTYPE_SYSTEM,
        /** "PUBLIC" */
        DOCTYPE_PUBLIC,
        /** "NDATA" */
        DOCTYPE_NDATA,
        /** <!ELEMENT */
        DOCTYPE_ELEMENT,
        /** <!ATTLIST */
        DOCTYPE_ATTLIST,
        /** <!ENTITY */
        DOCTYPE_ENTITY,
        /** <!NOTATION */
        DOCTYPE_NOTATION,
        /** Something between quotes in a doctype */
        DOCTYPE_QUOTED_TEXT,
        /** "[" */
        DOCTYPE_BEGIN_SUBSET,
        /** "]" */
        DOCTYPE_END_SUBSET,
        /** ">" */
        DOCTYPE_END,
        /** "-- comment --" inside of a doctype */
        DOCTYPE_COMMENT,
        /** "(" */
        DOCTYPE_BEGIN_GROUP,
        /** ")" */
        DOCTYPE_END_GROUP,
        /** "|" */
        DOCTYPE_ALTERNATIVE,
        /** "?" */
        DOCTYPE_ZERO_OR_ONE,
        /** "*" */
        DOCTYPE_ZERO_OR_MORE,
        /** "+" */
        DOCTYPE_ONE_OR_MORE,
        /** "%" */
        DOCTYPE_PARAMETER_ENTITY,
        /** ";" */
        DOCTYPE_PARAMETER_ENTITY_END,
        /** "#PCDATA" */
        DOCTYPE_PCDATA,
        /** "#IMPLIED" */
        DOCTYPE_IMPLIED,
        /** "#REQUIRED" */
        DOCTYPE_REQUIRED,
        /** "#FIXED" */
        DOCTYPE_FIXED,
        /** "," */
        DOCTYPE_SEQUENCE,
        /** "CDATA" */
        DOCTYPE_CDATA;
    }
    
    protected final XMLSource source;
    /** The current position in the source */
    protected int pos;
    /** true if we're currently inside of a start tag */
    protected boolean inStartElement;
    /** Should the tokenizer return entities or treat them as text? Default is true. */
    private boolean treatEntitiesAsText = true;
    /** The character validator for this tokenizer. */
    private CharValidator charValidator = new CharValidator ();
    /** The entity resolver to use to expand and verify entities. */
    private EntityResolver entityResolver;
    
    public XMLTokenizer (XMLSource source)
    {
        this.source = source;
    }
    
    public XMLTokenizer setTreatEntitiesAsText (boolean treatEntitiesAsText)
    {
        this.treatEntitiesAsText = treatEntitiesAsText;
        return this;
    }
    
    public boolean isTreatEntitiesAsText ()
    {
        return treatEntitiesAsText;
    }
    
    public CharValidator getCharValidator ()
    {
        return charValidator;
    }

    public XMLTokenizer setCharValidator (CharValidator charValidator)
    {
        if (charValidator == null)
            throw new IllegalArgumentException ("charValidator is null");
        
        this.charValidator = charValidator;
        return this;
    }

    public EntityResolver getEntityResolver ()
    {
        return entityResolver;
    }
    
    public XMLTokenizer setEntityResolver (EntityResolver resolver)
    {
        this.entityResolver = resolver;
        return this;
    }
    
    /** Fetch the next token from the source. Returns <code>null</code> if
     * there are no more tokens in the input.
     * 
     * @return The next token or <code>null</code> at EOF
     */
    public Token next ()
    {
        if (pos >= source.length ())
            return null;
        
        Token token = createToken ();
        
        char c = source.charAt (pos);
        if (inStartElement)
        {
            skipWhiteSpace ();
            
            c = source.charAt (pos);
            if (c == '>')
            {
                pos ++;
                token.setType (Type.BEGIN_ELEMENT_END);
                inStartElement = false;
            }
            else if (c == '/') // Empty element
            {
                pos ++;
                if (pos >= source.length () || source.charAt (pos) != '>')
                    throw new XMLParseException ("Expected '/>'", source, pos-1);
                
                pos ++;
                
                token.setType (Type.BEGIN_ELEMENT_END);
                inStartElement = false;
            }
            else
            {
                parseAttribute (token);
            }
        }
        else if (c == '<')
        {
            pos ++;
            parseBeginSomething (token);
        }
        else if (!treatEntitiesAsText && c == '&')
        {
            pos ++;
            parseEntity (token);
        }
        else
        {
            parseText (token);
        }
        
        token.setEndOffset (pos);
        
        return token;
    }

    /** All tokens are created here.
     * 
     * <p>Use this method to create custom tokens with
     * additional information.
     * 
     * @return a new, pre-initialized token
     */
    protected Token createToken ()
    {
        Token token = new Token ();
        token.setSource (source);
        token.setStartOffset (pos);
        return token;
    }

    public XMLSource getSource ()
    {
        return source;
    }
    
    /** Get the current parsing position (for error handling, for example).
     * 
     * <p>This value is not very accurate because the tokenizer might be
     * anywhere in the stream.
     */
    public int getOffset ()
    {
        return pos;
    }
    
    /** Set the current parsing position. You can use this to restart
     *  parsing after an error or to jump around in the input. */
    public void setOffset (int offset)
    {
        this.pos = offset;
    }
    
    /** Read one of "&lt;tag", "&lt;?pi", "&lt;!--", "&lt;![CDATA[" or a end tag. */
    protected void parseBeginSomething (Token token)
    {
        if (pos >= source.length ())
            throw new XMLParseException ("Unexpected end of input. Expected start or end tag, processing instruction, comment or CDATA", source, pos);
        
        char c = source.charAt (pos);
        switch (c)
        {
        case '?':
            pos ++;
            parseProcessingInstruction (token);
            break;
            
        case '!':
            pos ++;
            parseExcalamation (token);
            break;
        
        case '/':
            pos ++;
            parseEndElement (token);
            break;
        
        default:
            parseBeginElement (token);
            break;
        }
    }

    /** Read the name of an element.
     * 
     *  <p>The resulting token will contain the '&lt;' plus any whitespace between
     *  it and the name plus the name itself but no whitespace after the name.
     */
    protected void parseBeginElement (Token token)
    {
        token.setType (Type.BEGIN_ELEMENT);
        inStartElement = true;
        
        skipWhiteSpace ();
        
        int nameStartOffset = pos;
        
        parseName ("start tag");
        
        if (pos == nameStartOffset)
            throw new XMLParseException ("Missing element name", token);
        
        if (pos >= source.length ())
            throw new XMLParseException ("Missing '>' of start tag", source, pos);
        
        char c = source.charAt (pos);
        if (!charValidator.isWhitespace (c) && c != '/' && c != '>')
            throw new XMLParseException ("Expected whitespace, '>' or '/>' after element name", source, pos);
    }

    /**
     * Read an end tag.
     * 
     * <p>The resulting token will contain the '&lt;/' and '&gt;' plus the
     * name plus any whitespace between those three.
     */
    protected void parseEndElement (Token token)
    {
        token.setType (Type.END_ELEMENT);
        
        skipWhiteSpace ();
        
        parseName ("end tag");
        
        skipWhiteSpace ();
        
        expect ('>');
    }

    /** Parse "&lt;!--" or  "&lt;![CDATA[" */
    protected void parseExcalamation (Token token)
    {
        char c = source.charAt (pos);
        if (c == '-')
        {
            pos ++;
            parseComment (token);
        }
        else if (c == '[')
        {
            pos ++;
            parseCData (token);
        }
        else if (c == 'D')
        {
            pos ++;
            parseDocType (token);
        }
        else
            throw new XMLParseException ("Expected '<!--' or '<![CDATA['", source, pos-2);
    }

    /** Parse a doctype declaration
    *
    *  <p>The resulting token will contain "<!DOCTYPE"
    */
    protected void parseDocType (Token token)
    {
        token.setType (Type.DOCTYPE);
        nextChars ("<!DOCTYPE", pos - 3, "Expected '<!DOCTYPE'");
    }

    /** Parse a CDATA element.
     * 
     *  <p>The resulting token will contain the "&lt;![CDATA[" plus the
     *  terminating "]]&gt;".
     */
    protected void parseCData (Token token)
    {
        token.setType (Type.CDATA);
        
        nextChars ("<![CDATA[", pos - 3, "Expected '<![CDATA['");

        while (true)
        {
            if (pos >= source.length ())
                throw new XMLParseException ("Expected ']]>'", source, pos);
            
            char c = source.charAt (pos);
            if (c == ']')
            {
                int errorPos = pos;
                pos ++;
                if (pos+1 >= source.length ())
                    throw new XMLParseException ("Expected ']]>'"+lookAheadForErrorMessage ("but found", errorPos, 20), source, errorPos);
                
                c = source.charAt (pos);
                if (c != ']')
                    continue;
                
                c = source.charAt (pos + 1);
                if (c == '>')
                {
                    pos += 2;
                    break;
                }
            }
            else
            {
                String msg = charValidator.isValid (source, pos);
                if (msg != null)
                    throw new XMLParseException ("Illegal character found in CDATA. "+msg, source, pos);
                
                skipChar (c);
            }
        }
    }

    /** Read a comment.
     * 
     *  <p>The resulting token will contain the "&lt;!--" plus the
     *  terminating "--&gt;".
     */
    protected void parseComment (Token token)
    {
        token.setType (Type.COMMENT);
        
        if (pos >= source.length () || source.charAt (pos) != '-')
            throw new XMLParseException ("Expected '<!--'", source, pos-3);
        
        pos ++;
        
        while (true)
        {
            if (pos >= source.length ())
                throw new XMLParseException ("Expected '-->'", source, pos);
            
            char c = source.charAt (pos);
            if (c == '-')
            {
                pos ++;
                if (pos >= source.length ())
                    throw new XMLParseException ("Expected '-->'", source, pos-1);
                
                c = source.charAt (pos ++);
                if (c != '-')
                    continue;
                
                if (pos >= source.length ())
                    throw new XMLParseException ("Expected '-->'", source, pos-2);
                
                c = source.charAt (pos ++);
                if (c != '>')
                    throw new XMLParseException ("XML comments must not contain '--'", source, pos-3);

                break;
            }
            else
            {
                String msg = charValidator.isValid (source, pos);
                if (msg != null)
                    throw new XMLParseException ("Illegal character found in comment. "+msg, source, pos);
                
                skipChar (c);
            }
        }
    }

    /** Read a processing instruction.
     * 
     *  <p>The resulting token will contain the "&lt;?" plus the
     *  terminating "?&gt;".
     */
    protected void parseProcessingInstruction (Token token)
    {
        token.setType (Type.PROCESSING_INSTRUCTION);
        int errorPos = pos - 2;
        
        while (true)
        {
            if (pos >= source.length ())
                throw new XMLParseException ("Missing end of processing instruction", source, errorPos);
            
            char c = source.charAt (pos);
            if (c == '?')
            {
                pos ++;
                if (pos >= source.length ())
                    throw new XMLParseException ("Expected '>' after '?'", source, pos);
                
                if (source.charAt (pos) == '>')
                {
                    pos ++;
                    break;
                }
            }
            else
            {
                String msg = charValidator.isValid (source, pos);
                if (msg != null)
                    throw new XMLParseException ("Illegal character found in processing instruction. "+msg, source, pos);

                skipChar (c);
            }
        }
    }

    /** Read the attribute of an element.
     * 
     *  <p>The resulting token will contain the name, "=" plus the
     *  quotes and the value.
     */
    protected void parseAttribute (Token token)
    {
        token.setType (Type.ATTRIBUTE);

        parseName ("attribute");
        
        if (pos == token.getStartOffset())
            throw new XMLParseException ("Expected attribute name", source, pos);
        
        skipWhiteSpace ();
        expect ('=');
        skipWhiteSpace ();
        
        char c = 0;
        if (pos < source.length ())
            c = source.charAt (pos);
        if (c != '\'' && c != '"')
            throw new XMLParseException ("Expected single or double quotes", source, pos);
        
        char endChar = c;
        boolean insideEntity = false;
        int errorPos = pos;
        
        while (true)
        {
            pos ++;
            if (pos >= source.length ())
            {
                int i = Math.min (20, source.length () - token.getStartOffset ());
                throw new XMLParseException ("Missing end quote ("+endChar+") of attribute: "
                        +lookAheadForErrorMessage (null, token.getStartOffset (), i), token);
            }
            
            c = source.charAt (pos);
            if (c == endChar)
                break;
            
            if (c == '<' || c == '>')
                throw new XMLParseException ("Illegal character in attribute value: '"+c+"'", source, pos);
            
            if (c == '&')
            {
                insideEntity = true;
                errorPos = pos;
            }
            else if (c == ';')
            {
                verifyEntity (errorPos, pos+1);
                insideEntity = false;
            }
            else
            {
                String msg = charValidator.isValid (source, pos);
                if (msg != null)
                    throw new XMLParseException ("Illegal character found in attribute value. "+msg, source, pos);
                
                skipChar (c);
                pos --;
            }
        }
        
        if (insideEntity)
        {
            throw new XMLParseException ("Missing ';' after '&': "+lookAheadForErrorMessage (null, errorPos, 20), source, errorPos);
        }
        
        // Skip end-char
        pos ++;
    }

    /** Read an XML name */
    protected void parseName (String objectName)
    {
        int startPos = pos;
        
        if (pos < source.length () && charValidator.isNameStartChar (source.charAt (pos)))
        {
            pos ++;
        
            while (pos < source.length () && charValidator.isNameChar (source.charAt (pos)))
                pos ++;
        }
        
        if (pos == startPos)
        {
            throw new XMLParseException ("Expected valid XML name for "+objectName+lookAheadForErrorMessage ("but found", startPos, 20), source, startPos);
        }
    }
    
    /** Read a piece of text.
     * 
     *  <p>The resulting token will contain the text as is with all
     *  the entity and numeric character references. 
     */
    protected void parseText (Token token)
    {
        token.setType (Type.TEXT);
        int errorPos = pos - 1;
        boolean insideEntity = false;
        
        while (pos < source.length ())
        {
            char c = source.charAt (pos);
            if (c == '<')
                break;
            
            if (c == '&')
            {
                if (!treatEntitiesAsText)
                    break;
                
                errorPos = pos;
                insideEntity = true;
            }
            else if (c == ';')
            {
                if (insideEntity)
                {
                    verifyEntity (errorPos, pos + 1);
                }
                
                insideEntity = false;
            }
            else if (c == ']' && pos + 2 < source.length ())
            {
                if (source.charAt (pos+1) == ']' && source.charAt (pos+2) == '>')
                    throw new XMLParseException ("Please replace the '>' of ']]>' in character data with '&gt;'", source, pos+2);
            }
            
            String msg = charValidator.isValid (source, pos);
            if (msg != null)
                throw new XMLParseException ("Illegal character found in text. "+msg, source, pos);
            
            skipChar (c);
        }
        
        if (insideEntity)
        {
            throw new XMLParseException ("Missing ';' after '&': "+lookAheadForErrorMessage (null, errorPos, 20), source, errorPos);
        }
    }

    /**
     * Advance one or two positions, depending on whether the current character if
     * the high part of a surrogate pair.
     */
    protected void skipChar (char c)
    {
        pos ++;
        if (Character.isHighSurrogate (c))
            pos ++;
    }

    /**
     * Verify an entity. If no entityResolver is installed, this does nothing.
     */
    protected void verifyEntity (int start, int end)
    {
        if (entityResolver == null)
            return;
        
        String entity = source.substring (start, end);

        try
        {
            entityResolver.validateEntity (entity);
        }
        catch (IllegalArgumentException e)
        {
            throw new XMLParseException (e.getMessage (), e).setSource (source, start);
        }
    }
    
    protected void parseEntity (Token token)
    {
        token.setType (Type.ENTITY);
        
        char c;
        if (pos < source.length ())
        {
            c = source.charAt (pos);
            if (c == '#')
                pos ++;
        }
        
        while (pos < source.length ())
        {
            c = source.charAt (pos);
            if (c == ';')
                break;
            
            if (!charValidator.isNameChar (c))
                throw new XMLParseException ("Illegal character in entity: ["+c+"] ("+Integer.toHexString (c)+")", source, pos);
            
            pos ++;
        }
        
        expect (';');
        
        verifyEntity (token.getStartOffset (), pos);
    }

    protected void nextChars (String expected, int startPos, String errorMessage)
    {
        int len = expected.length () - (pos - startPos);
        if (pos + len > source.length ())
            throw new XMLParseException (errorMessage, source, startPos);
        
        String s = source.substring (startPos, startPos + expected.length ());
        if (!expected.equals (s))
            throw new XMLParseException (errorMessage, source, startPos);
        
        pos += len;
    }

    protected char nextChar (String errorMessage)
    {
        if (pos >= source.length ())
            throw new XMLParseException (errorMessage, source, pos);
        return source.charAt (pos ++);
    }

    /**
     * Check that the next character is {@code expected} and skip it
     */
    protected void expect (char expected)
    {
        if (pos >= source.length () || source.charAt (pos) != expected)
        {
            throw new XMLParseException ("Expected '"+expected+"'"+lookAheadForErrorMessage ("but found", pos, 20), source, pos);
        }
        pos ++;
    }

    protected String lookAheadForErrorMessage (String conditionalPrefix, int pos, int len)
    {
        String found = "";
        if (pos < source.length ())
        {
            int len2 = source.length () - pos;
            len = Math.min (len, len2);
            String s = source.substring (pos, pos+len);
            if (len != len2)
                s += "...";
            if (conditionalPrefix == null)
                found = TextUtils.escapeJavaString (s);
            else
                found = " " + conditionalPrefix + " " + TextUtils.escapeJavaString (s);
        }
        return found;
    }

    /** Advance the current position past any whitespace in the input */
    protected void skipWhiteSpace ()
    {
        while (pos < source.length () && charValidator.isWhitespace (source.charAt (pos)))
            pos ++;
    }

}
