root/trunk/mango/xml/sax/parser/teqXML.d

/*******************************************************************************
    License:
        Copyright (c) 2005 John Demme

        This software is provided 'as-is', without any express or implied
        warranty. In no event will the authors be held liable for damages
        of any kind arising from the use of this software.

        Permission is hereby granted to anyone to use this software for any
        purpose, including commercial applications, and to alter it and/or
        redistribute it freely, subject to the following restrictions:

        1. The origin of this software must not be misrepresented; you must
           not claim that you wrote the original software. If you use this
           software in a product, an acknowledgment within documentation of
           said product would be appreciated but is not required.

        2. Altered source versions must be plainly marked as such, and must
           not be misrepresented as being the original software.

        3. This notice may not be removed or altered from any distribution
           of the source.

        4. Derivative works are permitted, but they must carry this notice
           in full and credit the original source.


        Date:       Initial version, December 2005
        Authors:        John Demme (me@teqdruid.com)

    Version:    0.1b
    History:    Converted from the original mango.xml.dom.parser.teqXML

    See_Also: ISAXReader, ISAXHandler

*******************************************************************************/

module mango.xml.sax.parser.teqXML;

private import  Text = tango.text.Util;

private import  tango.text.convert.UnicodeBom,
        Integer = tango.text.convert.Integer,
        Utf = tango.text.convert.Utf;

private import  mango.xml.sax.Exceptions,
        mango.xml.sax.model.ISAXHandler,
        mango.xml.sax.model.ISAXParser;

private import  tango.io.model.IConduit,
        tango.io.model.IBuffer;

private import  tango.io.Buffer;

private import tango.stdc.stdlib;

private import tango.io.Stdout;

public class TeqXMLException : SAXException
{
    char[] msg;
    
    this(char[] msg)
    {
        super(msg);
        this.msg = msg;
    }
    
    char[] toString()
    {
        return msg;
    }
}

public class TeqXMLEofException : TeqXMLException 
{
    this()
    {
        super("Unexpectedly reached EOF");
    }
}

deprecated alias XMLWriter TeqXMLWriter;

/**
   A simplistic XML encoder.  Reusable, so a pool of these would work rather
   nicely.  Not thread safe, however, so be sure that only one thread is trying
   to use it at once.
*/
class XMLWriter(T=char): ISAXWriter!(T)
{
    static assert ( is (T == char)
            || is (T == wchar)
            || is (T == dchar));

    private class MyWriter {
        MyWriter opCall(T[] str) {
            void[] output = encoder.encode(str);
            buffer.consume(output);
            return this;
        }       

        MyWriter opCall(T c) {
            T[] str = (&c)[0..1];
            void[] output = encoder.encode(str);
            buffer.consume(output);
            return this;
        }
    }

    private static enum Status
    {
        Start,
        End,
        InTag,
        InElement
    }
    
    private Status status;
    private uint tabs;
    private bool pretty;
    private UnicodeBom!(T) encoder;
    private MyWriter writer;
    private IBuffer buffer;

    private static struct Const
    {
        static:
        const T[] lt = "<";
        const T[] ltE = "&lt;"; // The capitol E is for entity
        const T[] gt = ">";
        const T[] gtE = "&gt;";
        const T[] ampE = "&amp;";
        const T[] aposE = "&apos;";
        const T[] quest = "?";
        const T[] space = " ";
        const T[] tab = "\t";
        const T[] dqm = "\"";
        const T[] dqmE = "&quot;";
        const T[] equals = "=";
        const T[] fs = "/";
        const T[] openComment = "<!--";
        const T[] closeComment = "-->";
        const T[] preambleStart = "<?xml version=\"1.0\" encoding=\"";
        const T[] preambleEnd = "\"?>";
        const T[] utf8 = "UTF-8";
        const T[] utf16 = "UTF-16";
        const T[] utf32 = "UTF32";
        const T[] CR = "\n";          
    }
    
    public this()
    {
        status = Status.End;
        tabs = 0;
        writer = new MyWriter();
        encoder = new UnicodeBom!(T)(Encoding.Unknown);
    }
    
    private void writeTabs()
    {
        if (pretty)
            for(uint i=0; i<tabs; i++)
                writer(Const.tab);
    }
  
    void startDocument(OutputStream outStream, Encoding outputType = Encoding.UTF_8N, bool pretty=true)
    {
        buffer = cast(IBuffer) outStream;
        if (buffer is null)
            buffer = new Buffer(outStream.conduit());

        assert(status == Status.End);
        if (outputType == Encoding.UTF_8)
            outputType = Encoding.UTF_8N;
        encoder.setup(outputType);

        status = Status.Start;
        tabs = 0;
        this.pretty = pretty;

        writer(Const.preambleStart);
        switch (outputType) {
        case Encoding.UTF_8:
        case Encoding.UTF_8N:
            writer(Const.utf8);
            break;
        case Encoding.UTF_16:
        case Encoding.UTF_16LE:
        case Encoding.UTF_16BE:
            writer(Const.utf16);
            break;
        case Encoding.UTF_32:
        case Encoding.UTF_32LE:
        case Encoding.UTF_32BE:
            writer(Const.utf32);
            break;
        default:
            throw new TeqXMLException("Specified encoding not supported");          
        }
        writer(Const.preambleEnd);
        if (pretty) writer(Const.CR);
    }

    void endDocument()
    {
        buffer.flush();
        status = Status.End;
    }

    void processingInstruction(T[] target, T[] data)
    {
        assert(status != Status.End);
        writeTabs();
        writer(Const.lt)(Const.quest)(target)(Const.space)(data)(Const.quest)(Const.gt);
        if (pretty) writer(Const.CR);
    }
  
    void startElement(T[] name)
    {
        assert(status == Status.Start || status == Status.InElement || status == Status.InTag);
        if (status == Status.InTag)
        {
            status = Status.InElement;
            writer(Const.gt);
            if (pretty) writer(Const.CR);
        }
        writeTabs();
        writer(Const.lt)(name);
        tabs++;
        status = Status.InTag;
    }
  
    void addAttribute(T[] key, T[] value)
    {
        assert(status == Status.InTag);
        writer(Const.space)(key)(Const.equals)(Const.dqm);
        outputChecked(value);
        writer(Const.dqm);
    }
    
    void endElement(T[] name)
    {
        tabs--;
        if (status == Status.InTag)
        {
            writer(Const.fs)(Const.gt);
            status = Status.InElement;
        }
        else
        {
            assert(status == Status.InElement);
            if (pretty) writeTabs();
            writer(Const.lt)(Const.fs)(name)(Const.gt);
        }
        
        if (pretty) writer(Const.CR);
    }
  
    void characterData (T[] data, CDataStatus cstatus, bool check = true)
    {
        assert(status != Status.End);
        
        if (status == Status.InElement)
        {
            if (cstatus.containsStart())
                writeTabs();
        }
        else if (status == Status.InTag)
        {
            assert(cstatus.containsStart());
            status = Status.InElement;
            writer(Const.gt);
            if (pretty) writer(Const.CR);
            writeTabs();
        }
        
        if (!check)
        {
            writer(data);
        }
        else
        {
            outputChecked(data);
        }
        
        if (pretty && cstatus.containsEnd())
            writer(Const.CR);
    }
    
    private void outputChecked(T[] data)
    {
        foreach(T c; data)
        {
            if (c == '<')
                writer(Const.ltE);
            else if (c == '>')
                writer(Const.gtE);
            else if (c == '&')
                writer(Const.ampE);
            else if (c == '\'')
                writer(Const.aposE);
            else if (c == '"')
                writer(Const.dqmE);
            else
                writer(c);
        }
        return;
    }

    void comment(T[] text)
    {
        assert(status != Status.End);
        writeTabs();
        writer(Const.openComment)(text)(Const.closeComment);
        if (pretty) writer(Const.CR);
    }
}

private final const uint LONG_ENTITY = 1024; //Longest entity allowed is 1024 - 2

deprecated alias XMLReader TeqXMLReader;

/**
   A fairly simplistic SAX parser.  Reusable, so a pool of these would work nicely. This Reader
   is not thread-safe, however, so be sure that only one thread attempts to call parse at once.
   Non-validating.  Memory efficient: the only heap allocations after initialization are increasing
   the size of the input buffer (if necessary) and (small constant)*(maximum depth of XML tree)
   for a tag validation stack.

   Standards: Mostly conforms to the XML 1.0 spec

   Bugs:
   Does not support parsed entities or external references.
*/
class XMLReader(T=char): ISAXReader!(T)
{
    private alias bool function(T) PropCheck;

    private ISAXHandler!(T) myHandler;
    private uint[] myStack; //Just store hashes instead of names

    package ReaderString!(T) reader;
    private ReaderString!(T).Marker start, end;

    private T[][T[]] entities;
    private T[] amp = "amp", lt = "lt", gt = "gt", apos = "apos", quot = "quot";
    private T[] ampV = "&",  ltV = "<", gtV = ">", aposV = "'",   quotV = "\"";

    private bool first;
  
    /**
        Initializes the parser.  This parser is reusable, so you can process multiple documents with
        only one instance (just not at once).

        Params:
        bufferLength =  When we initialize our internal buffer, we'll use this value (if supplied).  Unit
        is code-points, so a buffer length of 1024 allows for 1024 transcoded code-points in the buffer.
        This buffer size must be at least a large as the longest tag in a document if resizing is to be
        avoided.  Default is 512.
    */
    public this(uint bufferLength=0)
    {
        if (bufferLength == 0)
            bufferLength = 512;
        reader = new ReaderString!(T)(bufferLength);
        start = reader.markers[0];
        end = reader.markers[1];
    }

    private void parse(ISAXHandler!(T) handler) {
        myStack.length = 0;
        this.myHandler = handler;
        foreach(T[] key; entities.keys)
            entities.remove(key);
        entities[amp] = ampV;
        entities[lt] = ltV;
        entities[gt] = gtV;
        entities[apos] = aposV;
        entities[quot] = quotV;
        myHandler.startDocument();
        go();
        myHandler.endDocument();
    }
  
    /**
        Tells the parser to begin processing a document

        Params:
        source =    The IBuffer that the parser should draw xml content from
        handler =   The client interface who's methods the parser should call
        encoding =  If the client knows it, tell the parser the text encoding of the incoming XML.
        If not specified, the parser will guess, and failing that assume UTF8N.  The enum for these types
        is in mango.convert.Unicode.
    */
    public void parse(InputStream source, ISAXHandler!(T) handler, Encoding inputEncoding=Encoding.Unknown) 
    in 
    {
        assert(source !is null);
        assert(handler !is null);
    }
    body
    {
        //Initialize the parser
        reader.init(source, inputEncoding);
        parse(handler);
    }

    private static bool isWhiteSpace(T wc)
    {
        uint n = cast(uint)wc;
        return (n == 32 || n == 13 || n == 10 || n == 9);
    }

    private static bool isWhiteSpace(T[] str)
    {
        foreach(T wc; str)
            if (!isWhiteSpace(wc))
                return false;
        return true;
    }

    private void go()
    {
        first = true;
        CDataStatus status = CDataStatus.End;
        T[] whiteSpace;
        do
        {
            end.advanceToNoFill('<');

            if (start.isBefore(end))
            {
                T[] text = start.sliceTo(end);
                if (!status.containsEnd() || !isWhiteSpace(text))
                {
                    if (myStack.length == 0) {
                        static if (is(T == char)) {
                            throw new TeqXMLException("Cannot have text (" ~ text ~ ") before first tag!");
                        } else {
                            throw new TeqXMLException("Cannot have text (" ~ Utf.toString(text) ~ ") before first tag!");
                        }
                    }

                    if (whiteSpace.length > 0) {
                        status = CDataStatus.Start;
                        sendCData(whiteSpace, status);
                    }

                    int amp = Text.locatePrior!(T)(text, '&');
                    if (amp < text.length) {
                        T[] lct = text[amp..$];
                        int semi = Text.locate!(T)(lct, ';');
                        if (semi >= lct.length) {
                            if (amp == 0 && text.length > LONG_ENTITY) {  //Entities can only be so long
                                throw new TeqXMLException("Entity name longer than allowed or semicolon missing after &");
                            }
                            if (status.containsEnd())
                                status = CDataStatus.Start;
                            else
                                status = CDataStatus.Append;

                            sendCData(text[0..amp], status);
                            end.backupTo('&');
                            start.moveTo(end);
                            reader.fillBuffer();
                            goto doneCData;
                        }
                    }

                    if (end.getAdvance() == '<')
                        if (status.containsEnd())
                            status = CDataStatus.All;
                        else
                            status = CDataStatus.End;
                    else
                        if (status.containsEnd())
                            status = CDataStatus.Start;
                        else
                            status = CDataStatus.Append;
                    
                    text = start.sliceTo(end);
                    sendCData(text, status);
                } else if (isWhiteSpace(text)) {
                    whiteSpace ~= text;
                }
            }

        doneCData:
            start.moveTo(end);

            if (end.read() == '<')
            {
                whiteSpace.length = 0;;
                parseOpen();
                start.moveTo(end);
            }
        } while (first || myStack.length > 0);
    }
    
    //If the data to be sent has any entities, they must be entirely contained within one call.
    void sendCData(T[] data, CDataStatus status)
    {
        int ampIndx = Text.locate!(T)(data, '&');
        int last = 0;
        bool sentStart = false;
        while (ampIndx < data.length)
        {
            int scIndx = Text.locate!(T)(data, ';', ampIndx);
            if (scIndx >= data.length)
                break;

            T[] str = data[last..ampIndx];
            if (status.containsStart && !sentStart)
            {
                myHandler.characterData(str, CDataStatus.Start);
                sentStart = true;
            }
            else
                myHandler.characterData(str, CDataStatus.Append);
            
            T[] ent = data[ampIndx+1..scIndx];
            T[1] chr;
            if (!(ent in entities)) {
                if (ent[0] == '#' && ent.length >= 2) {
                    ent = ent[1..$];
                    if (ent[0] == 'x' && ent.length >= 2)
                        chr[0] = Integer.toInt(ent, 16);
                    else
                        chr[0] = Integer.toInt(ent, 10);
                    str = chr;
                } else {
                    char[] str8;
                    static if (is(T == char)) {
                        str8 = str;
                    } else {
                        str8 = tango.text.convert.Utf.toString(str);
                    }
                    throw new TeqXMLException(
                        "Cannot resolve entity '" ~ str8 ~ "'");
                }
            } else {
                str = entities[ent];
            }

            if (status.containsEnd() && scIndx == (data.length-1))
                myHandler.characterData(str, CDataStatus.End);
            else
                myHandler.characterData(str, CDataStatus.Append);

            last = scIndx + 1;
            ampIndx = Text.locate!(T)(data, '&', scIndx);
        }
        
        if (last < data.length)
        {
            T[] str = data[last..$];
            if (last > 0)
                if (status.containsEnd)
                    myHandler.characterData(str, CDataStatus.End);
                else
                    myHandler.characterData(str, CDataStatus.Append);
            else
                myHandler.characterData(str, status);
        }
    }

    void parseOpen()
    {
        start.moveTo(end);
        switch (end.read())
        {
        case '/':
            parseCloseTag();
            break;
        case '?':
            parseQuest();
            break;
        case '!':
            parseExcl();
            break;
        default:
            parseElement();
            break;
        }
    }

    void parseElement() {
        static bool stop(