Ticket #641: UnicodeConverter.2.d

/*******************************************************************************

        copyright:      Copyright (c) 2007 Peter Triller. All rights reserved

        license:        BSD style: $(LICENSE)

        version:        Initial release: Sept 2007

        authors:        Peter

        Provides case mapping Functions for Unicode Strings. As of now it is
        only 99 % complete, because it does not take into account Conditional
        case mappings. This means the Greek Letter Sigma will not be correctly
        case mapped at the end of a Word, and the Locales Lithuanian, Turkish 
        and Azeri are not taken into account during Case Mappings. This means
        all in all around 12 Characters will not be mapped correctly under
        some circumstances.
        
        ICU4j also does not handle these cases at the moment.
        
        Unittests are written against output from ICU4j
        
        This Module tries to minimize Memory allocation and usage. You can
        always pass the output buffer that should be used to the case mapping
        function, which will be resized if necessary.

*******************************************************************************/

module UnicodeConverter;

private import UnicodeData;
private import tango.text.convert.Utf;



/**
 * Converts an Utf8 String to Upper case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
char[] blockToUpper(char[] input, char[] output = null, dchar[] working = null) {

    // ?? How much preallocation ?? This is worst case allocation
    if (working is null)
        working.length = input.length;

    size_t produced = 0;
    size_t ate;
    size_t oprod = 0;
    foreach(dchar ch; input) {
        // TODO Conditional Case Mapping
        UnicodeData **d = (ch in unicodeData);
        if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
            SpecialCaseData **s = (ch in specialCaseData);
            debug {
                assert(s !is null);
            }
            if((*s).upperCaseMapping !is null) {
                // To speed up, use worst case for memory prealocation
                // since the length of an UpperCaseMapping list is at most 4
                // Make sure no relocation is made in the toUtf8 Method
                // better allocation algorithm ?
                int len = (*s).upperCaseMapping.length;
                if(produced + len >= working.length)
                    working.length = working.length + working.length / 2 +  len;
                oprod = produced;
                produced += len;
                working[oprod..produced] = (*s).upperCaseMapping;
                continue;
            }           
        } 
        // Make sure no relocation is made in the toUtf8 Method
        if(produced + 1 >= output.length)
            working.length = working.length + working.length / 2 + 1;
        working[produced++] =  d is null ? ch:(*d).simpleUpperCaseMapping;
    }
    return toUtf8(working[0..produced],output);
}



/**
 * Converts an Utf8 String to Upper case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
char[] toUpper(char[] input, char[] output = null) {

    dchar[1] buf;
    // assume most common case: String stays the same length
    if (output.length < input.length)
        output.length = input.length;

    size_t produced = 0;
    size_t ate;
    foreach(dchar ch; input) {
        // TODO Conditional Case Mapping
        UnicodeData **d = (ch in unicodeData);
        if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
            SpecialCaseData **s = (ch in specialCaseData);
            debug {
                assert(s !is null);
            }
            if((*s).upperCaseMapping !is null) {
                // To speed up, use worst case for memory prealocation
                // since the length of an UpperCaseMapping list is at most 4
                // Make sure no relocation is made in the toUtf8 Method
                // better allocation algorithm ?
                if(produced + (*s).upperCaseMapping.length * 4 >= output.length)
                        output.length = output.length + output.length / 2 +  (*s).upperCaseMapping.length * 4;
                char[] res = toUtf8((*s).upperCaseMapping, output[produced..output.length], &ate);
                debug {
                    assert(ate == (*s).upperCaseMapping.length);
                    assert(res.ptr == output[produced..output.length].ptr);
                }
                produced += res.length;
                continue;
            }           
        } 
        // Make sure no relocation is made in the toUtf8 Method
        if(produced + 4 >= output.length)
            output.length = output.length + output.length / 2 + 4;
        buf[0] = d is null ? ch:(*d).simpleUpperCaseMapping;
        char[] res = toUtf8(buf, output[produced..output.length], &ate);
        debug {
            assert(ate == 1);
            assert(res.ptr == output[produced..output.length].ptr);
        }
        produced += res.length;
    }
    return output[0..produced];
}


/**
 * Converts an Utf16 String to Upper case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
wchar[] toUpper(wchar[] input, wchar[] output = null) {

    dchar[1] buf;
    // assume most common case: String stays the same length
    if (output.length < input.length)
        output.length = input.length;

    size_t produced = 0;
    size_t ate;
    foreach(dchar ch; input) {
        // TODO Conditional Case Mapping
        UnicodeData **d = (ch in unicodeData);
        if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
            SpecialCaseData **s = (ch in specialCaseData);
            debug {
                assert(s !is null);
            }
            if((*s).upperCaseMapping !is null) {
                // To speed up, use worst case for memory prealocation
                // Make sure no relocation is made in the toUtf16 Method
                // better allocation algorithm ?
                if(produced + (*s).upperCaseMapping.length * 2 >= output.length)
                    output.length = output.length + output.length / 2 +  (*s).upperCaseMapping.length * 3;
                wchar[] res = toUtf16((*s).upperCaseMapping, output[produced..output.length], &ate);
                debug {
                    assert(ate == (*s).upperCaseMapping.length);
                    assert(res.ptr == output[produced..output.length].ptr);
                }
                produced += res.length;
                continue;
            }
        }
        // Make sure no relocation is made in the toUtf16 Method
        if(produced + 4 >= output.length)
            output.length = output.length + output.length / 2 + 3;
        buf[0] = d is null ? ch:(*d).simpleUpperCaseMapping;
        wchar[] res = toUtf16(buf, output[produced..output.length], &ate);
        debug {
            assert(ate == 1);
            assert(res.ptr == output[produced..output.length].ptr);
        }
        produced += res.length;
    }
    return output[0..produced];
}

/**
 * Converts an Utf32 String to Upper case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
dchar[] toUpper(dchar[] input, dchar[] output = null) {

    // assume most common case: String stays the same length
    if (input.length > output.length)
        output.length = input.length;

    size_t produced = 0;
    if (input.length)
        foreach(dchar orig; input) {
            // TODO Conditional Case Mapping
            UnicodeData **d = (orig in unicodeData);
            if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
                SpecialCaseData **s = (orig in specialCaseData);
                debug {
                    assert(s !is null);
                }
                if((*s).upperCaseMapping !is null) {
                    // Better resize strategy ???
                    if(produced + (*s).upperCaseMapping.length  > output.length)
                        output.length = output.length + output.length / 2 + (*s).upperCaseMapping.length;
                    foreach(ch; (*s).upperCaseMapping) {
                        output[produced++] = ch;
                    }
                }
                continue;
            }
            if(produced >= output.length)
                output.length = output.length + output.length / 2;
            output[produced++] = d is null ? orig:(*d).simpleUpperCaseMapping;
        }
    return output[0..produced];
}


/**
 * Converts an Utf8 String to Lower case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
char[] toLower(char[] input, char[] output = null) {

    dchar[1] buf;
    // assume most common case: String stays the same length
    if (output.length < input.length)
        output.length = input.length;

    size_t produced = 0;
    size_t ate;
    foreach(dchar ch; input) {
        // TODO Conditional Case Mapping
        UnicodeData **d = (ch in unicodeData);
        if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
            SpecialCaseData **s = (ch in specialCaseData);
            debug {
                assert(s !is null);
            }
            if((*s).lowerCaseMapping !is null) {
                // To speed up, use worst case for memory prealocation
                // since the length of an LowerCaseMapping list is at most 4
                // Make sure no relocation is made in the toUtf8 Method
                // better allocation algorithm ?
                if(produced + (*s).lowerCaseMapping.length * 4 >= output.length)
                        output.length = output.length + output.length / 2 +  (*s).lowerCaseMapping.length * 4;
                char[] res = toUtf8((*s).lowerCaseMapping, output[produced..output.length], &ate);
                debug {
                    assert(ate == (*s).lowerCaseMapping.length);
                    assert(res.ptr == output[produced..output.length].ptr);
                }
                produced += res.length;
                continue;
            }           
        } 
        // Make sure no relocation is made in the toUtf8 Method
        if(produced + 4 >= output.length)
            output.length = output.length + output.length / 2 + 4;
        buf[0] = d is null ? ch:(*d).simpleLowerCaseMapping;
        char[] res = toUtf8(buf, output[produced..output.length], &ate);
        debug {
            assert(ate == 1);
            assert(res.ptr == output[produced..output.length].ptr);
        }
        produced += res.length;
    }
    return output[0..produced];
}


/**
 * Converts an Utf16 String to Lower case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
wchar[] toLower(wchar[] input, wchar[] output = null) {

    dchar[1] buf;
    // assume most common case: String stays the same length
    if (output.length < input.length)
        output.length = input.length;

    size_t produced = 0;
    size_t ate;
    foreach(dchar ch; input) {
        // TODO Conditional Case Mapping
        UnicodeData **d = (ch in unicodeData);
        if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
            SpecialCaseData **s = (ch in specialCaseData);
            debug {
                assert(s !is null);
            }
            if((*s).lowerCaseMapping !is null) {
                // To speed up, use worst case for memory prealocation
                // Make sure no relocation is made in the toUtf16 Method
                // better allocation algorithm ?
                if(produced + (*s).lowerCaseMapping.length * 2 >= output.length)
                    output.length = output.length + output.length / 2 +  (*s).lowerCaseMapping.length * 3;
                wchar[] res = toUtf16((*s).lowerCaseMapping, output[produced..output.length], &ate);
                debug {
                    assert(ate == (*s).lowerCaseMapping.length);
                    assert(res.ptr == output[produced..output.length].ptr);
                }
                produced += res.length;
                continue;
            }
        }
        // Make sure no relocation is made in the toUtf16 Method
        if(produced + 4 >= output.length)
            output.length = output.length + output.length / 2 + 3;
        buf[0] = d is null ? ch:(*d).simpleLowerCaseMapping;
        wchar[] res = toUtf16(buf, output[produced..output.length], &ate);
        debug {
            assert(ate == 1);
            assert(res.ptr == output[produced..output.length].ptr);
        }
        produced += res.length;
    }
    return output[0..produced];
}


/**
 * Converts an Utf32 String to Lower case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
dchar[] toLower(dchar[] input, dchar[] output = null) {

    // assume most common case: String stays the same length
    if (input.length > output.length)
        output.length = input.length;

    size_t produced = 0;
    if (input.length)
        foreach(dchar orig; input) {
            // TODO Conditional Case Mapping
            UnicodeData **d = (orig in unicodeData);
            if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
                SpecialCaseData **s = (orig in specialCaseData);
                debug {
                    assert(s !is null);
                }
                if((*s).lowerCaseMapping !is null) {
                    // Better resize strategy ???
                    if(produced + (*s).lowerCaseMapping.length  > output.length)
                        output.length = output.length + output.length / 2 + (*s).lowerCaseMapping.length;
                    foreach(ch; (*s).lowerCaseMapping) {
                        output[produced++] = ch;
                    }
                }
                continue;
            }
            if(produced >= output.length)
                output.length = output.length + output.length / 2;
            output[produced++] = d is null ? orig:(*d).simpleLowerCaseMapping;
        }
    return output[0..produced];
}

/**
 * Converts an Utf8 String to Folding case
 * Folding case is used for case insensitive comparsions.
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
char[] toFold(char[] input, char[] output = null) {

    dchar[1] buf;
    // assume most common case: String stays the same length
    if (output.length < input.length)
        output.length = input.length;

    size_t produced = 0;
    size_t ate;
    foreach(dchar ch; input) {
        FoldingCaseData **s = (ch in foldingCaseData);
        if(s !is null) {
            // To speed up, use worst case for memory prealocation
            // since the length of an UpperCaseMapping list is at most 4
            // Make sure no relocation is made in the toUtf8 Method
            // better allocation algorithm ?
            if(produced + (*s).mapping.length * 4 >= output.length)
                output.length = output.length + output.length / 2 +  (*s).mapping.length * 4;
            char[] res = toUtf8((*s).mapping, output[produced..output.length], &ate);
            debug {
                assert(ate == (*s).mapping.length);
                assert(res.ptr == output[produced..output.length].ptr);
            }
            produced += res.length;
            continue;
        }
        // Make sure no relocation is made in the toUtf8 Method
        if(produced + 4 >= output.length)
            output.length = output.length + output.length / 2 + 4;
        buf[0] = ch;
        char[] res = toUtf8(buf, output[produced..output.length], &ate);
        debug {
            assert(ate == 1);
            assert(res.ptr == output[produced..output.length].ptr);
        }
        produced += res.length;
    }
    return output[0..produced];
}

/**
 * Converts an Utf16 String to Folding case
 * Folding case is used for case insensitive comparsions.
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
wchar[] toFold(wchar[] input, wchar[] output = null) {

    dchar[1] buf;
    // assume most common case: String stays the same length
    if (output.length < input.length)
        output.length = input.length;

    size_t produced = 0;
    size_t ate;
    foreach(dchar ch; input) {
        FoldingCaseData **s = (ch in foldingCaseData);
        if(s !is null) {
            // To speed up, use worst case for memory prealocation
            // Make sure no relocation is made in the toUtf16 Method
            // better allocation algorithm ?
            if(produced + (*s).mapping.length * 2 >= output.length)
                output.length = output.length + output.length / 2 +  (*s).mapping.length * 3;
            wchar[] res = toUtf16((*s).mapping, output[produced..output.length], &ate);
            debug {
                assert(ate == (*s).mapping.length);
                assert(res.ptr == output[produced..output.length].ptr);
            }
            produced += res.length;
            continue;
        }
        // Make sure no relocation is made in the toUtf16 Method
        if(produced + 4 >= output.length)
            output.length = output.length + output.length / 2 + 3;
        buf[0] = ch;
        wchar[] res = toUtf16(buf, output[produced..output.length], &ate);
        debug {
            assert(ate == 1);
            assert(res.ptr == output[produced..output.length].ptr);
        }
        produced += res.length;
    }
    return output[0..produced];
}

/**
 * Converts an Utf32 String to Folding case
 * Folding case is used for case insensitive comparsions.
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
dchar[] toFold(dchar[] input, dchar[] output = null) {

    // assume most common case: String stays the same length
    if (input.length > output.length)
        output.length = input.length;

    size_t produced = 0;
    if (input.length)
        foreach(dchar orig; input) {
            FoldingCaseData **d = (orig in foldingCaseData);
            if(d !is null ) {
                // Better resize strategy ???
                if(produced + (*d).mapping.length  > output.length)
                    output.length = output.length + output.length / 2 + (*d).mapping.length;
                foreach(ch; (*d).mapping) {
                    output[produced++] = ch;
                }
                continue;
            }
            if(produced >= output.length)