Ticket #641: UnicodeConverter.d

/*******************************************************************************

        copyright:      Copyright (c) 2007 Peter Triller. All rights reserved

        license:        BSD style: $(LICENSE)

        version:        Initial release: Sept 2007

        authors:        Peter

        Provides case mapping Functions for Unicode Strings. As of now it is
        only 99 % complete, because it does not take into account Conditional
        case mappings. This means the Greek Letter Sigma will not be correctly
        case mapped at the end of a Word, and the Locales Lithuanian, Turkish 
        and Azeri are not taken into account during Case Mappings. This means
        all in all around 12 Characters will not be mapped correctly under
        some circumstances.
        
        ICU4j also does not handle these cases at the moment.
        
        Unittests are written against output from ICU4j
        
        This Module tries to minimize Memory allocation and usage. You can
        always pass the output buffer that should be used to the case mapping
        function, which will be resized if necessary.

*******************************************************************************/

module UnicodeConverter;

private import UnicodeData;
private import tango.text.convert.Utf;



/**
 * Converts an Utf8 String to Upper case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
char[] blockToUpper(char[] input, char[] output = null, dchar[] working = null) {

    // ?? How much preallocation ?? This is worst case allocation
    if (working is null)
        working.length = input.length;

    size_t produced = 0;
    size_t ate;
    size_t oprod = 0;
    foreach(dchar ch; input) {
        // TODO Conditional Case Mapping
        UnicodeData **d = (ch in unicodeData);
        if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
            SpecialCaseData **s = (ch in specialCaseData);
            debug {
                assert(s !is null);
            }
            if((*s).upperCaseMapping !is null) {
                // To speed up, use worst case for memory prealocation
                // since the length of an UpperCaseMapping list is at most 4
                // Make sure no relocation is made in the toUtf8 Method
                // better allocation algorithm ?
                int len = (*s).upperCaseMapping.length;
                if(produced + len >= working.length)
                    working.length = working.length + working.length / 2 +  len;
                oprod = produced;
                produced += len;
                working[oprod..produced] = (*s).upperCaseMapping;
                continue;
            }           
        } 
        // Make sure no relocation is made in the toUtf8 Method
        if(produced + 1 >= output.length)
            working.length = working.length + working.length / 2 + 1;
        working[produced++] =  d is null ? ch:(*d).simpleUpperCaseMapping;
    }
    return toUtf8(working[0..produced],output);
}



/**
 * Converts an Utf8 String to Upper case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
char[] toUpper(char[] input, char[] output = null) {

    dchar[1] buf;
    // assume most common case: String stays the same length
    if (output.length < input.length)
        output.length = input.length;

    size_t produced = 0;
    size_t ate;
    foreach(dchar ch; input) {
        // TODO Conditional Case Mapping
        UnicodeData **d = (ch in unicodeData);
        if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
            SpecialCaseData **s = (ch in specialCaseData);
            debug {
                assert(s !is null);
            }
            if((*s).upperCaseMapping !is null) {
                // To speed up, use worst case for memory prealocation
                // since the length of an UpperCaseMapping list is at most 4
                // Make sure no relocation is made in the toUtf8 Method
                // better allocation algorithm ?
                if(produced + (*s).upperCaseMapping.length * 4 >= output.length)
                        output.length = output.length + output.length / 2 +  (*s).upperCaseMapping.length * 4;
                char[] res = toUtf8((*s).upperCaseMapping, output[produced..output.length], &ate);
                debug {
                    assert(ate == (*s).upperCaseMapping.length);
                    assert(res.ptr == output[produced..output.length].ptr);
                }
                produced += res.length;
                continue;
            }           
        } 
        // Make sure no relocation is made in the toUtf8 Method
        if(produced + 4 >= output.length)
            output.length = output.length + output.length / 2 + 4;
        buf[0] = d is null ? ch:(*d).simpleUpperCaseMapping;
        char[] res = toUtf8(buf, output[produced..output.length], &ate);
        debug {
            assert(ate == 1);
            assert(res.ptr == output[produced..output.length].ptr);
        }
        produced += res.length;
    }
    return output[0..produced];
}


/**
 * Converts an Utf16 String to Upper case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
wchar[] toUpper(wchar[] input, wchar[] output = null) {

    dchar[1] buf;
    // assume most common case: String stays the same length
    if (output.length < input.length)
        output.length = input.length;

    size_t produced = 0;
    size_t ate;
    foreach(dchar ch; input) {
        // TODO Conditional Case Mapping
        UnicodeData **d = (ch in unicodeData);
        if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
            SpecialCaseData **s = (ch in specialCaseData);
            debug {
                assert(s !is null);
            }
            if((*s).upperCaseMapping !is null) {
                // To speed up, use worst case for memory prealocation
                // Make sure no relocation is made in the toUtf8 Method
                // better allocation algorithm ?
                if(produced + (*s).upperCaseMapping.length * 2 >= output.length)
                    output.length = output.length + output.length / 2 +  (*s).upperCaseMapping.length * 3;
                wchar[] res = toUtf16((*s).upperCaseMapping, output[produced..output.length], &ate);
                debug {
                    assert(ate == (*s).upperCaseMapping.length);
                    assert(res.ptr == output[produced..output.length].ptr);
                }
                produced += res.length;
                continue;
            }
        }
        // Make sure no relocation is made in the toUtf8 Method
        if(produced + 4 >= output.length)
            output.length = output.length + output.length / 2 + 3;
        buf[0] = d is null ? ch:(*d).simpleUpperCaseMapping;
        wchar[] res = toUtf16(buf, output[produced..output.length], &ate);
        debug {
            assert(ate == 1);
            assert(res.ptr == output[produced..output.length].ptr);
        }
        produced += res.length;
    }
    return output[0..produced];
}

/**
 * Converts an Utf32 String to Upper case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
dchar[] toUpper(dchar[] input, dchar[] output = null) {

    // assume most common case: String stays the same length
    if (input.length > output.length)
        output.length = input.length;

    size_t produced = 0;
    if (input.length)
        foreach(dchar orig; input) {
            // TODO Conditional Case Mapping
            UnicodeData **d = (orig in unicodeData);
            if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
                SpecialCaseData **s = (orig in specialCaseData);
                debug {
                    assert(s !is null);
                }
                if((*s).upperCaseMapping !is null) {
                    // Better resize strategy ???
                    if(produced + (*s).upperCaseMapping.length  > output.length)
                        output.length = output.length + output.length / 2 + (*s).upperCaseMapping.length;
                    foreach(ch; (*s).upperCaseMapping) {
                        output[produced++] = ch;
                    }
                }
                continue;
            }
            if(produced >= output.length)
                output.length = output.length + output.length / 2;
            output[produced++] = d is null ? orig:(*d).simpleUpperCaseMapping;
        }
    return output[0..produced];
}


/**
 * Converts an Utf8 String to Lower case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
char[] toLower(char[] input, char[] output = null) {

    dchar[1] buf;
    // assume most common case: String stays the same length
    if (output.length < input.length)
        output.length = input.length;

    size_t produced = 0;
    size_t ate;
    foreach(dchar ch; input) {
        // TODO Conditional Case Mapping
        UnicodeData **d = (ch in unicodeData);
        if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
            SpecialCaseData **s = (ch in specialCaseData);
            debug {
                assert(s !is null);
            }
            if((*s).lowerCaseMapping !is null) {
                // To speed up, use worst case for memory prealocation
                // since the length of an LowerCaseMapping list is at most 4
                // Make sure no relocation is made in the toUtf8 Method
                // better allocation algorithm ?
                if(produced + (*s).lowerCaseMapping.length * 4 >= output.length)
                        output.length = output.length + output.length / 2 +  (*s).lowerCaseMapping.length * 4;
                char[] res = toUtf8((*s).lowerCaseMapping, output[produced..output.length], &ate);
                debug {
                    assert(ate == (*s).lowerCaseMapping.length);
                    assert(res.ptr == output[produced..output.length].ptr);
                }
                produced += res.length;
                continue;
            }           
        } 
        // Make sure no relocation is made in the toUtf8 Method
        if(produced + 4 >= output.length)
            output.length = output.length + output.length / 2 + 4;
        buf[0] = d is null ? ch:(*d).simpleLowerCaseMapping;
        char[] res = toUtf8(buf, output[produced..output.length], &ate);
        debug {
            assert(ate == 1);
            assert(res.ptr == output[produced..output.length].ptr);
        }
        produced += res.length;
    }
    return output[0..produced];
}


/**
 * Converts an Utf16 String to Lower case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
wchar[] toLower(wchar[] input, wchar[] output = null) {

    dchar[1] buf;
    // assume most common case: String stays the same length
    if (output.length < input.length)
        output.length = input.length;

    size_t produced = 0;
    size_t ate;
    foreach(dchar ch; input) {
        // TODO Conditional Case Mapping
        UnicodeData **d = (ch in unicodeData);
        if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
            SpecialCaseData **s = (ch in specialCaseData);
            debug {
                assert(s !is null);
            }
            if((*s).lowerCaseMapping !is null) {
                // To speed up, use worst case for memory prealocation
                // Make sure no relocation is made in the toUtf8 Method
                // better allocation algorithm ?
                if(produced + (*s).lowerCaseMapping.length * 2 >= output.length)
                    output.length = output.length + output.length / 2 +  (*s).lowerCaseMapping.length * 3;
                wchar[] res = toUtf16((*s).lowerCaseMapping, output[produced..output.length], &ate);
                debug {
                    assert(ate == (*s).lowerCaseMapping.length);
                    assert(res.ptr == output[produced..output.length].ptr);
                }
                produced += res.length;
                continue;
            }
        }
        // Make sure no relocation is made in the toUtf8 Method
        if(produced + 4 >= output.length)
            output.length = output.length + output.length / 2 + 3;
        buf[0] = d is null ? ch:(*d).simpleLowerCaseMapping;
        wchar[] res = toUtf16(buf, output[produced..output.length], &ate);
        debug {
            assert(ate == 1);
            assert(res.ptr == output[produced..output.length].ptr);
        }
        produced += res.length;
    }
    return output[0..produced];
}

/**
 * Converts an Utf32 String to Lower case
 * 
 * Params:
 *     input = String to be case mapped
 *     output = this output buffer will be used unless too small
 * Returns: the case mapped string
 */
dchar[] toLower(dchar[] input, dchar[] output = null) {

    // assume most common case: String stays the same length
    if (input.length > output.length)
        output.length = input.length;

    size_t produced = 0;
    if (input.length)
        foreach(dchar orig; input) {
            // TODO Conditional Case Mapping
            UnicodeData **d = (orig in unicodeData);
            if(d !is null && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.SpecialMapping)) {
                SpecialCaseData **s = (orig in specialCaseData);
                debug {
                    assert(s !is null);
                }
                if((*s).lowerCaseMapping !is null) {
                    // Better resize strategy ???
                    if(produced + (*s).lowerCaseMapping.length  > output.length)
                        output.length = output.length + output.length / 2 + (*s).lowerCaseMapping.length;
                    foreach(ch; (*s).lowerCaseMapping) {
                        output[produced++] = ch;
                    }
                }
                continue;
            }
            if(produced >= output.length)
                output.length = output.length + output.length / 2;
            output[produced++] = d is null ? orig:(*d).simpleLowerCaseMapping;
        }
    return output[0..produced];
}




/**
 * Determines if a character is a digit. It returns true for decimal
 * digits only.
 * 
 * Params:
 *     ch = the character to be inspected
 */
bool isDigit(dchar ch) {
    UnicodeData **d = (ch in unicodeData);
    return (d !is null) && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.Nd);
}


/**
 * Determines if a character is a letter.
 * 
 * Params:
 *     ch = the character to be inspected
 */
bool isLetter(int ch) {
    UnicodeData **d = (ch in unicodeData);
    return (d !is null) && ((*d).generalCategory &
        ( UnicodeData.UnicodeData.GeneralCategory.Lu
        | UnicodeData.UnicodeData.GeneralCategory.Ll
        | UnicodeData.UnicodeData.GeneralCategory.Lt
        | UnicodeData.UnicodeData.GeneralCategory.Lm
        | UnicodeData.UnicodeData.GeneralCategory.Lo));
}

/**
 * Determines if a character is a letter or a
 * decimal digit.
 * 
 * Params:
 *     ch = the character to be inspected
 */
bool isLetterOrDigit(int ch) {
    UnicodeData **d = (ch in unicodeData);
    return (d !is null) && ((*d).generalCategory & 
        ( UnicodeData.UnicodeData.GeneralCategory.Lu
        | UnicodeData.UnicodeData.GeneralCategory.Ll
        | UnicodeData.UnicodeData.GeneralCategory.Lt
        | UnicodeData.UnicodeData.GeneralCategory.Lm
        | UnicodeData.UnicodeData.GeneralCategory.Lo
        | UnicodeData.UnicodeData.GeneralCategory.Nd));
}

/**
 * Determines if a character is a lower case letter.
 * Params:
 *     ch = the character to be inspected
 */
bool isLower(dchar ch) {
    UnicodeData **d = (ch in unicodeData);
    return (d !is null) && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.Ll);
}

/**
 * Determines if a character is a title case letter.
 * Params:
 *     ch = the character to be inspected
 */
bool isTitle(dchar ch) {
    UnicodeData **d = (ch in unicodeData);
    return (d !is null) && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.Lt);
}

/**
 * Determines if a character is a upper case letter.
 * Params:
 *     ch = the character to be inspected
 */
bool isUpper(dchar ch) {
    UnicodeData **d = (ch in unicodeData);
    return (d !is null) && ((*d).generalCategory & UnicodeData.UnicodeData.GeneralCategory.Lu);
}

/**
 * Determines if a character is a Whitespace character.
 * Whitespace characters are characters in the 
 * General Catetories Zs, Zl, Zp without the No Break
 * spaces plus the control characters out of the ASCII
 * range, that are used as spaces:
 * TAB VT LF FF CR FS GS RS US NL
 * 
 * WARNING: look at isSpace, maybe that function does
 *          more what you expect.
 * 
 * Params:
 *     ch = the character to be inspected
 */
bool isWhitespace(dchar ch) {
    if((ch >= 0x0009 && ch <= 0x000D) || (ch >= 0x001C && ch <= 0x001F))
        return true;
    UnicodeData **d = (ch in unicodeData);
    return (d !is null) && ((*d).generalCategory & 
            ( UnicodeData.UnicodeData.GeneralCategory.Zs
            | UnicodeData.UnicodeData.GeneralCategory.Zl
            | UnicodeData.UnicodeData.GeneralCategory.Zp))
            && ch != 0x00A0 // NBSP
            && ch != 0x202F // NARROW NBSP
            && ch != 0xFEFF; // ZERO WIDTH NBSP 
}

/**
 * Detemines if a character is a Space character as
 * specified in the Unicode Standart.
 * 
 * WARNING: look at isWhitepace, maybe that function does
 *          more what you expect.
 * 
 * Params:
 *     ch = the character to be inspected
 */
bool isSpace(dchar ch) {
    UnicodeData **d = (ch in unicodeData);
    return (d !is null) && ((*d).generalCategory & 
            ( UnicodeData.UnicodeData.GeneralCategory.Zs
            | UnicodeData.UnicodeData.GeneralCategory.Zl
            | UnicodeData.UnicodeData.GeneralCategory.Zp));
}


/**
 * Detemines if a character is a printable character as
 * specified in the Unicode Standart.
 * 
 * 
 * WARNING: look at isWhitepace, maybe that function does
 *          more what you expect.
 * 
 * Params:
 *     ch = the character to be inspected
 */
bool isPrintable(dchar ch) {
    UnicodeData **d = (ch in unicodeData);
    return (d !is null) && ((*d).generalCategory & 
            ( UnicodeData.UnicodeData.GeneralCategory.Cn
            | UnicodeData.UnicodeData.GeneralCategory.Cc
            | UnicodeData.UnicodeData.GeneralCategory.Cf
            | UnicodeData.UnicodeData.GeneralCategory.Co
            | UnicodeData.UnicodeData.GeneralCategory.Cs));
}


debug (UnitTest) {       

unittest {
    
    
    // 1) No Buffer passed, no resize, no SpecialCase
    
    char[] testString1utf8 = "\u00E4\u00F6\u00FC";  
    wchar[] testString1utf16 = "\u00E4\u00F6\u00FC";
    dchar[] testString1utf32 = "\u00E4\u00F6\u00FC";
    char[] refString1utf8 = "\u00C4\u00D6\u00DC";
    wchar[] refString1utf16 = "\u00C4\u00D6\u00DC";
    dchar[]