Ticket #641: reader.pl

#!/usr/bin/perl -w

use strict;

open(FILE ,"<SpecialCasing.txt") || die "Unable to open SpecialCasing.txt file";

my %data;
while(my $i = <FILE>) {
    $i =~ s/\s*#.*//g;
    next if $i =~ /^\s*$/;
    
    my @line = split /\s*;\s*/, $i;
    if($#line >= 4 && length($line[4]) > 0) {
        print "Conditional Mapping skipped: ".$i;
        next;
    }
    $data{$line[0]} = $i;
}

close(FILE);


open(FILE ,"<UnicodeData.txt") || die "Unable to open file";
open(OUTPUT, ">src/UnicodeData.d") || die "Unable to open output file";

print OUTPUT <<EOF
/*******************************************************************************

        copyright:      Copyright (c) 2007 Peter Triller. All rights reserved

        license:        BSD style: $(LICENSE)

        version:        Initial release: Sept 2007

        authors:        Peter

        Contains the Unicode Data files converted to structs and simple
        accessor functions. This file is generated by a Perl script. All
        necessary changes should be made in the script, not in this file. 

*******************************************************************************/

module UnicodeData;

struct UnicodeData {
    
    enum GeneralCategory {
        Lu = 1 <<  0, //  Letter, Uppercase
        Ll = 1 <<  1, //  Letter, Lowercase
        Lt = 1 <<  2, //  Letter, Titlecase
        Lm = 1 <<  3, //  Letter, Modifier
        Lo = 1 <<  4, //  Letter, Other
        Mn = 1 <<  5, //  Mark, Nonspacing
        Mc = 1 <<  6, //  Mark, Spacing Combining
        Me = 1 <<  7, //  Mark, Enclosing
        Nd = 1 <<  8, //  Number, Decimal Digit
        Nl = 1 <<  9, //  Number, Letter
        No = 1 << 10, //  Number, Other
        Pc = 1 << 11, //  Punctuation, Connector
        Pd = 1 << 12, //  Punctuation, Dash
        Ps = 1 << 13, //  Punctuation, Open
        Pe = 1 << 14, //  Punctuation, Close
        Pi = 1 << 15, //  Punctuation, Initial quote (may behave like Ps or Pe depending on usage)
        Pf = 1 << 16, //  Punctuation, Final quote (may behave like Ps or Pe depending on usage)
        Po = 1 << 17, //  Punctuation, Other
        Sm = 1 << 18, //  Symbol, Math
        Sc = 1 << 19, //  Symbol, Currency
        Sk = 1 << 20, //  Symbol, Modifier
        So = 1 << 21, //  Symbol, Other
        Zs = 1 << 22, //  Separator, Space
        Zl = 1 << 23, //  Separator, Line
        Zp = 1 << 24, //  Separator, Paragraph
        Cc = 1 << 25, //  Other, Control
        Cf = 1 << 26, //  Other, Format
        Cs = 1 << 27, //  Other, Surrogate
        Co = 1 << 28, //  Other, Private Use
        Cn = 1 << 29, //  Other, Not Assigned (no characters in the file have this property)
        SpecialMapping = 1 << 30 // Special Bit for detection of specialMappings
    }
    
    
    enum BidiClass {
        L   = 1 <<  0, //  Left-to-Right
        LRE = 1 <<  1, //  Left-to-Right Embedding
        LRO = 1 <<  2, //  Left-to-Right Override
        R   = 1 <<  3, //  Right-to-Left
        AL  = 1 <<  4, //  Right-to-Left Arabic
        RLE = 1 <<  5, //  Right-to-Left Embedding
        RLO = 1 <<  6, //  Right-to-Left Override
        PDF = 1 <<  7, //  Pop Directional Format
        EN  = 1 <<  8, //  European Number
        ES  = 1 <<  9, //  European Number Separator
        ET  = 1 << 10, //  European Number Terminator
        AN  = 1 << 11, //  Arabic Number
        CS  = 1 << 12, //  Common Number Separator
        NSM = 1 << 13, //  Non-Spacing Mark
        BN  = 1 << 14, //  Boundary Neutral
        B   = 1 << 15, //  Paragraph Separator
        S   = 1 << 16, //  Segment Separator
        WS  = 1 << 17, //  Whitespace
        ON  = 1 << 18  //  Other Neutrals
    }
    
    enum DecompositionType {
        None     = 1 <<  0, // Custom type signaling no Decomposition
        Font     = 1 <<  1, //    A font variant (e.g. a blackletter form).
        NoBreak  = 1 <<  2, //    A no-break version of a space or hyphen.
        Initial  = 1 <<  3, //    An initial presentation form (Arabic).
        Medial   = 1 <<  4, //    A medial presentation form (Arabic).
        Final    = 1 <<  5, //    A final presentation form (Arabic).
        Isolated = 1 <<  6, //    An isolated presentation form (Arabic).
        Circle   = 1 <<  7, //    An encircled form.
        Super    = 1 <<  8, //    A superscript form.
        Sub      = 1 <<  9, //    A subscript form.
        Vertical = 1 << 10, //    A vertical layout presentation form.
        Wide     = 1 << 11, //    A wide (or zenkaku) compatibility character.
        Narrow   = 1 << 12, //    A narrow (or hankaku) compatibility character.
        Small    = 1 << 13, //    A small variant form (CNS compatibility).
        Square   = 1 << 14, //    A CJK squared font variant.
        Fraction = 1 << 15, //    A vulgar fraction form.
        Compat   = 1 << 16  //    Otherwise unspecified compatibility character.
    }
    
    enum BidiMirrored {
        Y = 1, // Yes
        N = 2 // No
    }
    
    dchar code;
    
//    char[] name;
    
    GeneralCategory generalCategory;
    
//    short canonicalCombiningClass;
    
    //TODO the defaults are not yet set correctly
    
//    BidiClass bidiClass;
    
    //TODO end
    
//    DecompositionType decompositionType;
    
//    dchar[] decompositionMapping;
    
    
    // TODO Check handling
    
//    int numeric_1;
    
//    int numeric_2;
    
//    double numeric_3;
    
    // TODO end
    
//    BidiMirrored bidiMirrored;
    
//    char[] unicode1Name;
    
//    char [] isoComment;
    
    dchar simpleUpperCaseMapping;
    
    dchar simpleLowerCaseMapping;
    
    dchar simpleTitleCaseMapping;
      
}

struct SpecialCaseData {
    
    dchar code;
    
    dchar[] upperCaseMapping;

    dchar[] lowerCaseMapping;

    dchar[] titleCaseMapping;

}

struct FoldingCaseData {
    
    dchar code;
    
    dchar[] mapping;
    
}

UnicodeData *unicodeData[dchar];

SpecialCaseData *specialCaseData[dchar];

FoldingCaseData *foldingCaseData[dchar];

private {
    UnicodeData internalUnicodeData[] = [
EOF
;
my $first = 0;
my $last = -1;
my $num;
while(my $i = <FILE>) {
    my @line = split /;/, $i;
    
    foreach my $i (@line) {
        $i =~ s/^\s+//;
        $i =~ s/\s+$//;    
    }
    
    my $ucm = (length($line[12]) > 0 ? $line[12]:$line[0]);

    print $num."\n";
    $num++;

    print OUTPUT " { code:0x$line[0]\n";
    
    print OUTPUT "  ,generalCategory:UnicodeData.GeneralCategory.".(length($line[2]) > 0? $line[2] :"Cn");
    
    if(exists $data{$line[0]}) {
        print OUTPUT " | UnicodeData.GeneralCategory.SpecialMapping"
    }
    print OUTPUT "\n";

    print OUTPUT "  ,simpleUpperCaseMapping:0x".$ucm."\n";
    
    print OUTPUT "  ,simpleLowerCaseMapping:0x".(length($line[13]) > 0 ? $line[13]:$line[0])."\n";

    print OUTPUT "  ,simpleTitleCaseMapping:0x".(length($line[14]) > 0 ? $line[14]:$ucm)."\n";

    print OUTPUT " },\n"
}
print OUTPUT "\n];\n";

print OUTPUT "    SpecialCaseData internalSpecialCaseData[] = [\n";

for my $key (sort keys %data) {
    
    my @line = split /\s*;\s*/, $data{$key};
    
    
    print OUTPUT " { code:0x$line[0]\n";
    print OUTPUT "  ,upperCaseMapping:";
    my @uc = split /\s+/, $line[3];
    if(length(@uc) > 0) {
        print OUTPUT "[ 0x". (join ", 0x" , @uc). " ]\n";
    } else {
        print OUTPUT "null\n";
    }
    print OUTPUT "  ,lowerCaseMapping:";
    my @lc = split /\s+/, $line[1];
    if(length(@lc) > 0) {
        print OUTPUT "[ 0x". (join ", 0x" , @lc). " ]\n";
    } else {
        print OUTPUT "null;\n";
    }
    print OUTPUT "  ,titleCaseMapping:";
    my @tc = split /\s+/, $line[2];
    if(length(@tc) > 0) {
        print OUTPUT "[ 0x". (join ", 0x" , @tc). " ]\n";
    } else {
        print OUTPUT "null\n";
    }
    
    print OUTPUT "\n },\n";
}
print OUTPUT "\n];\n";

close(FILE);

open(FILE ,"<CaseFolding.txt") || die "Unable to open CaseFolding.txt file";

print OUTPUT "    FoldingCaseData internalFoldingCaseData[] = [\n";

while(my $i = <FILE>) {
    $i =~ s/\s*#.*//g;
    next if $i =~ /^\s*$/;
    
    my @line = split /\s*;\s*/, $i;

    next if $line[1] eq "S";
    
    my @mapping = split / +/, $line[2];
    
    print OUTPUT " {\n";
    
    print OUTPUT "   code:0x$line[0]\n";
    print OUTPUT "  ,mapping: [ 0x".(join ", 0x" , @mapping)." ]\n";
    print OUTPUT " },\n";
}

print OUTPUT "\n];\n";
close(FILE);

print OUTPUT "\n}\n\n";

print OUTPUT <<EOF
static this() {
    foreach(inout entry; internalUnicodeData)
        unicodeData[entry.code] = &entry;
    unicodeData.rehash;
    
    foreach(inout entry; internalSpecialCaseData)
        specialCaseData[entry.code] = &entry;
    specialCaseData.rehash;
    
    foreach(inout entry; internalFoldingCaseData)
        foldingCaseData[entry.code] = &entry;
    foldingCaseData.rehash;
}
EOF
;

close(OUTPUT);