Changeset 37

Timestamp:

02/21/08 15:14:27 (11 months ago)

Author:

aarti_pl

Message:

- Tokenizer -> Scanner
- new module Storage
- reworked scanning functions
- more precise description of symbols (not everything is string)
- escaper module has ability to escape named characters (as an option)
- possibility to unescape \x, \u, \U. \oct etc.

Files:

• trunk/doost/core/Traits.d (modified) (1 diff)
• trunk/doost/storage (added)
• trunk/doost/storage/FileStorage.d (added)
• trunk/doost/storage/Storage.d (added)
• trunk/doost/text/Escaper.d (modified) (5 diffs)
• trunk/doost/text/Scanner.d (added)
• trunk/doost/text/Tokenizer.d (deleted)
• trunk/doost/util/DUnit.d (modified) (15 diffs)
• trunk/doost/util/serializer/Serializer.d (modified) (7 diffs)
• trunk/doost/util/serializer/archive/JsonArchive.d (modified) (36 diffs)
• trunk/doost/util/serializer/archive/TextArchive.d (modified) (39 diffs)
• trunk/examples/text/FunctionTest.d (deleted)
• trunk/examples/text/PackageTest.d (added)
• trunk/examples/text/text.cbp (modified) (3 diffs)
• trunk/examples/util/serializer/FunctionTest.d (modified) (13 diffs)
• trunk/examples/util/serializer/FunctionTest1.d (modified) (4 diffs)
• trunk/examples/util/serializer/util_serializer.cbp (modified) (4 diffs)

trunk/doost/core/Traits.d

@@ -37 +37 @@
-    else
-        static assert(false, "'" ~ T.stringof ~ "' is not an array.");
-}
-
-/*******************************************************************************
-    Evaluates to type of array element
- ******************************************************************************/
-//TODO: checki na spełnianie wymagań co do array
-template elementType(T) {
-    static if( is( T U : U[] ) ) {
-        pragma(msg, "elementType: " ~ T.stringof ~ " -> " ~ U.stringof);
-        alias U elementType;
-    } else
-    static if( is( typeof(T.get(1u)) U == U[]) ) {
-        pragma(msg, "elementType: " ~ T.stringof ~ " -> " ~ U.stringof);
-        alias U elementType;
-    } else
-    static assert(false, "'" ~ T.stringof ~ "' can not be threated as array.");
-}
-

trunk/doost/text/Escaper.d

@@ +1 @@
+/*******************************************************************************
+
+    License:    Boost Software License, v. 1.0
+                Academic Free License, v. 3.0
+                BSD License
+
+    Authors:    Marcin Kuszczak, www.zapytajmnie.com (author's christian site)
+
+    Version:    0.9.0
+    Date:       18 Jan 2008
+
+    History:    0.9.0   -   initial public beta version
+
+ ******************************************************************************/
+
-module doost.text.Escaper;
-
+import std.stdio;
+
+import doost.util.DUnit;
-import doost.core.Traits;
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-class UnescapeException : Exception {
-    this(string str) {
@@ -16 +40 @@
-}
-
+//------------------------------------------------------------------------------
+
-/*******************************************************************************
+    Params:
+            full = replaces also characters using named characters (slower)
- ******************************************************************************/
-str
+input, bool full = false
-    static assert(isString!(T), "Only unicode strings allowed");
-
+    uint pos;
+    Matcher m;
+    T result;
+
+    while(!input.eos(pos)) {
+        m = scan(input, SequenceSet!(T).escchar, pos);
+
+        if (cast(bool)m) {
+            result ~= input.get(pos);
+            input.get(m.position - pos);
+
+            result ~= '\\' ~ SequenceSet!(T).escseq.matching(m);
+            pos = 0;
+            continue;
+        }
+
+        if (full) {
+            m = scan(input, SequenceSet!(T).namedchar, pos);
+
+            if (cast(bool)m) {
+                result ~= input.get(pos);
+                input.get(m.position - pos);
+
+                result ~= r"\&" ~ SequenceSet!(T).namedseq.matching(m);
+                pos = 0;
+            continue;
+            }
+        }
+
+        ++pos;
+    }
+
+    result ~= input.get(pos);
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+T escapeOld(T)(T input, bool full = false) {
+    //poniŌsze jest 5x szybsze niŌ nowa implementacja
-    T result;
-str
+input
-        switch (c) {
-            case '\\'  : result ~= "\\\\"; break;
@@ -43 +112 @@
-//------------------------------------------------------------------------------
-
+unittest { testCase.repeat(1).execute("basic escape characters", {
+    assert(escape("\""[]) == "\\\"");
+    assert(escape("\?"[]) == "\\?");
+    assert(escape("\\"[]) == "\\\\");
+    assert(escape("\a"[]) == "\\a");
+    assert(escape("\b"[]) == "\\b");
+    assert(escape("\f"[]) == "\\f");
+    assert(escape("\n"[]) == "\\n");
+    assert(escape("\r"[]) == "\\r");
+    assert(escape("\t"[]) == "\\t");
+    assert(escape("\v"[]) == "\\v");
+});}
+
+//------------------------------------------------------------------------------
+
+unittest { testCase.repeat(1).execute("basic old implementation escape characters", {
+    assert(escapeOld("\""[]) == "\\\"");
+    assert(escapeOld("\?"[]) == "\\?");
+    assert(escapeOld("\\"[]) == "\\\\");
+    assert(escapeOld("\a"[]) == "\\a");
+    assert(escapeOld("\b"[]) == "\\b");
+    assert(escapeOld("\f"[]) == "\\f");
+    assert(escapeOld("\n"[]) == "\\n");
+    assert(escapeOld("\r"[]) == "\\r");
+    assert(escapeOld("\t"[]) == "\\t");
+    assert(escapeOld("\v"[]) == "\\v");
+});}
+
+//------------------------------------------------------------------------------
+
+unittest { testCase.execute("escape with named characters", {
+    //full escape with named characters
+    assert(escape("\♥"[], true) == r"\♥");
+    assert(escape("\∅"[], true) == r"\∅");
+    assert(escape("\∑"[], true) == r"\∑");
+});}
+
+//------------------------------------------------------------------------------
+
+private uint toDec(dchar c) {
+    uint code = c;
+    if (code >= 0x61) code -= 0x57;
+    if (code >= 0x41) code -= 0x37;
+    if (code >= 0x30) code -= 0x30;
+    return code;
+}
+
+private uint[] maxsize = [0u, 255, 65535, 4228250625];
+
-/*******************************************************************************
- ******************************************************************************/
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    bool isEscape=false;
-
-str
+input
-        if (isEscape==true) {
-            switch (c) {
@@ -64 +250 @@
-                case 't'  : result ~= '\t'; break;
-                case 'v'  : result ~= '\v'; break;
-
+
+
-            }
-            isEscape=false;
@@ -79 +266 @@
-    return result;
-}
+
+//------------------------------------------------------------------------------
+
+unittest { testCase.repeat(1).execute("unescape basic characters", {
+    assert(unescape("\\\\"[]) == "\\");
+    assert(unescape("\\\""[]) == "\"");
+    assert(unescape("\\?"[]) == "\?");
+    assert(unescape("\\a"[]) == "\a");
+    assert(unescape("\\b"[]) == "\b");
+    assert(unescape("\\f"[]) == "\f");
+    assert(unescape("\\n"[]) == "\n");
+    assert(unescape("\\r"[]) == "\r");
+    assert(unescape("\\t"[]) == "\t");
+    assert(unescape("\\v"[]) == "\v");
+
+    assert(unescape(r"\\\\\\\\ abc \\t\\n\\t \\t"[]) == r"\\\\ abc \t\n\t \t");
+});}
+
+//------------------------------------------------------------------------------
+
+unittest { testCase.repeat(1).execute("unescape stream wrapper", {
+    DummyStorage!(string) storage;
+
+    //storage.input = "\\\\"; assert(unescape(storage).output == "\\");
+    assert(unescape("\\\""[]) == "\"");
+    assert(unescape("\\?"[]) == "\?");
+    assert(unescape("\\a"[]) == "\a");
+    assert(unescape("\\b"[]) == "\b");
+    assert(unescape("\\f"[]) == "\f");
+    assert(unescape("\\n"[]) == "\n");
+    assert(unescape("\\r"[]) == "\r");
+    assert(unescape("\\t"[]) == "\t");
+    assert(unescape("\\v"[]) == "\v");
+
+    assert(unescape(r"\\\\\\\\ abc \\t\\n\\t \\t"[]) == r"\\\\ abc \t\n\t \t");
+});}
+
+//------------------------------------------------------------------------------
+
+unittest { testCase.repeat(1).execute("unescape basic characters old implementation", {
+    assert(unescapeOld("\\\\"[]) == "\\");
+    assert(unescapeOld("\\\""[]) == "\"");
+    assert(unescapeOld("\\?"[]) == "\?");
+    assert(unescapeOld("\\a"[]) == "\a");
+    assert(unescapeOld("\\b"[]) == "\b");
+    assert(unescapeOld("\\f"[]) == "\f");
+    assert(unescapeOld("\\n"[]) == "\n");
+    assert(unescapeOld("\\r"[]) == "\r");
+    assert(unescapeOld("\\t"[]) == "\t");
+    assert(unescapeOld("\\v"[]) == "\v");
+
+    assert(unescapeOld(r"\\\\\\\\ abc \\t\\n\\t \\t"[]) == r"\\\\ abc \t\n\t \t");
+});}
+
+//------------------------------------------------------------------------------
+
+unittest { testCase.execute("unescape extended characters", {
+    // 2 characters hexadecimal test (\x)
+    assert(unescape(r"\x41"[]) == "A"[]);
+    assert(unescape(r"\x41\x42\x43"[]) == "ABC"[]);
+
+    // 1-3 characters octal test (\ooo)
+    assert(unescape(r"\101"[]) == "A"[]);
+    assert(unescape(r"\101\102\103"[]) == "ABC"[]);
+    //Unknown escape sequence
+    assert(checkException!(UnescapeException)({unescape(r"\999"[]);}));
+    //Out of char range
+    assert(checkException!(UnescapeException)({unescape(r"\777"[]);}));
+    assert(unescape(r"\377"[]) == "\xff");
+
+    //4 hex digits
+    assert(unescape(r"\u4142"[]) == "AB"[]);
+    assert(unescape(r"\uC582"[]) == "ł"[]);
+
+    //8 hex digits
+    assert(unescape(r"\U41424344"[]) == "ABCD"[]);
+
+    //named chars
+    assert(unescape(r"\""[]) == "\"");
+    assert(unescape(r"\©"[]) == "\©");
+    assert(unescape(r"\Λ"[]) == "\Λ");
+    assert(unescape(r"\♥"[]) == "\♥");
+});}
+
+//------------------------------------------------------------------------------
+
+unittest { testCase.execute("escape/unescape characters", {
+    string input;
+
+    input = r"\\\\ abc \t\n\t \t"; assert(unescape(escape(input)) == input);
+    input = r"\\d\te\\f"; assert(unescape(escape(input))==input);
+    input = r"\\g\r\t\nhi"; assert(unescape(escape(input))==input);
+    input = r"\\jk;lkds[]\tl"; assert(unescape(escape(input))==input);
+    input = r"\fmno"; assert(unescape(escape(input))==input);
+    input = r"\r\npqr"; assert(unescape(escape(input))==input);
+    input = r"\\stu"; assert(unescape(escape(input))==input);
+    input = r"\\vwq"; assert(unescape(escape(input))==input);
+    input = r"\\xyz"; assert(unescape(escape(input))==input);
+});}
+
+
+//------------------------------------------------------------------------------
+
+unittest {
+    testSuite.finish;
+}

trunk/doost/util/DUnit.d

@@ -42 +42 @@
-public:
-    this(uint rwidth, Align aligning, string form="%s", uint border = 1) {
+        m_rwidth = rwidth;
-        m_cwidth = rwidth;
-        m_aligning = aligning;
@@ -83 +84 @@
-private:
-    uint m_cwidth;
+    uint m_rwidth;
-    Align m_aligning;
-    uint m_border;
@@ -187 +189 @@
-            if (auto elem = cast(CellElem)e) {
-                ccount++;
-c
+r
-                    if (dyncol !is null) throw new Exception("Only one column can have dynamic width");
-                    dyncol = elem;
-c
+r
-            }
-        }
@@ -237 +239 @@
-
-//------------------------------------------------------------------------------
+
+ulong rdtsc() {
+    asm
+    {
+        naked;
+        rdtsc;
+        ret;
+    }
+}
-
-struct TraceInfo {
@@ -280 +291 @@
-        m_testname = testname;
-        m_passed = false;
-real.nan
+0
-
-        int no;
@@ -290 +301 @@
-            m_number = m_owner.m_tccounter;
-            try {
-d_time starttime = getUTCtime()
+long starttime = rdtsc
-                for(uint i = 0; i<m_repeat; i++) {
-                    if (m_setup !is null) m_setup();
@@ -296 +307 @@
-                    if (m_teardown !is null) m_teardown();
-                }
-(cast(real)(getUTCtime() - starttime))/1000
+rdtsc - starttime
-                result = "OK";
-                m_passed = true;
@@ -306 +317 @@
-                m_owner.m_tcfailed~=this;
-                no = 0;
+                m_elapsedtime = 0;
-            }
-        } else result = "DISABLED";
@@ -317 +329 @@
-
-        //BUG: przy sprawdzaniu nan is oraz !is powinno działać
-if (!isNaN(m_elapsedtime)) 
+
-    }
-
@@ -378 +390 @@
-    bool m_dotraces;
-
-real
+long
-    uint m_number;
-    TestSuite m_owner;
@@ -449 +461 @@
-
-    void begin(string suitename) {
+
-        m_suitename = suitename;
+        m_firstingroup=true;
+        m_elapsedtime=0;
+        m_curgroup = null;
+        m_testgroups = null;
+        m_curcase = null;
+        m_testcases = null;
+        m_tcfailed = null;
-        m_tccounter = 0;
-        m_tcfcounter = 0;
-        m_tcfailed = null;
-        m_elapsedtime=0;
-
-        renderer.render(hl);
@@ -490 +508 @@
-        renderer.render(hl);
-        renderer.render(simple);
-me
+cks
-        renderer.render(stat,  " -    all  :", m_tccounter);
-        renderer.render(stat,  " - passed  :", tcpcounter, cast(double)(tcpcounter*100)/m_tccounter);
@@ -499 +517 @@
-        renderer.render(empty);
-
+        m_setup = null;
+        m_teardown = null;
+        m_dotiming = false;
+        m_dotraces = false;
+        m_reqtraces = false;
+        m_repeat = 1;
-    }
-
@@ -545 +569 @@
-        title   = [Sep, Cell(-1, Align.Center), Sep];
-        tc      = [Sep, Cell(5, Align.Right, "%-s", 0), Cell(-1, Align.Left), Cell(7, Align.Right), Sep];
-8, Align.Right), Sep, Cell(8, Align.Right, "%6.2f
+7, Align.Right), Sep, Cell(10, Align.Right, "%d
-        simple  = [Sep, Cell(-1, Align.Left), Sep];
-0, Align.Left, "%6.4f s
+5, Align.Right, "%d
-        stat    = [Sep, Cell(15, Align.Right), Cell(6, Align.Left), Cell(-1, Align.Left, "(%6.2f %%)"), Sep];
-        fail    = [Sep, Cell(5, Align.Right, "%d."), Cell(-1, Align.Left), Sep];
@@ -609 +633 @@
-    TestGroup m_curgroup;
-    TestGroup[] m_testgroups;
-real
+long
-
-    uint m_tccounter, m_tcfcounter;

trunk/doost/util/serializer/Serializer.d

@@ -18 +18 @@
-import doost.api.Common;
-import doost.core.Traits;
+import doost.storage.Storage;
-import doost.util.serializer.Registry;
-
@@ -88 +89 @@
-
-    enum Mode {LOAD, DUMP}
-
+
+
-
-    mixin CONCRETEARCHIVE!(Archive);
@@ -104 +106 @@
-    this() {
-        m_archive = new Archive;
-
+
+
+
-    }
-
@@ -143 +147 @@
-
-        archive.mode = Mode.DUMP;
-        archive.storage = STORAGE.init;
-
-        start(archive);
@@ -169 +172 @@
-    VALUE load(VALUE)(STORAGE storage, Archive archive = null) {
-        //TODO: static asserts in serializer --> check if all template functions exists in Archive
-
-        if (archive is null) archive = m_archive.dup_info;
-            else archive = archive.dup_info;
@@ -285 +287 @@
-            "'. Types like void*, union, enum are not serializable without additional information");
-        }
-e
+y
-        //loadUdt/dumpUdt
-    }
@@ -371 +373 @@
-                }
-            } else {
-                //FIXME:
-                static assert(false, "UDT '" ~ VALUE.stringof ~ "' is not serializable.");
-            }

trunk/doost/util/serializer/archive/JsonArchive.d

@@ -39 +39 @@
-    import doost.util.serializer.Registry;
-    import doost.text.Escaper;
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-    //------------------------------------------------------------------------------
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-    //--------------------------------------------------------------------------
@@ -142 +96 @@
-        uint indent = 0;
-
-zawartość[nazwa pola
-          //fields of specific class with values;
+fields of specific class with values: content[name of fields
+
-
-        Archive dup_infoext() {
@@ -159 +113 @@
-    static class TypeDescription(T) : Registration {
-        mixin TypeDescriptionBase!(T) DEFAULT;
-        public Pattern!(T) def;
-
-        public typeof(this) dup_info() {
-            typeof(this) result = DEFAULT.dup_info;
-            result.def = def.dup;
-            return result;
-        }
@@ -203 +155 @@
-     **************************************************************************/
-    bool loadString(VALUE)(ref VALUE value, ARCHIVE archive) {
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-        return true;
@@ -226 +176 @@
-     **************************************************************************/
-    bool dumpString(VALUE)(ref VALUE value, ARCHIVE archive) {
-to!(STORAGETYPE)(String.delim1)
+String.sdelimiter
-                            to!(STORAGETYPE)(escape(value)) ~
-to!(STORAGETYPE)(String.delim1)
-
-
-
-
-
-Position starts
-Position
-
-tarts(input, Pattern!(bool).positive); if (
-tarts(input, Pattern!(bool).negative); if (
-
-mismatch
+String.sdelimiter
+
+
+
+
+
+er scan
+er
+
+can(input, Bool.positive); if (cast(bool)
+can(input, Bool.negative); if (cast(bool)
+
+Matcher(0, Matcher.mismatch)
-    }
-
@@ -250 +200 @@
-     **************************************************************************/
-    bool loadBool(VALUE)(ref VALUE value, ARCHIVE archive) {
-_cc
+, uint.max
-        STORAGETYPE literal;
-
-
+
+
-            literal = munch(archive.storage, mp);
-        } else throw new ParsingException("Unknown bool literal.");
@@ -269 +220 @@
-     **************************************************************************/
-    bool dumpBool(VALUE)(ref VALUE value, ARCHIVE archive) {
-Pattern!(bool)
-Pattern!(bool)
+Bool
+Bool
-        return true;
-    }
@@ -277 +228 @@
-     **************************************************************************/
-    bool loadNumber(VALUE)(ref VALUE value, ARCHIVE archive) {
-_cc
-tartsWith
-
+, uint.max
+can
+cast(bool)
-
-        value = to!(VALUE)(munch(archive.storage, pos));
@@ -307 +258 @@
-     **************************************************************************/
-    bool dumpChar(VALUE)(ref VALUE value, ARCHIVE archive) {
-String.delim2 ~ value ~ String.delim2
-
-
-
-
-
-)(ref VALUE value, string
+"" ~ String.cdelimiter ~ value ~ String.cdelimiter
+
+
+
+
+
+, SEP)(ref VALUE value, SEP
-        if (archive.mismatch) throw new ParsingException("Missing elements separator.");
-        bool matched = traverse(value, archive);
-        if (!matched) return false;
-
-_cc
-new RegExp(esep)
-
-
-
-
-
-)(ref VALUE value, string
+, uint.max
+esep
+
+
+
+
+
+, SEP)(ref VALUE value, SEP
-        archive.storage.put(archive.rest);
-        bool res = traverse(value, archive);
@@ -335 +286 @@
-     **************************************************************************/
-    bool loadArray(VALUE)(ref VALUE value, ARCHIVE archive) {
-_cc
+, uint.max
-
-        if (!skip(archive.storage, Array.begin))
@@ -344 +295 @@
-        archive.mismatch = false;
-        do {
-_cc
+, uint.max
-            if (skip(archive.storage, Array.end)) break;
-            loadElement(elem, Array.separator, archive);
@@ -369 +320 @@
-    /***************************************************************************
-     **************************************************************************/
-)(ref KEY key, ref VALUE value, string kvsep, string
+, KVSEP, SEP)(ref KEY key, ref VALUE value, KVSEP kvsep, SEP
-        if (archive.mismatch) throw new ParsingException("Missing associative elements separator.");
-        bool matched = traverse(key, archive);
-        if(!matched) return false;
-
-_cc
+, uint.max
-        if (!skip(archive.storage, kvsep))
-            throw new ParsingException("Missing associative array literal key-value separator.");
@@ -380 +331 @@
-        traverse(value, archive);
-
-
-
-
+
+
-
-        return matched;
@@ -389 +339 @@
-    /***************************************************************************
-     **************************************************************************/
-)(ref KEY key, ref VALUE value, string kvsep, string
+, KVSEP, SEP)(ref KEY key, ref VALUE value, KVSEP kvsep, SEP
-        archive.storage.put(archive.rest);
-        traverse(key, archive);
@@ -404 +354 @@
-        alias typeof(VALUE.init.keys[0]) KeyType;
-
-_cc
+, uint.max
-        if (!skip(archive.storage, Udt.begin))
-            throw new ParsingException("Missing associative array literal begining delimiter.");
@@ -414 +364 @@
-        archive.mismatch = false;
-        do {
-_cc
+, uint.max
-            if (skip(archive.storage, Udt.end)) break;
-