Changeset 575

Timestamp:

02/13/08 13:16:04 (10 months ago)

Author:

Janice Caron

Message:

Various changes to ddoc comments
Added function utfName!(T)()
Fixed sanitize to recognize whole invalid sequences

Files:

• candidate/phobos/std/utf2.d (modified) (19 diffs)

candidate/phobos/std/utf2.d

@@ -220 +220 @@
-        enum MAX_SEQUENCE_LENGTH = 4;
-
+        invariant(char)[] utfName = "UTF-8";
+
-        byte[256] tailTable =
-        [
@@ -240 +242 @@
-        ];
-
+        bool isValidCodeUnit(T c)
+        {
+            return c < 0x80 || tails(c) >= 0;
+        }
+
-        int tails(T c)
-        {
@@ -248 +255 @@
-        {
-            return c < 0x80;
+        }
+
+        bool isHead(T c)
+        {
+            return tails(c) > 0;
-        }
-
@@ -274 +286 @@
-        alias wstring string;
-
+        invariant(char)[] utfName = "UTF-16";
+
+        bool isValidCodeUnit(T c)
+        {
+            return true;
+        }
+
-        enum MAX_SEQUENCE_LENGTH = 2;
-
@@ -286 +305 @@
-        }
-
+        bool isHead(T c)
+        {
+            return c >= 0xD800 && c < 0xDC00;
+        }
+
-        bool isTail(T c)
-        {
@@ -307 +331 @@
-        enum MAX_SEQUENCE_LENGTH = 1;
-
+        invariant(char)[] utfName = "UTF-32";
+
-        alias isValidCodepoint isSingle;
-
+        alias isValidCodepoint isValidCodeUnit;
+
-        int tails(T c)
-        {
@@ -314 +342 @@
-        }
-
+        bool isHead(T c)
+        {
+            return false;
+        }
+
-        bool isTail(T c)
-        {
@@ -328 +361 @@
-            s ~= cast(T)0xFFFD;
-        }
+    }
+
+    uint pseudoSequenceLength(string s)
+    {
+        assert(s.length != 0);
+        if (!isValidCodeUnit(s[0])) return 1;
+        int i = isHead(s[0]) ? 1 : 0;
+        for (; i < s.length; ++i)
+        {
+            if (!isTail(s[i])) break;
+        }
+        assert(i != 0);
+        return i;
-    }
-
@@ -409 +455 @@
-        {
-            appendReplacementChar(r);
-++i
+i += pseudoSequenceLength(s[i..$])
-            uint n = validatePartial(s[i..$]);
-            r ~= s[i..i+n];
@@ -694 +740 @@
-
-/**
+ * Returns the name of a UTF.
+ *
+ * The type of the output cannot be deduced. Therefore, it is necessary to explicitly
+ * specify the output type.
+ *
+ * If the template type is char, this function returns the string "UTF-8";
+ * If the template type is wchar, this function returns the string "UTF-16";
+ * If the template type is dchar, this function returns the string "UTF-32";
+ *
+ * Standards: Unicode 5.0
+ */
+string utfName(T)()
+{
+    return UTF!(T).utfName;
+}
+
+unittest
+{
+    assert(utfName!(char) == "UTF-8");
+    assert(utfName!(wchar) == "UTF-16");
+    assert(utfName!(dchar) == "UTF-32");
+}
+
+/**
- * Returns true if the character is a valid codepoint
- *
@@ -713 +783 @@
-
-/**
-
+encoded with 
- *
- * If the input is a string, this function tests for UTF-8;
@@ -738 +808 @@
- *
- * If the input string is already valid, this function returns the original, otherwise
-
+ sequence
- * Unicode replacement character, U-FFFD.
- *
@@ -755 +825 @@
-}
-
+unittest
+{
+    assert(sanitize("hello \xF0\x80world") == "hello \xEF\xBF\xBDworld");
+}
+
-/**
- * Returns the slice of the input string from the first character to the end of the
- * first UTF sequence. The resulting string may consist of multiple code units, but
-(Only if the input is empty
-will it return the empty string).
+If the input is the empty string,
+the return value will be the empty string
- *
- * The input to this function MUST be valid UTF.
+ * This is enforced by the function's in-contract.
- *
- * If the input is a string, this function operates in UTF-8;
@@ -780 +856 @@
- * Returns the slice of the input string from the start of the last UTF sequence
- * to the end of the string. The resulting string may consist of multiple code units,
-(Only if the input is empty
-will it return the empty string).
+If the input is the empty string,
+the return value will be the empty string
- *
- * The input to this function MUST be valid UTF.
+ * This is enforced by the function's in-contract.
- *
- * If the input is a string, this function operates in UTF-8;
@@ -803 +880 @@
- *
- * The input to this function MUST be valid UTF.
+ * This is enforced by the function's in-contract.
- *
- * If the input is a string, this function operates in UTF-8;
@@ -824 +902 @@
- *
- * The input to this function MUST be valid UTF.
+ * This is enforced by the function's in-contract.
- *
- * If the input is a string, this function operates in UTF-8;
@@ -845 +924 @@
- *
- * The input to this function MUST be valid UTF.
+ * This is enforced by the function's in-contract.
- *
- * If the input is a string, this function decodes UTF-8;
@@ -900 +980 @@
- *
- * The input to this function MUST be valid UTF.
+ * This is enforced by the function's in-contract.
- *
- * You can foreach either
@@ -970 +1051 @@
- *
- * The input to this function MUST be valid UTF.
+ * This is enforced by the function's in-contract.
- *
- * This function supercedes std.utf.toUTF8(), std.utf.toUTF16() and std.utf.toUTF32().