dlexer.d

Revision 14, 35.2 kB (checked in by Jaymz031602, 7 years ago)
--
Property svn:executable set to ``*

Line
1	// dlexer - D language lexer
2	// (C) Copyright 2004-2005 James Dunne
3
4	// Please feel free to distribute this module in any form you like. I only ask that you
5	// give credit where credit is due. You may make modifications to this module. If you do
6	// so, you may provide your modifications publicly, but you are not required to.
7
8	module dlexer;
9
10	import std.stream;
11	import std.string;
12	import std.ctype;
13
14	private import std.c.stdlib; // for strtod and strtold
15
16	alias std.ctype.isdigit isdigit;
17
18	// Set this version identifier to enable interpretation of escaped characters within parsed strings
19	// If this is disabled, the strings are copied directly, escape sequences and all.
20
21	// version = interpret_slashes;
22
23	// Enumeration of D language tokens taken from DMD's lexer.h
24	enum : uint {
25	TOKreserved,
26
27	// Other
28	TOKlparen, TOKrparen,
29	TOKlbracket, TOKrbracket,
30	TOKlcurly, TOKrcurly,
31	TOKcolon, TOKneg,
32	TOKsemicolon, TOKdotdotdot,
33	TOKeof, TOKcast,
34	TOKnull, TOKassert,
35	TOKtrue, TOKfalse,
36	TOKarray, TOKcall,
37	TOKaddress, TOKtypedot,
38	TOKtype, TOKthrow,
39	TOKnew, TOKdelete,
40	TOKstar, TOKsymoff,
41	TOKvar, TOKdotvar,
42	TOKdotti, TOKdotexp,
43	TOKdottype, TOKslice,
44	TOKarraylength, TOKversion,
45	TOKmodule, TOKdollar,
46	TOKtemplate, TOKinstance,
47	TOKdeclaration, TOKtypeof,
48	TOKpragma, TOKdsymbol,
49	TOKtypeid, TOKuadd,
50
51	// Operators
52	TOKlt, TOKgt,
53	TOKle, TOKge,
54	TOKequal, TOKnotequal,
55	TOKidentity, TOKnotidentity,
56	TOKindex,
57
58	// NCEG floating point compares
59	// !<>= <> <>= !> !>= !< !<= !<>
60	TOKunord,TOKlg,TOKleg,TOKule,TOKul,TOKuge,TOKug,TOKue,
61
62	TOKshl, TOKshr,
63	TOKshlass, TOKshrass,
64	TOKushr, TOKushrass,
65	TOKcat, TOKcatass, // ~ ~=
66	TOKadd, TOKmin, TOKaddass, TOKminass,
67	TOKmul, TOKdiv, TOKmod,
68	TOKmulass, TOKdivass, TOKmodass,
69	TOKand, TOKor, TOKxor,
70	TOKandass, TOKorass, TOKxorass,
71	TOKassign, TOKnot, TOKtilde,
72	TOKplusplus, TOKminusminus,
73	TOKdot, TOKarrow, TOKcomma,
74	TOKquestion, TOKandand, TOKoror,
75
76	// Numeric literals
77	TOKint32v, TOKuns32v,
78	TOKint64v, TOKuns64v,
79	TOKfloat32v, TOKfloat64v, TOKfloat80v,
80	TOKimaginary32v, TOKimaginary64v, TOKimaginary80v,
81
82	// Char constants
83	TOKcharv, TOKwcharv, TOKdcharv,
84
85	// Leaf operators
86	TOKidentifier, TOKstring,
87	TOKthis, TOKsuper,
88
89	// Basic types
90	TOKvoid,
91	TOKint8, TOKuns8,
92	TOKint16, TOKuns16,
93	TOKint32, TOKuns32,
94	TOKint64, TOKuns64,
95	TOKfloat32, TOKfloat64, TOKfloat80,
96	TOKimaginary32, TOKimaginary64, TOKimaginary80,
97	TOKcomplex32, TOKcomplex64, TOKcomplex80,
98	TOKchar, TOKwchar, TOKdchar, TOKbit,
99
100	// Aggregates
101	TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport,
102	TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction,
103	TOKmixin,
104
105	TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport,
106	TOKstatic, /TOKvirtual,/ TOKfinal, TOKconst, TOKabstract, TOKvolatile,
107	TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout,
108	TOKauto, TOKpackage,
109
110	// Statements
111	TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch,
112	TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith,
113	TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally,
114	TOKasm, TOKforeach,
115
116	// Contracts
117	TOKbody, TOKinvariant,
118
119	// Testing
120	TOKunittest,
121
122	TOKmax
123	}
124
125	// A table converting token values into string representations:
126	char[] toktostr[TOKmax] = [
127	TOKreserved : "reserved",
128
129	// Other
130	TOKlparen : "(",
131	TOKrparen : ")",
132	TOKlbracket : "[",
133	TOKrbracket : "]",
134	TOKlcurly : "{",
135	TOKrcurly : "}",
136	TOKcolon : ":",
137	TOKneg : "-",
138	TOKsemicolon : ";",
139	TOKdotdotdot : "...",
140	TOKeof : "EOF",
141	TOKcast : "cast",
142	TOKnull : "null",
143	TOKassert : "assert",
144	TOKtrue : "true",
145	TOKfalse : "false",
146	TOKarray : "[]",
147	TOKcall : "call",
148	TOKaddress : "#",
149	TOKtypedot : "typedot",
150	TOKtype : "type",
151	TOKthrow : "throw",
152	TOKnew : "new",
153	TOKdelete : "delete",
154	TOKstar : "*",
155	TOKsymoff : "symoff",
156	TOKvar : "var",
157	TOKdotvar : "dotvar",
158	TOKdotti : "dotti",
159	TOKdotexp : "dotexp",
160	TOKdottype : "dottype",
161	TOKslice : "..",
162	TOKarraylength : "arraylength",
163	TOKversion : "version",
164	TOKmodule : "module",
165	TOKdollar : "$",
166	TOKtemplate : "template",
167	TOKinstance : "instance",
168	TOKdeclaration : "declaration",
169	TOKtypeof : "typeof",
170	TOKpragma : "pragma",
171	TOKdsymbol : "dsymbol",
172	TOKtypeid : "typeid",
173	TOKuadd : "uadd",
174
175	// Operators
176	TOKlt : "<",
177	TOKgt : ">",
178	TOKle : "<=",
179	TOKge : ">=",
180	TOKequal : "==",
181	TOKnotequal : "!=",
182	TOKidentity : "===",
183	TOKnotidentity : "!==",
184	TOKindex : "[]",
185
186	// NCEG floating point compares
187	// !<>= <> <>= !> !>= !< !<= !<>
188
189	// NOTE: These could be horribly wrong
190	TOKunord : "!<>=",
191	TOKue : "!<>",
192	TOKlg : "<>",
193	TOKleg : "<>=",
194	TOKule : "!>",
195	TOKul : "!>=",
196	TOKuge : "!<",
197	TOKug : "!<=",
198
199	TOKshl : "<<",
200	TOKshr : ">>",
201	TOKshlass : "<<=",
202	TOKshrass : ">>=",
203	TOKushr : ">>>",
204	TOKushrass : ">>>=",
205	TOKcat : "~",
206	TOKcatass : "~=", // ~ ~=
207	TOKadd : "+",
208	TOKmin : "-",
209	TOKaddass : "+=",
210	TOKminass : "-=",
211	TOKmul : "*",
212	TOKdiv : "/",
213	TOKmod : "%",
214	TOKmulass : "*=",
215	TOKdivass : "/=",
216	TOKmodass : "%=",
217	TOKand : "&",
218	TOKor : "\|",
219	TOKxor : "^",
220	TOKandass : "&=",
221	TOKorass : "\|=",
222	TOKxorass : "^=",
223	TOKassign : "=",
224	TOKnot : "!",
225	TOKtilde : "~",
226	TOKplusplus : "++",
227	TOKminusminus : "--",
228	TOKdot : ".",
229	TOKarrow : "->",
230	TOKcomma : ",",
231	TOKquestion : "?",
232	TOKandand : "&&",
233	TOKoror : "\|\|",
234
235	// Numeric literals
236	TOKint32v : "int32v", TOKuns32v : "uns32v",
237	TOKint64v : "int64v", TOKuns64v : "uns64v",
238	TOKfloat32v : "float32v", TOKfloat64v : "float64v", TOKfloat80v : "float80v",
239	TOKimaginary32v : "imaginary32v", TOKimaginary64v : "imaginary64v", TOKimaginary80v : "imaginary80v",
240
241	// Char constants
242	TOKcharv : "charv", TOKwcharv : "wcharv", TOKdcharv : "dcharv",
243
244	// Leaf operators
245	TOKidentifier : "identifier", TOKstring : "string",
246	TOKthis : "this", TOKsuper : "super",
247
248	// Basic types
249	TOKvoid : "void",
250	TOKint8 : "byte", TOKuns8 : "ubyte",
251	TOKint16 : "short", TOKuns16 : "ushort",
252	TOKint32 : "int", TOKuns32 : "uint",
253	TOKint64 : "long", TOKuns64 : "ulong",
254	TOKfloat32 : "float", TOKfloat64 : "double", TOKfloat80 : "real",
255	TOKimaginary32 : "ifloat", TOKimaginary64 : "idouble", TOKimaginary80 : "ireal",
256	TOKcomplex32 : "cfloat", TOKcomplex64 : "cdouble", TOKcomplex80 : "creal",
257	TOKchar : "char", TOKwchar : "wchar", TOKdchar : "dchar", TOKbit : "bit",
258
259	// Aggregates
260	TOKstruct : "struct", TOKclass : "class", TOKinterface : "interface", TOKunion : "union", TOKenum : "enum", TOKimport : "import",
261	TOKtypedef : "typedef", TOKalias : "alias", TOKoverride : "override", TOKdelegate : "delegate", TOKfunction : "function",
262	TOKmixin : "mixin",
263
264	TOKalign : "align", TOKextern : "extern", TOKprivate : "private", TOKprotected : "protected", TOKpublic : "public", TOKexport : "export",
265	TOKstatic : "static", /TOKvirtual : "virtual",/ TOKfinal : "final", TOKconst : "const", TOKabstract : "abstract", TOKvolatile : "volatile",
266	TOKdebug : "debug", TOKdeprecated : "deprecated", TOKin : "in", TOKout : "out", TOKinout : "inout",
267	TOKauto : "auto", TOKpackage : "package",
268
269	// Statements
270	TOKif : "if", TOKelse : "else", TOKwhile : "while", TOKfor : "for", TOKdo : "do", TOKswitch : "switch",
271	TOKcase : "case", TOKdefault : "default", TOKbreak : "break", TOKcontinue : "continue", TOKwith : "with",
272	TOKsynchronized : "synchronized", TOKreturn : "return", TOKgoto : "goto", TOKtry : "try", TOKcatch : "catch", TOKfinally : "finally",
273	TOKasm : "asm", TOKforeach : "foreach",
274
275	// Contracts
276	TOKbody : "body", TOKinvariant : "invariant",
277
278	// Testing
279	TOKunittest : "unittest",
280	];
281
282	class Identifier {
283	int value;
284	char[] string;
285
286	this(char[] string, int value) {
287	this.value = value;
288	this.string = string;
289	};
290	};
291
292	// A language token:
293	class Token {
294	// The kind of token:
295	uint value;
296	Token next;
297
298	// The value for the token:
299	union {
300	// The identifier or value of the token:
301	Identifier ident;
302
303	char[] ustring;
304
305	// Integers
306	int int32value;
307	uint uns32value;
308	long int64value;
309	ulong uns64value;
310
311	// Floats
312	real float80value;
313	};
314
315	char[] toString() {
316	return toktostr[value];
317	};
318
319	static char[] toChars(uint value) {
320	return toktostr[value];
321	};
322	}
323
324	// Exception thrown during parsing of D language:
325	class DLexerException : Error {
326	public:
327	this(DLexer dlx, char[] msg) {
328	// Construct an error message with the filename and line number:
329	super(dlx.filename ~ "(" ~ format("%d", dlx.line) ~ ")" ~ ": " ~ msg);
330	}
331	}
332
333	// Check for octal digit:
334	int isodigit(dchar x) {
335	if (!isdigit(x) \|\| (x == '8') \|\| (x == '9')) return 0;
336	return -1;
337	}
338
339	class Loc {
340	public:
341	int linnum;
342
343	this() {
344	}
345	}
346
347	// The D language lexer (tokenizer):
348	class DLexer {
349	static uint[char[]] keywords;
350	// Initialize the keyword->tokenvalue AA:
351	static this() {
352	// Add all the keywords' values into the AA:
353	keywords["this"] = TOKthis;
354	keywords["super"] = TOKsuper;
355	keywords["assert"] = TOKassert;
356	keywords["null"] = TOKnull;
357	keywords["true"] = TOKtrue;
358	keywords["false"] = TOKfalse;
359	keywords["cast"] = TOKcast;
360	keywords["new"] = TOKnew;
361	keywords["delete"] = TOKdelete;
362	keywords["throw"] = TOKthrow;
363	keywords["module"] = TOKmodule;
364	keywords["pragma"] = TOKpragma;
365	keywords["typeof"] = TOKtypeof;
366	keywords["typeid"] = TOKtypeid;
367
368	keywords["template"] = TOKtemplate;
369	keywords["instance"] = TOKinstance;
370
371	keywords["void"] = TOKvoid;
372	keywords["byte"] = TOKint8;
373	keywords["ubyte"] = TOKuns8;
374	keywords["short"] = TOKint16;
375	keywords["ushort"] = TOKuns16;
376	keywords["int"] = TOKint32;
377	keywords["uint"] = TOKuns32;
378	keywords["long"] = TOKint64;
379	keywords["ulong"] = TOKuns64;
380	keywords["float"] = TOKfloat32;
381	keywords["double"] = TOKfloat64;
382	keywords["real"] = TOKfloat80;
383
384	keywords["bit"] = TOKbit;
385	keywords["char"] = TOKchar;
386	keywords["wchar"] = TOKwchar;
387	keywords["dchar"] = TOKdchar;
388
389	keywords["ifloat"] = TOKimaginary32;
390	keywords["idouble"] = TOKimaginary64;
391	keywords["ireal"] = TOKimaginary80;
392
393	keywords["cfloat"] = TOKcomplex32;
394	keywords["cdouble"] = TOKcomplex64;
395	keywords["creal"] = TOKcomplex80;
396
397	keywords["delegate"] = TOKdelegate;
398	keywords["function"] = TOKfunction;
399
400	keywords["is"] = TOKidentity;
401	keywords["if"] = TOKif;
402	keywords["else"] = TOKelse;
403	keywords["while"] = TOKwhile;
404	keywords["for"] = TOKfor;
405	keywords["do"] = TOKdo;
406	keywords["switch"] = TOKswitch;
407	keywords["case"] = TOKcase;
408	keywords["default"] = TOKdefault;
409	keywords["break"] = TOKbreak;
410	keywords["continue"] = TOKcontinue;
411	keywords["synchronized"] = TOKsynchronized;
412	keywords["return"] = TOKreturn;
413	keywords["goto"] = TOKgoto;
414	keywords["try"] = TOKtry;
415	keywords["catch"] = TOKcatch;
416	keywords["finally"] = TOKfinally;
417	keywords["with"] = TOKwith;
418	keywords["asm"] = TOKasm;
419	keywords["foreach"] = TOKforeach;
420
421	keywords["struct"] = TOKstruct;
422	keywords["class"] = TOKclass;
423	keywords["interface"] = TOKinterface;
424	keywords["union"] = TOKunion;
425	keywords["enum"] = TOKenum;
426	keywords["import"] = TOKimport;
427	keywords["mixin"] = TOKmixin;
428	keywords["static"] = TOKstatic;
429	/keywords["virtual"] = TOKvirtual;/
430	keywords["final"] = TOKfinal;
431	keywords["const"] = TOKconst;
432	keywords["typedef"] = TOKtypedef;
433	keywords["alias"] = TOKalias;
434	keywords["override"] = TOKoverride;
435	keywords["abstract"] = TOKabstract;
436	keywords["volatile"] = TOKvolatile;
437	keywords["debug"] = TOKdebug;
438	keywords["deprecated"] = TOKdeprecated;
439	keywords["in"] = TOKin;
440	keywords["out"] = TOKout;
441	keywords["inout"] = TOKinout;
442	keywords["auto"] = TOKauto;
443
444	keywords["align"] = TOKalign;
445	keywords["extern"] = TOKextern;
446	keywords["private"] = TOKprivate;
447	keywords["package"] = TOKpackage;
448	keywords["protected"] = TOKprotected;
449	keywords["public"] = TOKpublic;
450	keywords["export"] = TOKexport;
451
452	keywords["body"] = TOKbody;
453	keywords["invariant"] = TOKinvariant;
454	keywords["unittest"] = TOKunittest;
455	keywords["version"] = TOKversion;
456	}
457
458	protected:
459	Token token;
460	Identifier[char[]] identifiers;
461	Loc loc;
462
463	private:
464	char[] file; // The input file
465	uint p; // Current character
466
467	char[] wysiwygString(char tc) {
468	char c;
469	char[] s;
470	uint i;
471
472	i = 0;
473	++p;
474	s.length = 16;
475	while (p < file.length) {
476	c = file[p++];
477	switch (c) {
478	case '\n':
479	++line;
480	break;
481
482	case '\r':
483	if (file[p] == '\n')
484	continue; // ignore
485	c = '\n'; // treat EndOfLine as \n character
486	++line;
487	break;
488
489	case 0x1A:
490	error("unterminated string constant starting at %s", s);
491	return null;
492
493	case '"', '`':
494	if (c == tc) {
495	s.length = i;
496	return s;
497	}
498	break;
499
500	default:
501	break;
502	}
503
504	if (i == s.length) s.length = s.length * 2;
505	s[i++] = c;
506	}
507	s.length = i;
508	return s;
509	}
510
511	// Test this!
512	char[] hexString() {
513	char c;
514	uint n = 0, i = 0;
515	char[] s;
516	ubyte v;
517
518	p++;
519	s.length = 16;
520	while (p < file.length) {
521	c = file[p++];
522	switch (c) {
523	case ' ', '\t', '\v', '\f':
524	continue; // skip white space
525
526	case '\r':
527	if (file[p] == '\n')
528	continue; // ignore
529	// Treat isolated '\r' as if it were a '\n'
530	case '\n':
531	++line;
532	continue;
533
534	case 0x1A:
535	error("unterminated string constant starting at %s", s);
536	return null;
537
538	case '"':
539	if (n & 1) {
540	error("odd number (%d) of hex characters in hex string", n);
541	return null;
542	}
543	s.length = i;
544	return s;
545
546	default:
547	if (c >= '0' && c <= '9')
548	c -= '0';
549	else if (c >= 'a' && c <= 'f')
550	c -= 'a' - 10;
551	else if (c >= 'A' && c <= 'F')
552	c -= 'A' - 10;
553	else
554	error("non-hex character '%c'", c);
555	if (n & 1) {
556	v = (v << 4) \| c;
557	if (i == s.length) s.length = s.length * 2;
558	s[i++] = v;
559	} else
560	v = c;
561	++n;
562	break;
563	}
564	}
565	return null;
566	}
567
568	public:
569	char[] filename; // Filename
570	uint line; // Current line
571
572	// Initialize the lexer with the full source code as a string:
573	this(char[] filename, char[] src) {
574	file = src;
575	this.filename = filename;
576	restart();
577	}
578
579	// Possible to buffer up errors until some limit n.
580	void error(char[] msg, ...) {
581	throw new DLexerException(this, format(msg, _arguments));
582	}
583
584	// This function consumes a full D language token and returns it in the structure Token:
585	// null is returned if the end of the file is reached.
586	uint scan(Token t) {
587	uint start;
588
589	t.ident = null;
590	t.uns64value = 0;
591	t.value = TOKidentifier;
592
593	// Past the end of file? Return an EOF token:
594	if (p >= file.length) {
595	t.value = TOKeof;
596	return t.value;
597	}
598
599	// Read up to the next white-space char:
600	while (p < file.length) {
601	switch (file[p]) {
602	case ' ', '\n', '\r', '\v', '\t', '\f':
603	if (file[p] == '\n') ++line;
604	++p;
605	break;
606
607	case '.':
608	++p;
609	if (file[p] == '.') {
610	++p;
611	if (file[p] == '.') {
612	++p;
613	t.value = TOKdotdotdot;
614	} else
615	t.value = TOKslice;
616	} else
617	t.value = TOKdot;
618	return t.value;
619
620	case '&':
621	++p;
622	if (file[p] == '=') {
623	++p;
624	t.value = TOKandass;
625	} else if (file[p] == '&') {
626	++p;
627	t.value = TOKandand;
628	} else
629	t.value = TOKand;
630	return t.value;
631
632	case '\|':
633	++p;
634	if (file[p] == '=') {
635	++p;
636	t.value = TOKorass;
637	} else if (file[p] == '\|') {
638	++p;
639	t.value = TOKoror;
640	} else
641	t.value = TOKor;
642	return t.value;
643
644	case '-':
645	++p;
646	if (file[p] == '=') {
647	++p;
648	t.value = TOKminass;
649	} else if (file[p] == '-') {
650	++p;
651	t.value = TOKminusminus;
652	} else
653	t.value = TOKmin;
654	return t.value;
655
656	case '+':
657	++p;
658	if (file[p] == '=') {
659	++p;
660	t.value = TOKaddass;
661	} else if (file[p] == '+') {
662	++p;
663	t.value = TOKplusplus;
664	} else
665	t.value = TOKadd;
666	return t.value;
667
668	case '=':
669	++p;
670	if (file[p] == '=') {
671	++p;
672	if (file[p] == '=') {
673	++p;
674	t.value = TOKidentity;
675	} else
676	t.value = TOKequal;
677	} else
678	t.value = TOKassign;
679	return t.value;
680
681	case '<':
682	++p;
683	if (file[p] == '=') {
684	++p;
685	t.value = TOKle; // <=
686	} else if (file[p] == '<') {
687	++p;
688	if (file[p] == '=') {
689	++p;
690	t.value = TOKshlass; // <<=
691	} else
692	t.value = TOKshl; // <<
693	} else if (file[p] == '>') {
694	++p;
695	if (file[p] == '=') {
696	++p;
697	t.value = TOKleg; // <>=
698	} else
699	t.value = TOKlg; // <>
700	} else
701	t.value = TOKlt; // <
702	return t.value;
703
704	case '>':
705	++p;
706	if (file[p] == '=') {
707	++p;
708	t.value = TOKge; // >=
709	} else if (file[p] == '>') {
710	++p;
711	if (file[p] == '=') {
712	++p;
713	t.value = TOKshrass; // >>=
714	} else if (file[p] == '>') {
715	++p;
716	if (file[p] == '=') {
717	++p;
718	t.value = TOKushrass; // >>>=
719	} else
720	t.value = TOKushr; // >>>
721	} else
722	t.value = TOKshr; // >>
723	} else
724	t.value = TOKgt; // >
725	return t.value;
726
727	case '!':
728	++p;
729	if (file[p] == '=') {
730	++p;
731	if (file[p] == '=') {
732	++p;
733	t.value = TOKnotidentity; // !==
734	} else
735	t.value = TOKnotequal; // !=
736	} else if (file[p] == '<') {
737	++p;
738	if (file[p] == '>') {
739	++p;
740	if (file[p] == '=') {
741	++p;
742	t.value = TOKunord; // !<>=
743	} else
744	t.value = TOKue; // !<>
745	} else if (file[p] == '=') {
746	++p;
747	t.value = TOKug; // !<=
748	} else
749	t.value = TOKuge; // !<
750	} else if (file[p] == '>') {
751	++p;
752	if (file[p] == '=') {
753	++p;
754	t.value = TOKul; // !>=
755	} else
756	t.value = TOKule; // !>
757	} else
758	t.value = TOKnot; // !
759	return t.value;
760
761	case '*':
762	++p;
763	if (file[p] == '=') {
764	++p;
765	t.value = TOKmulass;
766	} else
767	t.value = TOKmul;
768	return t.value;
769	case '%':
770	++p;
771	if (file[p] == '=') {
772	++p;
773	t.value = TOKmodass;
774	} else
775	t.value = TOKmod;
776	return t.value;
777	case '^':
778	++p;
779	if (file[p] == '=') {
780	++p;
781	t.value = TOKxorass;
782	} else
783	t.value = TOKxor;
784	return t.value;
785	case '~':
786	++p;
787	if (file[p] == '=') {
788	++p;
789	t.value = TOKcatass;
790	} else
791	t.value = TOKtilde;
792	return t.value;
793
794	case '(': ++p; t.value = TOKlparen; return t.value;
795	case ')': ++p; t.value = TOKrparen; return t.value;
796	case '[':
797	++p;
798	if (file[p] == ']') {
799	++p;
800	t.value = TOKarray;
801	} else
802	t.value = TOKlbracket;
803	return t.value;
804	case ']': ++p; t.value = TOKrbracket; return t.value;
805	case '{': ++p; t.value = TOKlcurly; return t.value;
806	case '}': ++p; t.value = TOKrcurly; return t.value;
807	case ':': ++p; t.value = TOKcolon; return t.value;
808	case ';': ++p; t.value = TOKsemicolon; return t.value;
809	case '?': ++p; t.value = TOKquestion; return t.value;
810	case ',': ++p; t.value = TOKcomma; return t.value;
811
812	case '/':
813	++p;
814	switch (file[p]) {
815	case '=':
816	t.value = TOKdivass;
817	return t.value;
818
819	case '/':
820	// single-line // comment:
821	++p;
822	while (p < file.length) {
823	if (file[p] == '\n') {
824	++line;
825	++p;
826	break;
827	}
828	++p;
829	}
830	break;
831
832	case '+': {
833	int nest = 0;
834	// nested code comment /+:
835	++p;
836	while (p < file.length) {
837	if (file[p] == '\n') ++line;
838
839	if (file[p] == '+') {
840	++p;
841	if (file[p] == '/') {
842	++p;
843	--nest;
844	if (nest < 0) break;
845	}
846	} else if (file[p] == '/') {
847	++p;
848	if (file[p] == '+') {
849	++p;
850	++nest;
851	}
852	} else ++p;
853	}
854	break;
855	}
856
857	case '*':
858	// multi-line comment /*:
859	++p;
860	while (p < file.length) {
861	if (file[p] == '\n') ++line;
862
863	if (file[p] == '*') {
864	++p;
865	if (file[p] == '/') {
866	++p;
867	break;
868	}
869	} else ++p;
870	}
871	break;
872
873	default:
874	t.value = TOKdiv;
875	return t.value;
876	}
877	break;
878
879	case '\'': {
880	char[] tok;
881	version (interpret_slashes) {
882	// Interpret the escaped characters and insert them:
883	++p;
884	if (file[p] == '\\') {
885	tok.length = 1;
886	++p;
887	switch (file[p]) {
888	case 'n': tok[0] = '\n'; break;
889	case 'r': tok[0] = '\r'; break;
890	case 'v': tok[0] = '\v'; break;
891	case 't': tok[0] = '\t'; break;
892	case 'f': tok[0] = '\f'; break;
893	case '\'': tok[0] = '\''; break;
894	case '\"': tok[0] = '\"'; break;
895	case '\\': tok[0] = '\\'; break;
896	default: break;
897	}
898	++p;
899	++p;
900	t.value = TOKcharv;
901	t.ustring = tok;
902	return t.value;
903	} else {
904	tok.length = 1;
905	tok[0] = file[p];
906	++p;
907	++p;
908	t.value = TOKcharv;
909	t.ustring = tok;
910	return t.value;
911	}
912	} else {
913	// Just copy over the escape strings:
914	++p;
915	start = p;
916	while (p < file.length) {
917	if (file[p] == '\\') {
918	++p;
919	} else if (file[p] == '\'') {
920	break;
921	}
922	++p;
923	}
924	t.ustring = file[start .. p];
925	t.value = TOKcharv;
926	++p;
927	return t.value;
928	}
929	}
930
931	case '\"': {
932	char[] tok;
933	version (interpret_slashes) {
934	// Interpret the escaped characters and insert them:
935	uint l = 0;
936	tok.length = 64;
937	++p;
938	while (p < file.length) {
939	if (file[p] == '\\') {
940	++p;
941	if (p >= file.length) break;
942	switch (file[p]) {
943	case 'n': tok[l] = '\n'; break;
944	case 'r': tok[l] = '\r'; break;
945	case 't': tok[l] = '\t'; break;
946	case 'v': tok[l] = '\v'; break;
947	case 'f': tok[l] = '\f'; break;
948	case '\'': tok[l] = '\''; break;
949	case '\"': tok[l] = '\"'; break;
950	case '\\': tok[l] = '\\'; break;
951	default:
952	}
953	// Grow the token length:
954	if (++l >= tok.length) tok.length = tok.length * 2;
955	++p;
956	} else if (file[p] == '\"') {
957	break;
958	} else {
959	tok[l] = file[p];
960	// Grow the token length:
961	if (++l >= tok.length) tok.length = tok.length * 2;
962	++p;
963	}
964	}
965	t.value = TOKstring;
966	tok.length = l;
967	t.ustring = tok;
968	++p;
969	return t.value;
970	} else {
971	// Just copy over the escape strings:
972	++p;
973	start = p;
974	while (p < file.length) {
975	if (file[p] == '\\') {
976	++p;
977	} else if (file[p] == '\"') {
978	break;
979	}
980	++p;
981	}
982	t.value = TOKstring;
983	t.ustring = file[start .. p];
984	++p;
985	return t.value;
986	}
987	}
988
989	case 'r':
990	// WYSIWYG string?
991	++p;
992	if (file[p] != '\"') {
993	// Nope, just an identifier:
994	start = p - 1;
995	goto case_ident;
996	}
997	case '`':
998	t.value = TOKstring;
999	t.ustring = wysiwygString(file[p]);
1000	return t.value;
1001
1002	case 'x':
1003	// HEX string?
1004	++p;
1005	if (file[p] != '\"') {
1006	start = p - 1;
1007	goto case_ident;
1008	}
1009	t.value = TOKstring;
1010	t.ustring = hexString();
1011	return t.value;
1012
1013	// Identifier start with _ or a-z,A-Z:
1014	case 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
1015	'm', 'n', 'o', 'p', 'q', 's', 't', 'u', 'v', 'w',
1016	'y', 'z':
1017	case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L',
1018	'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
1019	'Y', 'Z', '_':
1020	// Set the starting point at this character:
1021	start = p;
1022	case_ident:
1023	// Extract the identifier:
1024	while (p < file.length) {
1025	if (isalnum(file[p]) \|\| (file[p] == '_'))
1026	++p;
1027	else
1028	break;
1029	}
1030	// Checks for the identifier in the keyword list, returns null if not there:
1031	char[] tok = file[start .. p];
1032	if (tok in keywords) {
1033	t.value = keywords[tok];
1034	} else {
1035	t.value = TOKidentifier;
1036	// Check for the identifier in the list:
1037	if (tok in identifiers) {
1038	// Use that.
1039	t.ident = identifiers[tok];
1040	} else {
1041	// Make a new one.
1042	t.ident = new Identifier(tok, 0);
1043	identifiers[tok] = t.ident;
1044	}
1045	}
1046	return t.value;
1047
1048	// Numeric literal:
1049	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': {
1050	return number(t);
1051	}
1052
1053	default:
1054	++p;
1055	}
1056	}
1057
1058	return 0;
1059	}
1060
1061	// Parse a real number:
1062	uint inreal(Token t) {
1063	printf("inreal\n");
1064
1065	int dblstate, i;
1066	char[] s;
1067	char c;
1068	char hex; // is this a hexadecimal-floating-constant?
1069	uint result;
1070
1071	//printf("Lexer::inreal()\n");
1072	i = 0;
1073	s.length = 16;
1074	dblstate = 0;
1075	hex = 0;
1076	Lnext:
1077	while (p < file.length) {
1078	// Get next char from input
1079	c = file[p++];
1080	while (1) {
1081	switch (dblstate) {
1082	case 0: // opening state
1083	if (c == '0')
1084	dblstate = 9;
1085	else
1086	dblstate = 1;
1087	break;
1088
1089	case 9:
1090	dblstate = 1;
1091	if (c == 'X' \|\| c == 'x') {
1092	hex++;
1093	break;
1094	}
1095	case 1: // digits to left of .
1096	case 3: // digits to right of .
1097	case 7: // continuing exponent digits
1098	if (!isdigit(c) && !(hex && isxdigit(c))) {
1099	if (c == '_')
1100	goto Lnext; // ignore embedded '_'
1101	dblstate++;
1102	continue;
1103	}
1104	break;
1105
1106	case 2: // no more digits to left of .
1107	if (c == '.') {
1108	dblstate++;
1109	break;
1110	}
1111	case 4: // no more digits to right of .
1112	if ((c == 'E' \|\| c == 'e') \|\| hex && (c == 'P' \|\| c == 'p')) {
1113	dblstate = 5;
1114	hex = 0; // exponent is always decimal
1115	break;
1116	}
1117	if (hex)
1118	error("binary-exponent-part required");
1119	goto done;
1120
1121	case 5: // looking immediately to right of E
1122	dblstate++;
1123	if (c == '-' \|\| c == '+')
1124	break;
1125	case 6: // 1st exponent digit expected
1126	if (!isdigit(c))
1127	error("exponent expected");
1128	dblstate++;
1129	break;
1130
1131	case 8: // past end of exponent digits
1132	goto done;
1133	}
1134	break;
1135	}
1136	if (i == s.length) s.length = s.length * 2;
1137	s[i++] = c;
1138	}
1139	done:
1140	p--;
1141	s.length = i;
1142
1143	setErrno(0);
1144
1145	switch (file[p]) {
1146	case 'F':
1147	case 'f':
1148	t.float80value = strtod(s.ptr, null);
1149	result = TOKfloat32v;
1150	p++;
1151	break;
1152
1153	default:
1154	t.float80value = strtod(s.ptr, null);
1155	result = TOKfloat64v;
1156	break;
1157
1158	case 'L':
1159	case 'l':
1160	t.float80value = strtold(s.ptr, null);
1161	result = TOKfloat80v;
1162	p++;
1163	break;
1164	}
1165	// Imaginary value:
1166	if (file[p] == 'i' \|\| file[p] == 'I') {
1167	p++;
1168	switch (result)
1169	{
1170	case TOKfloat32v:
1171	result = TOKimaginary32v;
1172	break;
1173	case TOKfloat64v:
1174	result = TOKimaginary64v;
1175	break;
1176	case TOKfloat80v:
1177	result = TOKimaginary80v;
1178	break;
1179	}
1180	}
1181	// Standard C:
1182	if (getErrno() != 0) // == ERANGE
1183	error("number is not representable");
1184	return result;
1185	}
1186
1187	// Parse a number and return the associated value:
1188	uint number(Token t) {
1189	// We use a state machine to collect numbers
1190	enum : uint { STATE_initial, STATE_0, STATE_decimal, STATE_octal, STATE_octale,
1191	STATE_hex, STATE_binary, STATE_hex0, STATE_binary0,
1192	STATE_hexh, STATE_error };
1193	uint state;
1194
1195	enum : uint {
1196	FLAGS_decimal = 1, // decimal
1197	FLAGS_unsigned = 2, // u or U suffix
1198	FLAGS_long = 4, // l or L suffix
1199	};
1200	uint flags = FLAGS_decimal;
1201
1202	uint i;
1203	int base;
1204	char c;
1205	char[] s;
1206
1207	uint start;
1208	ulong n;
1209
1210	state = STATE_initial;
1211	base = 0;
1212
1213	s.length = 8;
1214
1215	start = p;
1216	i = 0;
1217	while (p < file.length) {
1218	c = file[p];
1219	switch (state) {
1220	case STATE_initial: // opening state
1221	if (c == '0')
1222	state = STATE_0;
1223	else
1224	state = STATE_decimal;
1225	break;
1226
1227	case STATE_0:
1228	flags = (flags & ~FLAGS_decimal);
1229	switch (c) {
1230	case 'X':
1231	case 'x':
1232	state = STATE_hex0;
1233	break;
1234	case '.':
1235	if (file[p+1] == '.') // .. is a separate token
1236	goto done;
1237	case 'i':
1238	case 'f':
1239	case 'F':
1240	goto realnum;
1241	case 'B':
1242	case 'b':
1243	state = STATE_binary0;
1244	break;
1245
1246	case '0': case '1': case '2': case '3':
1247	case '4': case '5': case '6': case '7':
1248	state = STATE_octal;
1249	break;
1250
1251	case '_':
1252	state = STATE_octal;
1253	++p;
1254	continue;
1255
1256	default:
1257	goto done;
1258	}
1259	break;
1260
1261	case STATE_decimal: // reading decimal number
1262	if (!isdigit(c)) {
1263	if (c == '_') { // ignore embedded _
1264	++p;
1265	continue;
1266	}
1267	if (c == '.' && file[p+1] != '.')
1268	goto realnum;
1269	else if (c == 'i' \|\| c == 'f' \|\| c == 'F' \|\|
1270	c == 'e' \|\| c == 'E') {
1271	realnum: // It's a real number. Back up and rescan as a real
1272	p = start;
1273	return inreal(t);
1274	}
1275	goto done;
1276	}
1277	break;
1278
1279	case STATE_hex0: // reading hex number
1280	case STATE_hex:
1281	if (!isxdigit(c)) {
1282	if (c == '_') { // ignore embedded _
1283	++p;
1284	continue;
1285	}
1286	if (c == '.' && file[p+1] != '.')
1287	goto realnum;
1288	if (c == 'P' \|\| c == 'p' \|\| c == 'i')
1289	goto realnum;
1290	if (state == STATE_hex0)
1291	error("Hex digit expected, not '%c'", c);
1292	goto done;
1293	}
1294	state = STATE_hex;
1295	break;
1296
1297	case STATE_octal: // reading octal number
1298	case STATE_octale: // reading octal number with non-octal digits
1299	if (!isodigit(c)) {
1300	if (c == '_') { // ignore embedded _
1301	++p;
1302	continue;
1303	}
1304	if (c == '.' && file[p+1] != '.')
1305	goto realnum;
1306	if (c == 'i')
1307	goto realnum;
1308	if (isdigit(c))
1309	state = STATE_octale;
1310	else
1311	goto done;
1312	}
1313	break;
1314
1315	case STATE_binary0: // starting binary number
1316	case STATE_binary: // reading binary number
1317	if (c != '0' && c != '1') {
1318	if (c == '_') { // ignore embedded _
1319	++p;
1320	continue;
1321	}
1322	if (state == STATE_binary0) {
1323	error("binary digit expected");
1324	state = STATE_error;
1325	break;
1326	} else
1327	goto done;
1328	}
1329	state = STATE_binary;
1330	break;
1331
1332	case STATE_error: // for error recovery
1333	if (!isdigit(c)) // scan until non-digit
1334	goto done;
1335	break;
1336
1337	default:
1338	assert(0);
1339	}
1340
1341	if (i == s.length) s.length = s.length * 2;
1342	s[i++] = c;
1343	++p;
1344	}
1345
1346	done:
1347	s.length = i;
1348	if (state == STATE_octale)
1349	error("Octal digit expected");
1350
1351	if ((i == 1) && (state == STATE_decimal \|\| state == STATE_0))
1352	n = s[0] - '0';
1353	else {
1354	// Convert string to integer
1355	int q = 0;
1356	int r = 10, d;
1357
1358	if (s[q] == '0') {
1359	if (s[q+1] == 'x' \|\| s[q+1] == 'X') {
1360	q += 2, r = 16;
1361	} else if (s[q+1] == 'b' \|\| s[q+1] == 'B') {
1362	q += 2, r = 2;
1363	} else if (isdigit(s[q+1])) {
1364	q += 1, r = 8;
1365	}
1366	}
1367
1368	n = 0;
1369	while (q < s.length) {
1370	if (s[q] >= '0' && s[q] <= '9')
1371	d = s[q] - '0';
1372	else if (s[q] >= 'a' && s[q] <= 'z')
1373	d = s[q] - 'a' + 10;
1374	else if (s[q] >= 'A' && s[q] <= 'Z')
1375	d = s[q] - 'A' + 10;
1376	else
1377	break;
1378	if (d >= r)
1379	break;
1380	if (n * r + d < n) {
1381	error("integer overflow");
1382	break;
1383	}
1384
1385	n = n * r + d;
1386	++q;
1387	}
1388	}
1389
1390	// Parse trailing 'u', 'U', 'l' or 'L' in any combination
1391	while (p < file.length) {
1392	uint f;
1393
1394	switch (file[p]) {
1395	case 'U':
1396	case 'u':
1397	f = FLAGS_unsigned;
1398	goto L1;
1399	case 'L':
1400	case 'l':
1401	f = FLAGS_long;
1402	L1:
1403	++p;
1404	if (flags & f)
1405	error("unrecognized token");
1406	flags = (flags \| f);
1407	continue;
1408
1409	default:
1410	break;
1411	}
1412	break;
1413	}
1414
1415	switch (flags) {
1416	case 0:
1417	if (n & 0x8000000000000000L)
1418	t.value = TOKuns64v;
1419	else if (n & 0xFFFFFFFF00000000L)
1420	t.value = TOKint64v;
1421	else if (n & 0x80000000)
1422	t.value = TOKuns32v;
1423	else
1424	t.value = TOKint32v;
1425	break;
1426
1427	case FLAGS_decimal:
1428	if (n & 0x8000000000000000L) {
1429	error("signed integer overflow");
1430	t.value = TOKuns64v;
1431	}
1432	else if (n & 0xFFFFFFFF80000000L)
1433	t.value = TOKint64v;
1434	else
1435	t.value = TOKint32v;
1436	break;
1437
1438	case FLAGS_unsigned:
1439	case FLAGS_decimal \| FLAGS_unsigned:
1440	if (n & 0xFFFFFFFF00000000L)
1441	t.value = TOKuns64v;
1442	else
1443	t.value = TOKuns32v;
1444	break;
1445
1446	case FLAGS_decimal \| FLAGS_long:
1447	if (n & 0x8000000000000000L) {
1448	error("signed integer overflow");
1449	t.value = TOKuns64v;
1450	} else
1451	t.value = TOKint64v;
1452	break;
1453
1454	case FLAGS_long:
1455	if (n & 0x8000000000000000L)
1456	t.value = TOKuns64v;
1457	else
1458	t.value = TOKint64v;
1459	break;
1460
1461	case FLAGS_unsigned \| FLAGS_long:
1462	case FLAGS_decimal \| FLAGS_unsigned \| FLAGS_long:
1463	t.value = TOKuns64v;
1464	break;
1465
1466	default:
1467	assert(0);
1468	}
1469
1470	t.uns64value = n;
1471	return t.value;
1472	}
1473
1474	uint nextToken() {
1475	Token t;
1476
1477	if (token.next) {
1478	t = token.next;
1479	token = t;
1480	t.next = null;
1481	} else
1482	scan(token);
1483
1484	return token.value;
1485	}
1486
1487	// Only peek at the next token, don't consume it:
1488	Token peek(Token ct) {
1489	Token t;
1490
1491	if (ct.next) {
1492	t = ct.next;
1493	} else {
1494	t = new Token();
1495	scan(t);
1496	t.next = null;
1497	ct.next = t;
1498	}
1499
1500	return t;
1501	}
1502
1503	// Start the parsing at the beginning of the module:
1504	void restart() {
1505	p = 0;
1506	line = 1;
1507	}
1508	}

Note: See TracBrowser for help on using the browser.

bindings

root/trunk/dlexer.d

Download in other formats: