lexer.c

Revision 769, 90.2 kB (checked in by walter, 1 year ago)
belatedly merge in changes from changeset 717
Property svn:eol-style set to `native`

Line
1	// Compiler implementation of the D programming language
2	// Copyright (c) 1999-2010 by Digital Mars
3	// All Rights Reserved
4	// written by Walter Bright
5	// http://www.digitalmars.com
6	// License for redistribution is by either the Artistic License
7	// in artistic.txt, or the GNU General Public License in gnu.txt.
8	// See the included readme.txt for details.
9
10	/* Lexical Analyzer */
11
12	#include <stdio.h>
13	#include <string.h>
14	#include <ctype.h>
15	#include <stdarg.h>
16	#include <errno.h>
17	#include <wchar.h>
18	#include <stdlib.h>
19	#include <assert.h>
20	#include <time.h> // for time() and ctime()
21
22	#include "rmem.h"
23
24	#include "stringtable.h"
25
26	#include "lexer.h"
27	#include "utf.h"
28	#include "identifier.h"
29	#include "id.h"
30	#include "module.h"
31
32	#if _WIN32 && __DMC__
33	// from \dm\src\include\setlocal.h
34	extern "C" char * __cdecl __locale_decpoint;
35	#endif
36
37	extern int HtmlNamedEntity(unsigned char *p, int length);
38
39	#define LS 0x2028 // UTF line separator
40	#define PS 0x2029 // UTF paragraph separator
41
42	void unittest_lexer();
43
44	/********************************************
45	* Do our own char maps
46	*/
47
48	static unsigned char cmtable[256];
49
50	const int CMoctal = 0x1;
51	const int CMhex = 0x2;
52	const int CMidchar = 0x4;
53
54	inline unsigned char isoctal (unsigned char c) { return cmtable[c] & CMoctal; }
55	inline unsigned char ishex (unsigned char c) { return cmtable[c] & CMhex; }
56	inline unsigned char isidchar(unsigned char c) { return cmtable[c] & CMidchar; }
57
58	static void cmtable_init()
59	{
60	for (unsigned c = 0; c < sizeof(cmtable) / sizeof(cmtable[0]); c++)
61	{
62	if ('0' <= c && c <= '7')
63	cmtable[c] \|= CMoctal;
64	if (isdigit(c) \|\| ('a' <= c && c <= 'f') \|\| ('A' <= c && c <= 'F'))
65	cmtable[c] \|= CMhex;
66	if (isalnum(c) \|\| c == '_')
67	cmtable[c] \|= CMidchar;
68	}
69	}
70
71
72	/*********************** Token ********************************************/
73
74	const char *Token::tochars[TOKMAX];
75
76	void *Token::operator new(size_t size)
77	{ Token *t;
78
79	if (Lexer::freelist)
80	{
81	t = Lexer::freelist;
82	Lexer::freelist = t->next;
83	return t;
84	}
85
86	return ::operator new(size);
87	}
88
89	#ifdef DEBUG
90	void Token::print()
91	{
92	fprintf(stdmsg, "%s\n", toChars());
93	}
94	#endif
95
96	const char *Token::toChars()
97	{ const char *p;
98	static char buffer[3 + 3 * sizeof(value) + 1];
99
100	p = buffer;
101	switch (value)
102	{
103	case TOKint32v:
104	#if IN_GCC
105	sprintf(buffer,"%d",(d_int32)int64value);
106	#else
107	sprintf(buffer,"%d",int32value);
108	#endif
109	break;
110
111	case TOKuns32v:
112	case TOKcharv:
113	case TOKwcharv:
114	case TOKdcharv:
115	#if IN_GCC
116	sprintf(buffer,"%uU",(d_uns32)uns64value);
117	#else
118	sprintf(buffer,"%uU",uns32value);
119	#endif
120	break;
121
122	case TOKint64v:
123	sprintf(buffer,"%jdL",int64value);
124	break;
125
126	case TOKuns64v:
127	sprintf(buffer,"%juUL",uns64value);
128	break;
129
130	#if IN_GCC
131	case TOKfloat32v:
132	case TOKfloat64v:
133	case TOKfloat80v:
134	float80value.format(buffer, sizeof(buffer));
135	break;
136	case TOKimaginary32v:
137	case TOKimaginary64v:
138	case TOKimaginary80v:
139	float80value.format(buffer, sizeof(buffer));
140	// %% buffer
141	strcat(buffer, "i");
142	break;
143	#else
144	case TOKfloat32v:
145	sprintf(buffer,"%Lgf", float80value);
146	break;
147
148	case TOKfloat64v:
149	sprintf(buffer,"%Lg", float80value);
150	break;
151
152	case TOKfloat80v:
153	sprintf(buffer,"%LgL", float80value);
154	break;
155
156	case TOKimaginary32v:
157	sprintf(buffer,"%Lgfi", float80value);
158	break;
159
160	case TOKimaginary64v:
161	sprintf(buffer,"%Lgi", float80value);
162	break;
163
164	case TOKimaginary80v:
165	sprintf(buffer,"%LgLi", float80value);
166	break;
167	#endif
168
169	case TOKstring:
170	#if CSTRINGS
171	p = string;
172	#else
173	{ OutBuffer buf;
174
175	buf.writeByte('"');
176	for (size_t i = 0; i < len; )
177	{ unsigned c;
178
179	utf_decodeChar((unsigned char *)ustring, len, &i, &c);
180	switch (c)
181	{
182	case 0:
183	break;
184
185	case '"':
186	case '\\':
187	buf.writeByte('\\');
188	default:
189	if (isprint(c))
190	buf.writeByte(c);
191	else if (c <= 0x7F)
192	buf.printf("\\x%02x", c);
193	else if (c <= 0xFFFF)
194	buf.printf("\\u%04x", c);
195	else
196	buf.printf("\\U%08x", c);
197	continue;
198	}
199	break;
200	}
201	buf.writeByte('"');
202	if (postfix)
203	buf.writeByte('"');
204	buf.writeByte(0);
205	p = (char *)buf.extractData();
206	}
207	#endif
208	break;
209
210	case TOKidentifier:
211	case TOKenum:
212	case TOKstruct:
213	case TOKimport:
214	case BASIC_TYPES:
215	p = ident->toChars();
216	break;
217
218	default:
219	p = toChars(value);
220	break;
221	}
222	return p;
223	}
224
225	const char *Token::toChars(enum TOK value)
226	{ const char *p;
227	static char buffer[3 + 3 * sizeof(value) + 1];
228
229	p = tochars[value];
230	if (!p)
231	{ sprintf(buffer,"TOK%d",value);
232	p = buffer;
233	}
234	return p;
235	}
236
237	/************************* Lexer ******************************************/
238
239	Token *Lexer::freelist = NULL;
240	StringTable Lexer::stringtable;
241	OutBuffer Lexer::stringbuffer;
242
243	Lexer::Lexer(Module *mod,
244	unsigned char *base, unsigned begoffset, unsigned endoffset,
245	int doDocComment, int commentToken)
246	: loc(mod, 1)
247	{
248	//printf("Lexer::Lexer(%p,%d)\n",base,length);
249	//printf("lexer.mod = %p, %p\n", mod, this->loc.mod);
250	memset(&token,0,sizeof(token));
251	this->base = base;
252	this->end = base + endoffset;
253	p = base + begoffset;
254	this->mod = mod;
255	this->doDocComment = doDocComment;
256	this->anyToken = 0;
257	this->commentToken = commentToken;
258	//initKeywords();
259
260	/* If first line starts with '#!', ignore the line
261	*/
262
263	if (p[0] == '#' && p[1] =='!')
264	{
265	p += 2;
266	while (1)
267	{ unsigned char c = *p;
268	switch (c)
269	{
270	case '\n':
271	p++;
272	break;
273
274	case '\r':
275	p++;
276	if (*p == '\n')
277	p++;
278	break;
279
280	case 0:
281	case 0x1A:
282	break;
283
284	default:
285	if (c & 0x80)
286	{ unsigned u = decodeUTF();
287	if (u == PS \|\| u == LS)
288	break;
289	}
290	p++;
291	continue;
292	}
293	break;
294	}
295	loc.linnum = 2;
296	}
297	}
298
299
300	void Lexer::error(const char *format, ...)
301	{
302	if (mod && !global.gag)
303	{
304	char *p = loc.toChars();
305	if (*p)
306	fprintf(stdmsg, "%s: ", p);
307	mem.free(p);
308
309	va_list ap;
310	va_start(ap, format);
311	vfprintf(stdmsg, format, ap);
312	va_end(ap);
313
314	fprintf(stdmsg, "\n");
315	fflush(stdmsg);
316
317	if (global.errors >= 20) // moderate blizzard of cascading messages
318	fatal();
319	}
320	global.errors++;
321	}
322
323	void Lexer::error(Loc loc, const char *format, ...)
324	{
325	if (mod && !global.gag)
326	{
327	char *p = loc.toChars();
328	if (*p)
329	fprintf(stdmsg, "%s: ", p);
330	mem.free(p);
331
332	va_list ap;
333	va_start(ap, format);
334	vfprintf(stdmsg, format, ap);
335	va_end(ap);
336
337	fprintf(stdmsg, "\n");
338	fflush(stdmsg);
339
340	if (global.errors >= 20) // moderate blizzard of cascading messages
341	fatal();
342	}
343	global.errors++;
344	}
345
346	TOK Lexer::nextToken()
347	{ Token *t;
348
349	if (token.next)
350	{
351	t = token.next;
352	memcpy(&token,t,sizeof(Token));
353	t->next = freelist;
354	freelist = t;
355	}
356	else
357	{
358	scan(&token);
359	}
360	//token.print();
361	return token.value;
362	}
363
364	Token Lexer::peek(Token ct)
365	{ Token *t;
366
367	if (ct->next)
368	t = ct->next;
369	else
370	{
371	t = new Token();
372	scan(t);
373	t->next = NULL;
374	ct->next = t;
375	}
376	return t;
377	}
378
379	/***********************
380	* Look ahead at next token's value.
381	*/
382
383	TOK Lexer::peekNext()
384	{
385	return peek(&token)->value;
386	}
387
388	/***********************
389	* Look 2 tokens ahead at value.
390	*/
391
392	TOK Lexer::peekNext2()
393	{
394	Token *t = peek(&token);
395	return peek(t)->value;
396	}
397
398	/*********************************
399	* tk is on the opening (.
400	* Look ahead and return token that is past the closing ).
401	*/
402
403	Token Lexer::peekPastParen(Token tk)
404	{
405	//printf("peekPastParen()\n");
406	int parens = 1;
407	int curlynest = 0;
408	while (1)
409	{
410	tk = peek(tk);
411	//tk->print();
412	switch (tk->value)
413	{
414	case TOKlparen:
415	parens++;
416	continue;
417
418	case TOKrparen:
419	--parens;
420	if (parens)
421	continue;
422	tk = peek(tk);
423	break;
424
425	case TOKlcurly:
426	curlynest++;
427	continue;
428
429	case TOKrcurly:
430	if (--curlynest >= 0)
431	continue;
432	break;
433
434	case TOKsemicolon:
435	if (curlynest)
436	continue;
437	break;
438
439	case TOKeof:
440	break;
441
442	default:
443	continue;
444	}
445	return tk;
446	}
447	}
448
449	/**********************************
450	* Determine if string is a valid Identifier.
451	* Placed here because of commonality with Lexer functionality.
452	* Returns:
453	* 0 invalid
454	*/
455
456	int Lexer::isValidIdentifier(char *p)
457	{
458	size_t len;
459	size_t idx;
460
461	if (!p \|\| !*p)
462	goto Linvalid;
463
464	if (p >= '0' && p <= '9') // beware of isdigit() on signed chars
465	goto Linvalid;
466
467	len = strlen(p);
468	idx = 0;
469	while (p[idx])
470	{ dchar_t dc;
471
472	const char q = utf_decodeChar((unsigned char )p, len, &idx, &dc);
473	if (q)
474	goto Linvalid;
475
476	if (!((dc >= 0x80 && isUniAlpha(dc)) \|\| isalnum(dc) \|\| dc == '_'))
477	goto Linvalid;
478	}
479	return 1;
480
481	Linvalid:
482	return 0;
483	}
484
485	/****************************
486	* Turn next token in buffer into a token.
487	*/
488
489	void Lexer::scan(Token *t)
490	{
491	unsigned lastLine = loc.linnum;
492	unsigned linnum;
493
494	t->blockComment = NULL;
495	t->lineComment = NULL;
496	while (1)
497	{
498	t->ptr = p;
499	//printf("p = %p, p = '%c'\n",p,p);
500	switch (*p)
501	{
502	case 0:
503	case 0x1A:
504	t->value = TOKeof; // end of file
505	return;
506
507	case ' ':
508	case '\t':
509	case '\v':
510	case '\f':
511	p++;
512	continue; // skip white space
513
514	case '\r':
515	p++;
516	if (*p != '\n') // if CR stands by itself
517	loc.linnum++;
518	continue; // skip white space
519
520	case '\n':
521	p++;
522	loc.linnum++;
523	continue; // skip white space
524
525	case '0': case '1': case '2': case '3': case '4':
526	case '5': case '6': case '7': case '8': case '9':
527	t->value = number(t);
528	return;
529
530	#if CSTRINGS
531	case '\'':
532	t->value = charConstant(t, 0);
533	return;
534
535	case '"':
536	t->value = stringConstant(t,0);
537	return;
538
539	case 'l':
540	case 'L':
541	if (p[1] == '\'')
542	{
543	p++;
544	t->value = charConstant(t, 1);
545	return;
546	}
547	else if (p[1] == '"')
548	{
549	p++;
550	t->value = stringConstant(t, 1);
551	return;
552	}
553	#else
554	case '\'':
555	t->value = charConstant(t,0);
556	return;
557
558	case 'r':
559	if (p[1] != '"')
560	goto case_ident;
561	p++;
562	case '`':
563	t->value = wysiwygStringConstant(t, *p);
564	return;
565
566	case 'x':
567	if (p[1] != '"')
568	goto case_ident;
569	p++;
570	t->value = hexStringConstant(t);
571	return;
572
573	#if DMDV2
574	case 'q':
575	if (p[1] == '"')
576	{
577	p++;
578	t->value = delimitedStringConstant(t);
579	return;
580	}
581	else if (p[1] == '{')
582	{
583	p++;
584	t->value = tokenStringConstant(t);
585	return;
586	}
587	else
588	goto case_ident;
589	#endif
590
591	case '"':
592	t->value = escapeStringConstant(t,0);
593	return;
594
595	#if ! TEXTUAL_ASSEMBLY_OUT
596	case '\\': // escaped string literal
597	{ unsigned c;
598	unsigned char *pstart = p;
599
600	stringbuffer.reset();
601	do
602	{
603	p++;
604	switch (*p)
605	{
606	case 'u':
607	case 'U':
608	case '&':
609	c = escapeSequence();
610	stringbuffer.writeUTF8(c);
611	break;
612
613	default:
614	c = escapeSequence();
615	stringbuffer.writeByte(c);
616	break;
617	}
618	} while (*p == '\\');
619	t->len = stringbuffer.offset;
620	stringbuffer.writeByte(0);
621	t->ustring = (unsigned char *)mem.malloc(stringbuffer.offset);
622	memcpy(t->ustring, stringbuffer.data, stringbuffer.offset);
623	t->postfix = 0;
624	t->value = TOKstring;
625	#if DMDV2
626	if (!global.params.useDeprecated)
627	error("Escape String literal %.s is deprecated, use double quoted string literal \"%.s\" instead", p - pstart, pstart, p - pstart, pstart);
628	#endif
629	return;
630	}
631	#endif
632
633	case 'l':
634	case 'L':
635	#endif
636	case 'a': case 'b': case 'c': case 'd': case 'e':
637	case 'f': case 'g': case 'h': case 'i': case 'j':
638	case 'k': case 'm': case 'n': case 'o':
639	#if DMDV2
640	case 'p': /case 'q': case 'r':/ case 's': case 't':
641	#else
642	case 'p': case 'q': /case 'r':/ case 's': case 't':
643	#endif
644	case 'u': case 'v': case 'w': /case 'x':/ case 'y':
645	case 'z':
646	case 'A': case 'B': case 'C': case 'D': case 'E':
647	case 'F': case 'G': case 'H': case 'I': case 'J':
648	case 'K': case 'M': case 'N': case 'O':
649	case 'P': case 'Q': case 'R': case 'S': case 'T':
650	case 'U': case 'V': case 'W': case 'X': case 'Y':
651	case 'Z':
652	case '_':
653	case_ident:
654	{ unsigned char c;
655
656	while (1)
657	{
658	c = *++p;
659	if (isidchar(c))
660	continue;
661	else if (c & 0x80)
662	{ unsigned char *s = p;
663	unsigned u = decodeUTF();
664	if (isUniAlpha(u))
665	continue;
666	error("char 0x%04x not allowed in identifier", u);
667	p = s;
668	}
669	break;
670	}
671
672	StringValue sv = stringtable.update((char )t->ptr, p - t->ptr);
673	Identifier id = (Identifier ) sv->ptrvalue;
674	if (!id)
675	{ id = new Identifier(sv->lstring.string,TOKidentifier);
676	sv->ptrvalue = id;
677	}
678	t->ident = id;
679	t->value = (enum TOK) id->value;
680	anyToken = 1;
681	if (*t->ptr == '_') // if special identifier token
682	{
683	static char date[11+1];
684	static char time[8+1];
685	static char timestamp[24+1];
686
687	if (!date[0]) // lazy evaluation
688	{ time_t t;
689	char *p;
690
691	::time(&t);
692	p = ctime(&t);
693	assert(p);
694	sprintf(date, "%.6s %.4s", p + 4, p + 20);
695	sprintf(time, "%.8s", p + 11);
696	sprintf(timestamp, "%.24s", p);
697	}
698
699	#if DMDV1
700	if (mod && id == Id::FILE)
701	{
702	t->ustring = (unsigned char *)(loc.filename ? loc.filename : mod->ident->toChars());
703	goto Lstr;
704	}
705	else if (mod && id == Id::LINE)
706	{
707	t->value = TOKint64v;
708	t->uns64value = loc.linnum;
709	}
710	else
711	#endif
712	if (id == Id::DATE)
713	{
714	t->ustring = (unsigned char *)date;
715	goto Lstr;
716	}
717	else if (id == Id::TIME)
718	{
719	t->ustring = (unsigned char *)time;
720	goto Lstr;
721	}
722	else if (id == Id::VENDOR)
723	{
724	t->ustring = (unsigned char *)"Digital Mars D";
725	goto Lstr;
726	}
727	else if (id == Id::TIMESTAMP)
728	{
729	t->ustring = (unsigned char *)timestamp;
730	Lstr:
731	t->value = TOKstring;
732	Llen:
733	t->postfix = 0;
734	t->len = strlen((char *)t->ustring);
735	}
736	else if (id == Id::VERSIONX)
737	{ unsigned major = 0;
738	unsigned minor = 0;
739
740	for (const char *p = global.version + 1; 1; p++)
741	{
742	char c = *p;
743	if (isdigit(c))
744	minor = minor * 10 + c - '0';
745	else if (c == '.')
746	{ major = minor;
747	minor = 0;
748	}
749	else
750	break;
751	}
752	t->value = TOKint64v;
753	t->uns64value = major * 1000 + minor;
754	}
755	#if DMDV2
756	else if (id == Id::EOFX)
757	{
758	t->value = TOKeof;
759	// Advance scanner to end of file
760	while (!(p == 0 \|\| p == 0x1A))
761	p++;
762	}
763	#endif
764	}
765	//printf("t->value = %d\n",t->value);
766	return;
767	}
768
769	case '/':
770	p++;
771	switch (*p)
772	{
773	case '=':
774	p++;
775	t->value = TOKdivass;
776	return;
777
778	case '*':
779	p++;
780	linnum = loc.linnum;
781	while (1)
782	{
783	while (1)
784	{ unsigned char c = *p;
785	switch (c)
786	{
787	case '/':
788	break;
789
790	case '\n':
791	loc.linnum++;
792	p++;
793	continue;
794
795	case '\r':
796	p++;
797	if (*p != '\n')
798	loc.linnum++;
799	continue;
800
801	case 0:
802	case 0x1A:
803	error("unterminated /* */ comment");
804	p = end;
805	t->value = TOKeof;
806	return;
807
808	default:
809	if (c & 0x80)
810	{ unsigned u = decodeUTF();
811	if (u == PS \|\| u == LS)
812	loc.linnum++;
813	}
814	p++;
815	continue;
816	}
817	break;
818	}
819	p++;
820	if (p[-2] == '*' && p - 3 != t->ptr)
821	break;
822	}
823	if (commentToken)
824	{
825	t->value = TOKcomment;
826	return;
827	}
828	else if (doDocComment && t->ptr[2] == '*' && p - 4 != t->ptr)
829	{ // if / but not //
830	getDocComment(t, lastLine == linnum);
831	}
832	continue;
833
834	case '/': // do // style comments
835	linnum = loc.linnum;
836	while (1)
837	{ unsigned char c = *++p;
838	switch (c)
839	{
840	case '\n':
841	break;
842
843	case '\r':
844	if (p[1] == '\n')
845	p++;
846	break;
847
848	case 0:
849	case 0x1A:
850	if (commentToken)
851	{
852	p = end;
853	t->value = TOKcomment;
854	return;
855	}
856	if (doDocComment && t->ptr[2] == '/')
857	getDocComment(t, lastLine == linnum);
858	p = end;
859	t->value = TOKeof;
860	return;
861
862	default:
863	if (c & 0x80)
864	{ unsigned u = decodeUTF();
865	if (u == PS \|\| u == LS)
866	break;
867	}
868	continue;
869	}
870	break;
871	}
872
873	if (commentToken)
874	{
875	p++;
876	loc.linnum++;
877	t->value = TOKcomment;
878	return;
879	}
880	if (doDocComment && t->ptr[2] == '/')
881	getDocComment(t, lastLine == linnum);
882
883	p++;
884	loc.linnum++;
885	continue;
886
887	case '+':
888	{ int nest;
889
890	linnum = loc.linnum;
891	p++;
892	nest = 1;
893	while (1)
894	{ unsigned char c = *p;
895	switch (c)
896	{
897	case '/':
898	p++;
899	if (*p == '+')
900	{
901	p++;
902	nest++;
903	}
904	continue;
905
906	case '+':
907	p++;
908	if (*p == '/')
909	{
910	p++;
911	if (--nest == 0)
912	break;
913	}
914	continue;
915
916	case '\r':
917	p++;
918	if (*p != '\n')
919	loc.linnum++;
920	continue;
921
922	case '\n':
923	loc.linnum++;
924	p++;
925	continue;
926
927	case 0:
928	case 0x1A:
929	error("unterminated /+ +/ comment");
930	p = end;
931	t->value = TOKeof;
932	return;
933
934	default:
935	if (c & 0x80)
936	{ unsigned u = decodeUTF();
937	if (u == PS \|\| u == LS)
938	loc.linnum++;
939	}
940	p++;
941	continue;
942	}
943	break;
944	}
945	if (commentToken)
946	{
947	t->value = TOKcomment;
948	return;
949	}
950	if (doDocComment && t->ptr[2] == '+' && p - 4 != t->ptr)
951	{ // if /++ but not /++/
952	getDocComment(t, lastLine == linnum);
953	}
954	continue;
955	}
956	}
957	t->value = TOKdiv;
958	return;
959
960	case '.':
961	p++;
962	if (isdigit(*p))
963	{ /* Note that we don't allow ._1 and ._ as being
964	* valid floating point numbers.
965	*/
966	p--;
967	t->value = inreal(t);
968	}
969	else if (p[0] == '.')
970	{
971	if (p[1] == '.')
972	{ p += 2;
973	t->value = TOKdotdotdot;
974	}
975	else
976	{ p++;
977	t->value = TOKslice;
978	}
979	}
980	else
981	t->value = TOKdot;
982	return;
983
984	case '&':
985	p++;
986	if (*p == '=')
987	{ p++;
988	t->value = TOKandass;
989	}
990	else if (*p == '&')
991	{ p++;
992	t->value = TOKandand;
993	}
994	else
995	t->value = TOKand;
996	return;
997
998	case '\|':
999	p++;
1000	if (*p == '=')
1001	{ p++;
1002	t->value = TOKorass;
1003	}
1004	else if (*p == '\|')
1005	{ p++;
1006	t->value = TOKoror;
1007	}
1008	else
1009	t->value = TOKor;
1010	return;
1011
1012	case '-':
1013	p++;
1014	if (*p == '=')
1015	{ p++;
1016	t->value = TOKminass;
1017	}
1018	#if 0
1019	else if (*p == '>')
1020	{ p++;
1021	t->value = TOKarrow;
1022	}
1023	#endif
1024	else if (*p == '-')
1025	{ p++;
1026	t->value = TOKminusminus;
1027	}
1028	else
1029	t->value = TOKmin;
1030	return;
1031
1032	case '+':
1033	p++;
1034	if (*p == '=')
1035	{ p++;
1036	t->value = TOKaddass;
1037	}
1038	else if (*p == '+')
1039	{ p++;
1040	t->value = TOKplusplus;
1041	}
1042	else
1043	t->value = TOKadd;
1044	return;
1045
1046	case '<':
1047	p++;
1048	if (*p == '=')
1049	{ p++;
1050	t->value = TOKle; // <=
1051	}
1052	else if (*p == '<')
1053	{ p++;
1054	if (*p == '=')
1055	{ p++;
1056	t->value = TOKshlass; // <<=
1057	}
1058	else
1059	t->value = TOKshl; // <<
1060	}
1061	else if (*p == '>')
1062	{ p++;
1063	if (*p == '=')
1064	{ p++;
1065	t->value = TOKleg; // <>=
1066	}
1067	else
1068	t->value = TOKlg; // <>
1069	}
1070	else
1071	t->value = TOKlt; // <
1072	return;
1073
1074	case '>':
1075	p++;
1076	if (*p == '=')
1077	{ p++;
1078	t->value = TOKge; // >=
1079	}
1080	else if (*p == '>')
1081	{ p++;
1082	if (*p == '=')
1083	{ p++;
1084	t->value = TOKshrass; // >>=
1085	}
1086	else if (*p == '>')
1087	{ p++;
1088	if (*p == '=')
1089	{ p++;
1090	t->value = TOKushrass; // >>>=
1091	}
1092	else
1093	t->value = TOKushr; // >>>
1094	}
1095	else
1096	t->value = TOKshr; // >>
1097	}
1098	else
1099	t->value = TOKgt; // >
1100	return;
1101
1102	case '!':
1103	p++;
1104	if (*p == '=')
1105	{ p++;
1106	if (*p == '=' && global.params.Dversion == 1)
1107	{ p++;
1108	t->value = TOKnotidentity; // !==
1109	}
1110	else
1111	t->value = TOKnotequal; // !=
1112	}
1113	else if (*p == '<')
1114	{ p++;
1115	if (*p == '>')
1116	{ p++;
1117	if (*p == '=')
1118	{ p++;
1119	t->value = TOKunord; // !<>=
1120	}
1121	else
1122	t->value = TOKue; // !<>
1123	}
1124	else if (*p == '=')
1125	{ p++;
1126	t->value = TOKug; // !<=
1127	}
1128	else
1129	t->value = TOKuge; // !<
1130	}
1131	else if (*p == '>')
1132	{ p++;
1133	if (*p == '=')
1134	{ p++;
1135	t->value = TOKul; // !>=
1136	}
1137	else
1138	t->value = TOKule; // !>
1139	}
1140	else
1141	t->value = TOKnot; // !
1142	return;
1143
1144	case '=':
1145	p++;
1146	if (*p == '=')
1147	{ p++;
1148	if (*p == '=' && global.params.Dversion == 1)
1149	{ p++;
1150	t->value = TOKidentity; // ===
1151	}
1152	else
1153	t->value = TOKequal; // ==
1154	}
1155	else
1156	t->value = TOKassign; // =
1157	return;
1158
1159	case '~':
1160	p++;
1161	if (*p == '=')
1162	{ p++;
1163	t->value = TOKcatass; // ~=
1164	}
1165	else
1166	t->value = TOKtilde; // ~
1167	return;
1168
1169	#if DMDV2
1170	case '^':
1171	p++;
1172	if (*p == '^')
1173	{ p++;
1174	if (*p == '=')
1175	{ p++;
1176	t->value = TOKpowass; // ^^=
1177	}
1178	else
1179	t->value = TOKpow; // ^^
1180	}
1181	else if (*p == '=')
1182	{ p++;
1183	t->value = TOKxorass; // ^=
1184	}
1185	else
1186	t->value = TOKxor; // ^
1187	return;
1188	#endif
1189
1190	#define SINGLE(c,tok) case c: p++; t->value = tok; return;
1191
1192	SINGLE('(', TOKlparen)
1193	SINGLE(')', TOKrparen)
1194	SINGLE('[', TOKlbracket)
1195	SINGLE(']', TOKrbracket)
1196	SINGLE('{', TOKlcurly)
1197	SINGLE('}', TOKrcurly)
1198	SINGLE('?', TOKquestion)
1199	SINGLE(',', TOKcomma)
1200	SINGLE(';', TOKsemicolon)
1201	SINGLE(':', TOKcolon)
1202	SINGLE('$', TOKdollar)
1203	#if DMDV2
1204	SINGLE('@', TOKat)
1205	#endif
1206	#undef SINGLE
1207
1208	#define DOUBLE(c1,tok1,c2,tok2) \
1209	case c1: \
1210	p++; \
1211	if (*p == c2) \
1212	{ p++; \
1213	t->value = tok2; \
1214	} \
1215	else \
1216	t->value = tok1; \
1217	return;
1218
1219	DOUBLE('*', TOKmul, '=', TOKmulass)
1220	DOUBLE('%', TOKmod, '=', TOKmodass)
1221	#if DMDV1
1222	DOUBLE('^', TOKxor, '=', TOKxorass)
1223	#endif
1224	#undef DOUBLE
1225
1226	case '#':
1227	p++;
1228	pragma();
1229	continue;
1230
1231	default:
1232	{ unsigned c = *p;
1233
1234	if (c & 0x80)
1235	{ c = decodeUTF();
1236
1237	// Check for start of unicode identifier
1238	if (isUniAlpha(c))
1239	goto case_ident;
1240
1241	if (c == PS \|\| c == LS)
1242	{
1243	loc.linnum++;
1244	p++;
1245	continue;
1246	}
1247	}
1248	if (c < 0x80 && isprint(c))
1249	error("unsupported char '%c'", c);
1250	else
1251	error("unsupported char 0x%02x", c);
1252	p++;
1253	continue;
1254	}
1255	}
1256	}
1257	}
1258
1259	/*******************************************
1260	* Parse escape sequence.
1261	*/
1262
1263	unsigned Lexer::escapeSequence()
1264	{ unsigned c = *p;
1265
1266	#ifdef TEXTUAL_ASSEMBLY_OUT
1267	return c;
1268	#endif
1269	int n;
1270	int ndigits;
1271
1272	switch (c)
1273	{
1274	case '\'':
1275	case '"':
1276	case '?':
1277	case '\\':
1278	Lconsume:
1279	p++;
1280	break;
1281
1282	case 'a': c = 7; goto Lconsume;
1283	case 'b': c = 8; goto Lconsume;
1284	case 'f': c = 12; goto Lconsume;
1285	case 'n': c = 10; goto Lconsume;
1286	case 'r': c = 13; goto Lconsume;
1287	case 't': c = 9; goto Lconsume;
1288	case 'v': c = 11; goto Lconsume;
1289
1290	case 'u':
1291	ndigits = 4;
1292	goto Lhex;
1293	case 'U':
1294	ndigits = 8;
1295	goto Lhex;
1296	case 'x':
1297	ndigits = 2;
1298	Lhex:
1299	p++;
1300	c = *p;
1301	if (ishex(c))
1302	{ unsigned v;
1303
1304	n = 0;
1305	v = 0;
1306	while (1)
1307	{
1308	if (isdigit(c))
1309	c -= '0';
1310	else if (islower(c))
1311	c -= 'a' - 10;
1312	else
1313	c -= 'A' - 10;
1314	v = v * 16 + c;
1315	c = *++p;
1316	if (++n == ndigits)
1317	break;
1318	if (!ishex(c))
1319	{ error("escape hex sequence has %d hex digits instead of %d", n, ndigits);
1320	break;
1321	}
1322	}
1323	if (ndigits != 2 && !utf_isValidDchar(v))
1324	{ error("invalid UTF character \\U%08x", v);
1325	v = '?'; // recover with valid UTF character
1326	}
1327	c = v;
1328	}
1329	else
1330	error("undefined escape hex sequence \\%c\n",c);
1331	break;
1332
1333	case '&': // named character entity
1334	for (unsigned char *idstart = ++p; 1; p++)
1335	{
1336	switch (*p)
1337	{
1338	case ';':
1339	c = HtmlNamedEntity(idstart, p - idstart);
1340	if (c == ~0)
1341	{ error("unnamed character entity &%.*s;", (int)(p - idstart), idstart);
1342	c = ' ';
1343	}
1344	p++;
1345	break;
1346
1347	default:
1348	if (isalpha(*p) \|\|
1349	(p != idstart + 1 && isdigit(*p)))
1350	continue;
1351	error("unterminated named entity");
1352	break;
1353	}
1354	break;
1355	}
1356	break;
1357
1358	case 0:
1359	case 0x1A: // end of file
1360	c = '\\';
1361	break;
1362
1363	default:
1364	if (isoctal(c))
1365	{ unsigned v;
1366
1367	n = 0;
1368	v = 0;
1369	do
1370	{
1371	v = v * 8 + (c - '0');
1372	c = *++p;
1373	} while (++n < 3 && isoctal(c));
1374	c = v;
1375	if (c > 0xFF)
1376	error("0%03o is larger than a byte", c);
1377	}
1378	else
1379	error("undefined escape sequence \\%c\n",c);
1380	break;
1381	}
1382	return c;
1383	}
1384
1385	/**************************************
1386	*/
1387
1388	TOK Lexer::wysiwygStringConstant(Token *t, int tc)
1389	{ unsigned c;
1390	Loc start = loc;
1391
1392	p++;
1393	stringbuffer.reset();
1394	while (1)
1395	{
1396	c = *p++;
1397	switch (c)
1398	{
1399	case '\n':
1400	loc.linnum++;
1401	break;
1402
1403	case '\r':
1404	if (*p == '\n')
1405	continue; // ignore
1406	c = '\n'; // treat EndOfLine as \n character
1407	loc.linnum++;
1408	break;
1409
1410	case 0:
1411	case 0x1A:
1412	error("unterminated string constant starting at %s", start.toChars());
1413	t->ustring = (unsigned char *)"";
1414	t->len = 0;
1415	t->postfix = 0;
1416	return TOKstring;
1417
1418	case '"':
1419	case '`':
1420	if (c == tc)
1421	{
1422	t->len = stringbuffer.offset;
1423	stringbuffer.writeByte(0);
1424	t->ustring = (unsigned char *)mem.malloc(stringbuffer.offset);
1425	memcpy(t->ustring, stringbuffer.data, stringbuffer.offset);
1426	stringPostfix(t);
1427	return TOKstring;
1428	}
1429	break;
1430
1431	default:
1432	if (c & 0x80)
1433	{ p--;
1434	unsigned u = decodeUTF();
1435	p++;
1436	if (u == PS \|\| u == LS)
1437	loc.linnum++;
1438	stringbuffer.writeUTF8(u);
1439	continue;
1440	}
1441	break;
1442	}
1443	stringbuffer.writeByte(c);
1444	}
1445	}
1446
1447	/**************************************
1448	* Lex hex strings:
1449	* x"0A ae 34FE BD"
1450	*/
1451
1452	TOK Lexer::hexStringConstant(Token *t)
1453	{ unsigned c;
1454	Loc start = loc;
1455	unsigned n = 0;
1456	unsigned v;
1457
1458	p++;
1459	stringbuffer.reset();
1460	while (1)
1461	{
1462	c = *p++;
1463	switch (c)
1464	{
1465	case ' ':
1466	case '\t':
1467	case '\v':
1468	case '\f':
1469	continue; // skip white space
1470
1471	case '\r':
1472	if (*p == '\n')
1473	continue; // ignore
1474	// Treat isolated '\r' as if it were a '\n'
1475	case '\n':
1476	loc.linnum++;
1477	continue;
1478
1479	case 0:
1480	case 0x1A:
1481	error("unterminated string constant starting at %s", start.toChars());
1482	t->ustring = (unsigned char *)"";
1483	t->len = 0;
1484	t->postfix = 0;
1485	return TOKstring;
1486
1487	case '"':
1488	if (n & 1)
1489	{ error("odd number (%d) of hex characters in hex string", n);
1490	stringbuffer.writeByte(v);
1491	}
1492	t->len = stringbuffer.offset;
1493	stringbuffer.writeByte(0);
1494	t->ustring = (unsigned char *)mem.malloc(stringbuffer.offset);
1495	memcpy(t->ustring, stringbuffer.data, stringbuffer.offset);
1496	stringPostfix(t);
1497	return TOKstring;
1498
1499	default:
1500	if (c >= '0' && c <= '9')
1501	c -= '0';
1502	else if (c >= 'a' && c <= 'f')
1503	c -= 'a' - 10;
1504	else if (c >= 'A' && c <= 'F')
1505	c -= 'A' - 10;
1506	else if (c & 0x80)
1507	{ p--;
1508	unsigned u = decodeUTF();
1509	p++;
1510	if (u == PS \|\| u == LS)
1511	loc.linnum++;
1512	else
1513	error("non-hex character \\u%04x", u);
1514	}
1515	else
1516	error("non-hex character '%c'", c);
1517	if (n & 1)
1518	{ v = (v << 4) \| c;
1519	stringbuffer.writeByte(v);
1520	}
1521	else
1522	v = c;
1523	n++;
1524	break;
1525	}
1526	}
1527	}
1528
1529
1530	#if DMDV2
1531	/**************************************
1532	* Lex delimited strings:
1533	* q"(foo(xxx))" // "foo(xxx)"
1534	* q"[foo(]" // "foo("
1535	* q"/foo]/" // "foo]"
1536	* q"HERE
1537	* foo
1538	* HERE" // "foo\n"
1539	* Input:
1540	* p is on the "
1541	*/
1542
1543	TOK Lexer::delimitedStringConstant(Token *t)
1544	{ unsigned c;
1545	Loc start = loc;
1546	unsigned delimleft = 0;
1547	unsigned delimright = 0;
1548	unsigned nest = 1;
1549	unsigned nestcount;
1550	Identifier *hereid = NULL;
1551	unsigned blankrol = 0;
1552	unsigned startline = 0;
1553
1554	p++;
1555	stringbuffer.reset();
1556	while (1)
1557	{
1558	c = *p++;
1559	//printf("c = '%c'\n", c);
1560	switch (c)
1561	{
1562	case '\n':
1563	Lnextline:
1564	loc.linnum++;
1565	startline = 1;
1566	if (blankrol)
1567	{ blankrol = 0;
1568	continue;
1569	}
1570	if (hereid)
1571	{
1572	stringbuffer.writeUTF8(c);
1573	continue;
1574	}
1575	break;
1576
1577	case '\r':
1578	if (*p == '\n')
1579	continue; // ignore
1580	c = '\n'; // treat EndOfLine as \n character
1581	goto Lnextline;
1582
1583	case 0:
1584	case 0x1A:
1585	goto Lerror;
1586
1587	default:
1588	if (c & 0x80)
1589	{ p--;
1590	c = decodeUTF();
1591	p++;
1592	if (c == PS \|\| c == LS)
1593	goto Lnextline;
1594	}
1595	break;
1596	}
1597	if (delimleft == 0)
1598	{ delimleft = c;
1599	nest = 1;
1600	nestcount = 1;
1601	if (c == '(')
1602	delimright = ')';
1603	else if (c == '{')
1604	delimright = '}';
1605	else if (c == '[')
1606	delimright = ']';
1607	else if (c == '<')
1608	delimright = '>';
1609	else if (isalpha(c) \|\| c == '_' \|\| (c >= 0x80 && isUniAlpha(c)))
1610	{ // Start of identifier; must be a heredoc
1611	Token t;
1612	p--;
1613	scan(&t); // read in heredoc identifier
1614	if (t.value != TOKidentifier)
1615	{ error("identifier expected for heredoc, not %s", t.toChars());
1616	delimright = c;
1617	}
1618	else
1619	{ hereid = t.ident;
1620	//printf("hereid = '%s'\n", hereid->toChars());
1621	blankrol = 1;
1622	}
1623	nest = 0;
1624	}
1625	else
1626	{ delimright = c;
1627	nest = 0;
1628	#if DMDV2
1629	if (isspace(c))
1630	error("delimiter cannot be whitespace");
1631	#endif
1632	}
1633	}
1634	else
1635	{
1636	if (blankrol)
1637	{ error("heredoc rest of line should be blank");
1638	blankrol = 0;
1639	continue;
1640	}
1641	if (nest == 1)
1642	{
1643	if (c == delimleft)
1644	nestcount++;
1645	else if (c == delimright)
1646	{ nestcount--;
1647	if (nestcount == 0)
1648	goto Ldone;
1649	}
1650	}
1651	else if (c == delimright)
1652	goto Ldone;
1653	if (startline && isalpha(c)
1654	#if DMDV2
1655	&& hereid
1656	#endif
1657	)
1658	{ Token t;
1659	unsigned char *psave = p;
1660	p--;
1661	scan(&t); // read in possible heredoc identifier
1662	//printf("endid = '%s'\n", t.ident->toChars());
1663	if (t.value == TOKidentifier && t.ident->equals(hereid))
1664	{ /* should check that rest of line is blank
1665	*/
1666	goto Ldone;
1667	}
1668	p = psave;
1669	}
1670	stringbuffer.writeUTF8(c);
1671	startline = 0;
1672	}
1673	}
1674
1675	Ldone:
1676	if (*p == '"')
1677	p++;
1678	else
1679	error("delimited string must end in %c\"", delimright);
1680	t->len = stringbuffer.offset;
1681	stringbuffer.writeByte(0);
1682	t->ustring = (unsigned char *)mem.malloc(stringbuffer.offset);
1683	memcpy(t->ustring, stringbuffer.data, stringbuffer.offset);
1684	stringPostfix(t);
1685	return TOKstring;
1686
1687	Lerror:
1688	error("unterminated string constant starting at %s", start.toChars());
1689	t->ustring = (unsigned char *)"";
1690	t->len = 0;
1691	t->postfix = 0;
1692	return TOKstring;
1693	}
1694
1695	/**************************************
1696	* Lex delimited strings:
1697	* q{ foo(xxx) } // " foo(xxx) "
1698	* q{foo(} // "foo("
1699	* q{{foo}"}"} // "{foo}"}""
1700	* Input:
1701	* p is on the q
1702	*/
1703
1704	TOK Lexer::tokenStringConstant(Token *t)
1705	{
1706	unsigned nest = 1;
1707	Loc start = loc;
1708	unsigned char *pstart = ++p;
1709
1710	while (1)
1711	{ Token tok;
1712
1713	scan(&tok);
1714	switch (tok.value)
1715	{
1716	case TOKlcurly:
1717	nest++;
1718	continue;
1719
1720	case TOKrcurly:
1721	if (--nest == 0)
1722	goto Ldone;
1723	continue;
1724
1725	case TOKeof:
1726	goto Lerror;
1727
1728	default:
1729	continue;
1730	}
1731	}
1732
1733	Ldone:
1734	t->len = p - 1 - pstart;
1735	t->ustring = (unsigned char *)mem.malloc(t->len + 1);
1736	memcpy(t->ustring, pstart, t->len);
1737	t->ustring[t->len] = 0;
1738	stringPostfix(t);
1739	return TOKstring;
1740
1741	Lerror:
1742	error("unterminated token string constant starting at %s", start.toChars());
1743	t->ustring = (unsigned char *)"";
1744	t->len = 0;
1745	t->postfix = 0;
1746	return TOKstring;
1747	}
1748
1749	#endif
1750
1751
1752	/**************************************
1753	*/
1754
1755	TOK Lexer::escapeStringConstant(Token *t, int wide)
1756	{ unsigned c;
1757	Loc start = loc;
1758
1759	p++;
1760	stringbuffer.reset();
1761	while (1)
1762	{
1763	c = *p++;
1764	switch (c)
1765	{
1766	#if !( TEXTUAL_ASSEMBLY_OUT )
1767	case '\\':
1768	switch (*p)
1769	{
1770	case 'u':
1771	case 'U':
1772	case '&':
1773	c = escapeSequence();
1774	stringbuffer.writeUTF8(c);
1775	continue;
1776
1777	default:
1778	c = escapeSequence();
1779	break;
1780	}
1781	break;
1782	#endif
1783	case '\n':
1784	loc.linnum++;
1785	break;
1786
1787	case '\r':
1788	if (*p == '\n')
1789	continue; // ignore
1790	c = '\n'; // treat EndOfLine as \n character
1791	loc.linnum++;
1792	break;
1793
1794	case '"':
1795	t->len = stringbuffer.offset;
1796	stringbuffer.writeByte(0);
1797	t->ustring = (unsigned char *)mem.malloc(stringbuffer.offset);
1798	memcpy(t->ustring, stringbuffer.data, stringbuffer.offset);
1799	stringPostfix(t);
1800	return TOKstring;
1801
1802	case 0:
1803	case 0x1A:
1804	p--;
1805	error("unterminated string constant starting at %s", start.toChars());
1806	t->ustring = (unsigned char *)"";
1807	t->len = 0;
1808	t->postfix = 0;
1809	return TOKstring;
1810
1811	default:
1812	if (c & 0x80)
1813	{
1814	p--;
1815	c = decodeUTF();
1816	if (c == LS \|\| c == PS)
1817	{ c = '\n';
1818	loc.linnum++;
1819	}
1820	p++;
1821	stringbuffer.writeUTF8(c);
1822	continue;
1823	}
1824	break;
1825	}
1826	stringbuffer.writeByte(c);
1827	}
1828	}
1829
1830	/**************************************
1831	*/
1832
1833	TOK Lexer::charConstant(Token *t, int wide)
1834	{
1835	unsigned c;
1836	TOK tk = TOKcharv;
1837
1838	//printf("Lexer::charConstant\n");
1839	p++;
1840	c = *p++;
1841	switch (c)
1842	{
1843	#if ! TEXTUAL_ASSEMBLY_OUT
1844	case '\\':
1845	switch (*p)
1846	{
1847	case 'u':
1848	t->uns64value = escapeSequence();
1849	tk = TOKwcharv;
1850	break;
1851
1852	case 'U':
1853	case '&':
1854	t->uns64value = escapeSequence();
1855	tk = TOKdcharv;
1856	break;
1857
1858	default:
1859	t->uns64value = escapeSequence();
1860	break;
1861	}
1862	break;
1863	#endif
1864	case '\n':
1865	L1:
1866	loc.linnum++;
1867	case '\r':
1868	case 0:
1869	case 0x1A:
1870	case '\'':
1871	error("unterminated character constant");
1872	return tk;
1873
1874	default:
1875	if (c & 0x80)
1876	{
1877	p--;
1878	c = decodeUTF();
1879	p++;
1880	if (c == LS \|\| c == PS)
1881	goto L1;
1882	if (c < 0xD800 \|\| (c >= 0xE000 && c < 0xFFFE))
1883	tk = TOKwcharv;
1884	else
1885	tk = TOKdcharv;
1886	}
1887	t->uns64value = c;
1888	break;
1889	}
1890
1891	if (*p != '\'')
1892	{ error("unterminated character constant");
1893	return tk;
1894	}
1895	p++;
1896	return tk;
1897	}
1898
1899	/***************************************
1900	* Get postfix of string literal.
1901	*/
1902
1903	void Lexer::stringPostfix(Token *t)
1904	{
1905	switch (*p)
1906	{
1907	case 'c':
1908	case 'w':
1909	case 'd':
1910	t->postfix = *p;
1911	p++;
1912	break;
1913
1914	default:
1915	t->postfix = 0;
1916	break;
1917	}
1918	}
1919
1920	/***************************************
1921	* Read \u or \U unicode sequence
1922	* Input:
1923	* u 'u' or 'U'
1924	*/
1925
1926	#if 0
1927	unsigned Lexer::wchar(unsigned u)
1928	{
1929	unsigned value;
1930	unsigned n;
1931	unsigned char c;
1932	unsigned nchars;
1933
1934	nchars = (u == 'U') ? 8 : 4;
1935	value = 0;
1936	for (n = 0; 1; n++)
1937	{
1938	++p;
1939	if (n == nchars)
1940	break;
1941	c = *p;
1942	if (!ishex(c))
1943	{ error("\\%c sequence must be followed by %d hex characters", u, nchars);
1944	break;
1945	}
1946	if (isdigit(c))
1947	c -= '0';
1948	else if (islower(c))
1949	c -= 'a' - 10;
1950	else
1951	c -= 'A' - 10;
1952	value <<= 4;
1953	value \|= c;
1954	}
1955	return value;
1956	}
1957	#endif
1958
1959	/**************************************
1960	* Read in a number.
1961	* If it's an integer, store it in tok.TKutok.Vlong.
1962	* integers can be decimal, octal or hex
1963	* Handle the suffixes U, UL, LU, L, etc.
1964	* If it's double, store it in tok.TKutok.Vdouble.
1965	* Returns:
1966	* TKnum
1967	* TKdouble,...
1968	*/
1969
1970	TOK Lexer::number(Token *t)
1971	{
1972	// We use a state machine to collect numbers
1973	enum STATE { STATE_initial, STATE_0, STATE_decimal, STATE_octal, STATE_octale,
1974	STATE_hex, STATE_binary, STATE_hex0, STATE_binary0,
1975	STATE_hexh, STATE_error };
1976	enum STATE state;
1977
1978	enum FLAGS
1979	{ FLAGS_decimal = 1, // decimal
1980	FLAGS_unsigned = 2, // u or U suffix
1981	FLAGS_long = 4, // l or L suffix
1982	};
1983	enum FLAGS flags = FLAGS_decimal;
1984
1985	int i;
1986	int base;
1987	unsigned c;
1988	unsigned char *start;
1989	TOK result;
1990
1991	//printf("Lexer::number()\n");
1992	state = STATE_initial;
1993	base = 0;
1994	stringbuffer.reset();
1995	start = p;
1996	while (1)
1997	{
1998	c = *p;
1999	switch (state)
2000	{
2001	case STATE_initial: // opening state
2002	if (c == '0')
2003	state = STATE_0;
2004	else
2005	state = STATE_decimal;
2006	break;
2007
2008	case STATE_0:
2009	flags = (FLAGS) (flags & ~FLAGS_decimal);
2010	switch (c)
2011	{
2012	#if ZEROH
2013	case 'H': // 0h
2014	case 'h':
2015	goto hexh;
2016	#endif
2017	case 'X':
2018	case 'x':
2019	state = STATE_hex0;
2020	break;
2021
2022	case '.':
2023	if (p[1] == '.') // .. is a separate token
2024	goto done;
2025	case 'i':
2026	case 'f':
2027	case 'F':
2028	goto real;
2029	#if ZEROH
2030	case 'E':
2031	case 'e':
2032	goto case_hex;
2033	#endif
2034	case 'B':
2035	case 'b':
2036	state = STATE_binary0;
2037	break;
2038
2039	case '0': case '1': case '2': case '3':
2040	case '4': case '5': case '6': case '7':
2041	state = STATE_octal;
2042	break;
2043
2044	#if ZEROH
2045	case '8': case '9': case 'A':
2046	case 'C': case 'D': case 'F':
2047	case 'a': case 'c': case 'd': case 'f':
2048	case_hex:
2049	state = STATE_hexh;
2050	break;
2051	#endif
2052	case '_':
2053	state = STATE_octal;
2054	p++;
2055	continue;
2056
2057	case 'L':
2058	if (p[1] == 'i')
2059	goto real;
2060	goto done;
2061
2062	default:
2063	goto done;
2064	}
2065	break;
2066
2067	case STATE_decimal: // reading decimal number
2068	if (!isdigit(c))
2069	{
2070	#if ZEROH
2071	if (ishex(c)
2072	\|\| c == 'H' \|\| c == 'h'
2073	)
2074	goto hexh;
2075	#endif
2076	if (c == '_') // ignore embedded _
2077	{ p++;
2078	continue;
2079	}
2080	if (c == '.' && p[1] != '.')
2081	goto real;
2082	else if (c == 'i' \|\| c == 'f' \|\| c == 'F' \|\|
2083	c == 'e' \|\| c == 'E')
2084	{
2085	real: // It's a real number. Back up and rescan as a real
2086	p = start;
2087	return inreal(t);
2088	}
2089	else if (c == 'L' && p[1] == 'i')
2090	goto real;
2091	goto done;
2092	}
2093	break;
2094
2095	case STATE_hex0: // reading hex number
2096	case STATE_hex:
2097	if (!ishex(c))
2098	{
2099	if (c == '_') // ignore embedded _
2100	{ p++;
2101	continue;
2102	}
2103	if (c == '.' && p[1] != '.')
2104	goto real;
2105	if (c == 'P' \|\| c == 'p' \|\| c == 'i')
2106	goto real;
2107	if (state == STATE_hex0)
2108	error("Hex digit expected, not '%c'", c);
2109	goto done;
2110	}
2111	state = STATE_hex;
2112	break;
2113
2114	#if ZEROH
2115	hexh:
2116	state = STATE_hexh;
2117	case STATE_hexh: // parse numbers like 0FFh
2118	if (!ishex(c))
2119	{
2120	if (c == 'H' \|\| c == 'h')
2121	{
2122	p++;
2123	base = 16;
2124	goto done;
2125	}
2126	else
2127	{
2128	// Check for something like 1E3 or 0E24
2129	if (memchr((char *)stringbuffer.data, 'E', stringbuffer.offset) \|\|
2130	memchr((char *)stringbuffer.data, 'e', stringbuffer.offset))
2131	goto real;
2132	error("Hex digit expected, not '%c'", c);
2133	goto done;
2134	}
2135	}
2136	break;
2137	#endif
2138
2139	case STATE_octal: // reading octal number
2140	case STATE_octale: // reading octal number with non-octal digits
2141	if (!isoctal(c))
2142	{
2143	#if ZEROH
2144	if (ishex(c)
2145	\|\| c == 'H' \|\| c == 'h'
2146	)
2147	goto hexh;
2148	#endif
2149	if (c == '_') // ignore embedded _
2150	{ p++;
2151	continue;
2152	}
2153	if (c == '.' && p[1] != '.')
2154	goto real;
2155	if (c == 'i')
2156	goto real;
2157	if (isdigit(c))
2158	{
2159	state = STATE_octale;
2160	}
2161	else
2162	goto done;
2163	}
2164	break;
2165
2166	case STATE_binary0: // starting binary number
2167	case STATE_binary: // reading binary number
2168	if (c != '0' && c != '1')
2169	{
2170	#if ZEROH
2171	if (ishex(c)
2172	\|\| c == 'H' \|\| c == 'h'
2173	)
2174	goto hexh;
2175	#endif
2176	if (c == '_') // ignore embedded _
2177	{ p++;
2178	continue;
2179	}
2180	if (state == STATE_binary0)
2181	{ error("binary digit expected");
2182	state = STATE_error;
2183	break;
2184	}
2185	else
2186	goto done;
2187	}
2188	state = STATE_binary;
2189	break;
2190
2191	case STATE_error: // for error recovery
2192	if (!isdigit(c)) // scan until non-digit
2193	goto done;
2194	break;
2195
2196	default:
2197	assert(0);
2198	}
2199	stringbuffer.writeByte(c);
2200	p++;
2201	}
2202	done:
2203	stringbuffer.writeByte(0); // terminate string
2204	if (state == STATE_octale)
2205	error("Octal digit expected");
2206
2207	uinteger_t n; // unsigned >=64 bit integer type
2208
2209	if (stringbuffer.offset == 2 && (state == STATE_decimal \|\| state == STATE_0))
2210	n = stringbuffer.data[0] - '0';
2211	else
2212	{
2213	// Convert string to integer
2214	#if __DMC__
2215	errno = 0;
2216	n = strtoull((char *)stringbuffer.data,NULL,base);
2217	if (errno == ERANGE)
2218	error("integer overflow");
2219	#else
2220	// Not everybody implements strtoull()
2221	char p = (char )stringbuffer.data;
2222	int r = 10, d;
2223
2224	if (*p == '0')
2225	{
2226	if (p[1] == 'x' \|\| p[1] == 'X')
2227	p += 2, r = 16;
2228	else if (p[1] == 'b' \|\| p[1] == 'B')
2229	p += 2, r = 2;
2230	else if (isdigit(p[1]))
2231	p += 1, r = 8;
2232	}
2233
2234	n = 0;
2235	while (1)
2236	{
2237	if (p >= '0' && p <= '9')
2238	d = *p - '0';
2239	else if (p >= 'a' && p <= 'z')
2240	d = *p - 'a' + 10;
2241	else if (p >= 'A' && p <= 'Z')
2242	d = *p - 'A' + 10;
2243	else
2244	break;
2245	if (d >= r)
2246	break;
2247	uinteger_t n2 = n * r;
2248	//printf("n2 / r = %llx, n = %llx\n", n2/r, n);
2249	if (n2 / r != n \|\| n2 + d < n)
2250	{
2251	error ("integer overflow");
2252	break;
2253	}
2254
2255	n = n2 + d;
2256	p++;
2257	}
2258	#endif
2259	if (sizeof(n) > 8 &&
2260	n > 0xFFFFFFFFFFFFFFFFULL) // if n needs more than 64 bits
2261	error("integer overflow");
2262	}
2263
2264	// Parse trailing 'u', 'U', 'l' or 'L' in any combination
2265	while (1)
2266	{ unsigned char f;
2267
2268	switch (*p)
2269	{ case 'U':
2270	case 'u':
2271	f = FLAGS_unsigned;
2272	goto L1;
2273
2274	case 'l':
2275	if (1 \|\| !global.params.useDeprecated)
2276	error("'l' suffix is deprecated, use 'L' instead");
2277	case 'L':
2278	f = FLAGS_long;
2279	L1:
2280	p++;
2281	if (flags & f)
2282	error("unrecognized token");
2283	flags = (FLAGS) (flags \| f);
2284	continue;
2285	default:
2286	break;
2287	}
2288	break;
2289	}
2290
2291	switch (flags)
2292	{
2293	case 0:
2294	/* Octal or Hexadecimal constant.
2295	* First that fits: int, uint, long, ulong
2296	*/
2297	if (n & 0x8000000000000000LL)
2298	result = TOKuns64v;
2299	else if (n & 0xFFFFFFFF00000000LL)
2300	result = TOKint64v;
2301	else if (n & 0x80000000)
2302	result = TOKuns32v;
2303	else
2304	result = TOKint32v;
2305	break;
2306
2307	case FLAGS_decimal:
2308	/* First that fits: int, long, long long
2309	*/
2310	if (n & 0x8000000000000000LL)
2311	{ error("signed integer overflow");
2312	result = TOKuns64v;
2313	}
2314	else if (n & 0xFFFFFFFF80000000LL)
2315	result = TOKint64v;
2316	else
2317	result = TOKint32v;
2318	break;
2319
2320	case FLAGS_unsigned:
2321	case FLAGS_decimal \| FLAGS_unsigned:
2322	/* First that fits: uint, ulong
2323	*/
2324	if (n & 0xFFFFFFFF00000000LL)
2325	result = TOKuns64v;
2326	else
2327	result = TOKuns32v;
2328	break;
2329
2330	case FLAGS_decimal \| FLAGS_long:
2331	if (n & 0x8000000000000000LL)
2332	{ error("signed integer overflow");
2333	result = TOKuns64v;
2334	}
2335	else
2336	result = TOKint64v;
2337	break;
2338
2339	case FLAGS_long:
2340	if (n & 0x8000000000000000LL)
2341	result = TOKuns64v;
2342	else
2343	result = TOKint64v;
2344	break;
2345
2346	case FLAGS_unsigned \| FLAGS_long:
2347	case FLAGS_decimal \| FLAGS_unsigned \| FLAGS_long:
2348	result = TOKuns64v;
2349	break;
2350
2351	default:
2352	#ifdef DEBUG
2353	printf("%x\n",flags);
2354	#endif
2355	assert(0);
2356	}
2357	t->uns64value = n;
2358	return result;
2359	}
2360
2361	/**************************************
2362	* Read in characters, converting them to real.
2363	* Bugs:
2364	* Exponent overflow not detected.
2365	* Too much requested precision is not detected.
2366	*/
2367
2368	TOK Lexer::inreal(Token *t)
2369	#ifdef __DMC__
2370	__in
2371	{
2372	assert(p == '.' \|\| isdigit(p));
2373	}
2374	__out (result)
2375	{
2376	switch (result)
2377	{
2378	case TOKfloat32v:
2379	case TOKfloat64v:
2380	case TOKfloat80v:
2381	case TOKimaginary32v:
2382	case TOKimaginary64v:
2383	case TOKimaginary80v:
2384	break;
2385
2386	default:
2387	assert(0);
2388	}
2389	}
2390	__body
2391	#endif /* __DMC__ */
2392	{ int dblstate;
2393	unsigned c;
2394	char hex; // is this a hexadecimal-floating-constant?
2395	TOK result;
2396
2397	//printf("Lexer::inreal()\n");
2398	stringbuffer.reset();
2399	dblstate = 0;
2400	hex = 0;
2401	Lnext:
2402	while (1)
2403	{
2404	// Get next char from input
2405	c = *p++;
2406	//printf("dblstate = %d, c = '%c'\n", dblstate, c);
2407	while (1)
2408	{
2409	switch (dblstate)
2410	{
2411	case 0: // opening state
2412	if (c == '0')
2413	dblstate = 9;
2414	else if (c == '.')
2415	dblstate = 3;
2416	else
2417	dblstate = 1;
2418	break;
2419
2420	case 9:
2421	dblstate = 1;
2422	if (c == 'X' \|\| c == 'x')
2423	{ hex++;
2424	break;
2425	}
2426	case 1: // digits to left of .
2427	case 3: // digits to right of .
2428	case 7: // continuing exponent digits
2429	if (!isdigit(c) && !(hex && isxdigit(c)))
2430	{
2431	if (c == '_')
2432	goto Lnext; // ignore embedded '_'
2433	dblstate++;
2434	continue;
2435	}
2436	break;
2437
2438	case 2: // no more digits to left of .
2439	if (c == '.')
2440	{ dblstate++;
2441	break;
2442	}
2443	case 4: // no more digits to right of .
2444	if ((c == 'E' \|\| c == 'e') \|\|
2445	hex && (c == 'P' \|\| c == 'p'))
2446	{ dblstate = 5;
2447	hex = 0; // exponent is always decimal
2448	break;
2449	}
2450	if (hex)
2451	error("binary-exponent-part required");
2452	goto done;
2453
2454	case 5: // looking immediately to right of E
2455	dblstate++;
2456	if (c == '-' \|\| c == '+')
2457	break;
2458	case 6: // 1st exponent digit expected
2459	if (!isdigit(c))
2460	error("exponent expected");
2461	dblstate++;
2462	break;
2463
2464	case 8: // past end of exponent digits
2465	goto done;
2466	}
2467	break;
2468	}
2469	stringbuffer.writeByte(c);
2470	}
2471	done:
2472	p--;
2473
2474	stringbuffer.writeByte(0);
2475
2476	#if _WIN32 && __DMC__
2477	char *save = __locale_decpoint;
2478	__locale_decpoint = ".";
2479	#endif
2480	#ifdef IN_GCC
2481	t->float80value = real_t::parse((char *)stringbuffer.data, real_t::LongDouble);
2482	#else
2483	t->float80value = strtold((char *)stringbuffer.data, NULL);
2484	#endif
2485	errno = 0;
2486	switch (*p)
2487	{
2488	case 'F':
2489	case 'f':
2490	#ifdef IN_GCC
2491	real_t::parse((char *)stringbuffer.data, real_t::Float);
2492	#else
2493	{ // Only interested in errno return
2494	float f = strtof((char *)stringbuffer.data, NULL);
2495	// Assign to f to keep gcc warnings at bay
2496	}
2497	#endif
2498	result = TOKfloat32v;
2499	p++;
2500	break;
2501
2502	default:
2503	#ifdef IN_GCC
2504	real_t::parse((char *)stringbuffer.data, real_t::Double);
2505	#else
2506	/* Should do our own strtod(), since dmc and linux gcc
2507	* accept 2.22507e-308, while apple gcc will only take
2508	* 2.22508e-308. Not sure who is right.
2509	*/
2510	{ // Only interested in errno return
2511	double d = strtod((char *)stringbuffer.data, NULL);
2512	// Assign to d to keep gcc warnings at bay
2513	}
2514	#endif
2515	result = TOKfloat64v;
2516	break;
2517
2518	case 'l':
2519	if (!global.params.useDeprecated)
2520	error("'l' suffix is deprecated, use 'L' instead");
2521	case 'L':
2522	result = TOKfloat80v;
2523	p++;
2524	break;
2525	}
2526	if (p == 'i' \|\| p == 'I')
2527	{
2528	if (!global.params.useDeprecated && *p == 'I')
2529	error("'I' suffix is deprecated, use 'i' instead");
2530	p++;
2531	switch (result)
2532	{
2533	case TOKfloat32v:
2534	result = TOKimaginary32v;
2535	break;
2536	case TOKfloat64v:
2537	result = TOKimaginary64v;
2538	break;
2539	case TOKfloat80v:
2540	result = TOKimaginary80v;
2541	break;
2542	}
2543	}
2544	#if _WIN32 && __DMC__
2545	__locale_decpoint = save;
2546	#endif
2547	if (errno == ERANGE)
2548	error("number is not representable");
2549	return result;
2550	}
2551
2552	/*********************************************
2553	* Do pragma.
2554	* Currently, the only pragma supported is:
2555	* #line linnum [filespec]
2556	*/
2557
2558	void Lexer::pragma()
2559	{
2560	Token tok;
2561	int linnum;
2562	char *filespec = NULL;
2563	Loc loc = this->loc;
2564
2565	scan(&tok);
2566	if (tok.value != TOKidentifier \|\| tok.ident != Id::line)
2567	goto Lerr;
2568
2569	scan(&tok);
2570	if (tok.value == TOKint32v \|\| tok.value == TOKint64v)
2571	linnum = tok.uns64value - 1;
2572	else
2573	goto Lerr;
2574
2575	while (1)
2576	{
2577	switch (*p)
2578	{
2579	case 0:
2580	case 0x1A:
2581	case '\n':
2582	Lnewline:
2583	this->loc.linnum = linnum;
2584	if (filespec)
2585	this->loc.filename = filespec;
2586	return;
2587
2588	case '\r':
2589	p++;
2590	if (*p != '\n')
2591	{ p--;
2592	goto Lnewline;
2593	}
2594	continue;
2595
2596	case ' ':
2597	case '\t':
2598	case '\v':
2599	case '\f':
2600	p++;
2601	continue; // skip white space
2602
2603	case '_':
2604	if (mod && memcmp(p, "__FILE__", 8) == 0)
2605	{
2606	p += 8;
2607	filespec = mem.strdup(loc.filename ? loc.filename : mod->ident->toChars());
2608	}
2609	continue;
2610
2611	case '"':
2612	if (filespec)
2613	goto Lerr;
2614	stringbuffer.reset();
2615	p++;
2616	while (1)
2617	{ unsigned c;
2618
2619	c = *p;
2620	switch (c)
2621	{
2622	case '\n':
2623	case '\r':
2624	case 0:
2625	case 0x1A:
2626	goto Lerr;
2627
2628	case '"':
2629	stringbuffer.writeByte(0);
2630	filespec = mem.strdup((char *)stringbuffer.data);
2631	p++;
2632	break;
2633
2634	default:
2635	if (c & 0x80)
2636	{ unsigned u = decodeUTF();
2637	if (u == PS \|\| u == LS)
2638	goto Lerr;
2639	}
2640	stringbuffer.writeByte(c);
2641	p++;
2642	continue;
2643	}
2644	break;
2645	}
2646	continue;
2647
2648	default:
2649	if (*p & 0x80)
2650	{ unsigned u = decodeUTF();
2651	if (u == PS \|\| u == LS)
2652	goto Lnewline;
2653	}
2654	goto Lerr;
2655	}
2656	}
2657
2658	Lerr:
2659	error(loc, "#line integer [\"filespec\"]\\n expected");
2660	}
2661
2662
2663	/********************************************
2664	* Decode UTF character.
2665	* Issue error messages for invalid sequences.
2666	* Return decoded character, advance p to last character in UTF sequence.
2667	*/
2668
2669	unsigned Lexer::decodeUTF()
2670	{
2671	dchar_t u;
2672	unsigned char c;
2673	unsigned char *s = p;
2674	size_t len;
2675	size_t idx;
2676	const char *msg;
2677
2678	c = *s;
2679	assert(c & 0x80);
2680
2681	// Check length of remaining string up to 6 UTF-8 characters
2682	for (len = 1; len < 6 && s[len]; len++)
2683	;
2684
2685	idx = 0;
2686	msg = utf_decodeChar(s, len, &idx, &u);
2687	p += idx - 1;
2688	if (msg)
2689	{
2690	error("%s", msg);
2691	}
2692	return u;
2693	}
2694
2695
2696	/***************************************************
2697	* Parse doc comment embedded between t->ptr and p.
2698	* Remove trailing blanks and tabs from lines.
2699	* Replace all newlines with \n.
2700	* Remove leading comment character from each line.
2701	* Decide if it's a lineComment or a blockComment.
2702	* Append to previous one for this token.
2703	*/
2704
2705	void Lexer::getDocComment(Token *t, unsigned lineComment)
2706	{
2707	/* ct tells us which kind of comment it is: '/', '*', or '+'
2708	*/
2709	unsigned char ct = t->ptr[2];
2710
2711	/* Start of comment text skips over / * *, / + +, or / / /
2712	*/
2713	unsigned char *q = t->ptr + 3; // start of comment text
2714
2715	unsigned char *qend = p;
2716	if (ct == '*' \|\| ct == '+')
2717	qend -= 2;
2718
2719	/* Scan over initial row of ****'s or ++++'s or ////'s
2720	*/
2721	for (; q < qend; q++)
2722	{
2723	if (*q != ct)
2724	break;
2725	}
2726
2727	/* Remove trailing row of ****'s or ++++'s
2728	*/
2729	if (ct != '/')
2730	{
2731	for (; q < qend; qend--)
2732	{
2733	if (qend[-1] != ct)
2734	break;
2735	}
2736	}
2737
2738	/* Comment is now [q .. qend].
2739	* Canonicalize it into buf[].
2740	*/
2741	OutBuffer buf;
2742	int linestart = 0;
2743
2744	for (; q < qend; q++)
2745	{
2746	unsigned char c = *q;
2747
2748	switch (c)
2749	{
2750	case '*':
2751	case '+':
2752	if (linestart && c == ct)
2753	{ linestart = 0;
2754	/* Trim preceding whitespace up to preceding \n
2755	*/
2756	while (buf.offset && (buf.data[buf.offset - 1] == ' ' \|\| buf.data[buf.offset - 1] == '\t'))
2757	buf.offset--;
2758	continue;
2759	}
2760	break;
2761
2762	case ' ':
2763	case '\t':
2764	break;
2765
2766	case '\r':
2767	if (q[1] == '\n')
2768	continue; // skip the \r
2769	goto Lnewline;
2770
2771	default:
2772	if (c == 226)
2773	{
2774	// If LS or PS
2775	if (q[1] == 128 &&
2776	(q[2] == 168 \|\| q[2] == 169))
2777	{
2778	q += 2;
2779	goto Lnewline;
2780	}
2781	}
2782	linestart = 0;
2783	break;
2784
2785	Lnewline:
2786	c = '\n'; // replace all newlines with \n
2787	case '\n':
2788	linestart = 1;
2789
2790	/* Trim trailing whitespace
2791	*/
2792	while (buf.offset && (buf.data[buf.offset - 1] == ' ' \|\| buf.data[buf.offset - 1] == '\t'))
2793	buf.offset--;
2794
2795	break;
2796	}
2797	buf.writeByte(c);
2798	}
2799
2800	// Always end with a newline
2801	if (!buf.offset \|\| buf.data[buf.offset - 1] != '\n')
2802	buf.writeByte('\n');
2803
2804	buf.writeByte(0);
2805
2806	// It's a line comment if the start of the doc comment comes
2807	// after other non-whitespace on the same line.
2808	unsigned char** dc = (lineComment && anyToken)
2809	? &t->lineComment
2810	: &t->blockComment;
2811
2812	// Combine with previous doc comment, if any
2813	if (*dc)
2814	dc = combineComments(dc, (unsigned char *)buf.data);
2815	else
2816	dc = (unsigned char )buf.extractData();
2817	}
2818
2819	/********************************************
2820	* Combine two document comments into one,
2821	* separated by a newline.
2822	*/
2823
2824	unsigned char Lexer::combineComments(unsigned char c1, unsigned char *c2)
2825	{
2826	//printf("Lexer::combineComments('%s', '%s')\n", c1, c2);
2827
2828	unsigned char *c = c2;
2829
2830	if (c1)
2831	{ c = c1;
2832	if (c2)
2833	{ size_t len1 = strlen((char *)c1);
2834	size_t len2 = strlen((char *)c2);
2835
2836	c = (unsigned char *)mem.malloc(len1 + 1 + len2 + 1);
2837	memcpy(c, c1, len1);
2838	if (len1 && c1[len1 - 1] != '\n')
2839	{ c[len1] = '\n';
2840	len1++;
2841	}
2842	memcpy(c + len1, c2, len2);
2843	c[len1 + len2] = 0;
2844	}
2845	}
2846	return c;
2847	}
2848
2849	/********************************************
2850	* Create an identifier in the string table.
2851	*/
2852
2853	Identifier Lexer::idPool(const char s)
2854	{
2855	size_t len = strlen(s);
2856	StringValue *sv = stringtable.update(s, len);
2857	Identifier id = (Identifier ) sv->ptrvalue;
2858	if (!id)
2859	{
2860	id = new Identifier(sv->lstring.string, TOKidentifier);
2861	sv->ptrvalue = id;
2862	}
2863	return id;
2864	}
2865
2866	/*********************************************
2867	* Create a unique identifier using the prefix s.
2868	*/
2869
2870	Identifier Lexer::uniqueId(const char s, int num)
2871	{ char buffer[32];
2872	size_t slen = strlen(s);
2873
2874	assert(slen + sizeof(num) * 3 + 1 <= sizeof(buffer));
2875	sprintf(buffer, "%s%d", s, num);
2876	return idPool(buffer);
2877	}
2878
2879	Identifier Lexer::uniqueId(const char s)
2880	{
2881	static int num;
2882	return uniqueId(s, ++num);
2883	}
2884
2885	/****************************************
2886	*/
2887
2888	struct Keyword
2889	{ const char *name;
2890	enum TOK value;
2891	};
2892
2893	static Keyword keywords[] =
2894	{
2895	// { "", TOK },
2896
2897	{ "this", TOKthis },
2898	{ "super", TOKsuper },
2899	{ "assert", TOKassert },
2900	{ "null", TOKnull },
2901	{ "true", TOKtrue },
2902	{ "false", TOKfalse },
2903	{ "cast", TOKcast },
2904	{ "new", TOKnew },
2905	{ "delete", TOKdelete },
2906	{ "throw", TOKthrow },
2907	{ "module", TOKmodule },
2908	{ "pragma", TOKpragma },
2909	{ "typeof", TOKtypeof },
2910	{ "typeid", TOKtypeid },
2911
2912	{ "template", TOKtemplate },
2913
2914	{ "void", TOKvoid },
2915	{ "byte", TOKint8 },
2916	{ "ubyte", TOKuns8 },
2917	{ "short", TOKint16 },
2918	{ "ushort", TOKuns16 },
2919	{ "int", TOKint32 },
2920	{ "uint", TOKuns32 },
2921	{ "long", TOKint64 },
2922	{ "ulong", TOKuns64 },
2923	{ "cent", TOKcent, },
2924	{ "ucent", TOKucent, },
2925	{ "float", TOKfloat32 },
2926	{ "double", TOKfloat64 },
2927	{ "real", TOKfloat80 },
2928
2929	{ "bool", TOKbool },
2930	{ "char", TOKchar },
2931	{ "wchar", TOKwchar },
2932	{ "dchar", TOKdchar },
2933
2934	{ "ifloat", TOKimaginary32 },
2935	{ "idouble", TOKimaginary64 },
2936	{ "ireal", TOKimaginary80 },
2937
2938	{ "cfloat", TOKcomplex32 },
2939	{ "cdouble", TOKcomplex64 },
2940	{ "creal", TOKcomplex80 },
2941
2942	{ "delegate", TOKdelegate },
2943	{ "function", TOKfunction },
2944
2945	{ "is", TOKis },
2946	{ "if", TOKif },
2947	{ "else", TOKelse },
2948	{ "while", TOKwhile },
2949	{ "for", TOKfor },
2950	{ "do", TOKdo },
2951	{ "switch", TOKswitch },
2952	{ "case", TOKcase },
2953	{ "default", TOKdefault },
2954	{ "break", TOKbreak },
2955	{ "continue", TOKcontinue },
2956	{ "synchronized", TOKsynchronized },
2957	{ "return", TOKreturn },
2958	{ "goto", TOKgoto },
2959	{ "try", TOKtry },
2960	{ "catch", TOKcatch },
2961	{ "finally", TOKfinally },
2962	{ "with", TOKwith },
2963	{ "asm", TOKasm },
2964	{ "foreach", TOKforeach },
2965	{ "foreach_reverse", TOKforeach_reverse },
2966	{ "scope", TOKscope },
2967
2968	{ "struct", TOKstruct },
2969	{ "class", TOKclass },
2970	{ "interface", TOKinterface },
2971	{ "union", TOKunion },
2972	{ "enum", TOKenum },
2973	{ "import", TOKimport },
2974	{ "mixin", TOKmixin },
2975	{ "static", TOKstatic },
2976	{ "final", TOKfinal },
2977	{ "const", TOKconst },
2978	{ "typedef", TOKtypedef },
2979	{ "alias", TOKalias },
2980	{ "override", TOKoverride },
2981	{ "abstract", TOKabstract },
2982	{ "volatile", TOKvolatile },
2983	{ "debug", TOKdebug },
2984	{ "deprecated", TOKdeprecated },
2985	{ "in", TOKin },
2986	{ "out", TOKout },
2987	{ "inout", TOKinout },
2988	{ "lazy", TOKlazy },
2989	{ "auto", TOKauto },
2990
2991	{ "align", TOKalign },
2992	{ "extern", TOKextern },
2993	{ "private", TOKprivate },
2994	{ "package", TOKpackage },
2995	{ "protected", TOKprotected },
2996	{ "public", TOKpublic },
2997	{ "export", TOKexport },
2998
2999	{ "body", TOKbody },
3000	{ "invariant", TOKinvariant },
3001	{ "unittest", TOKunittest },
3002	{ "version", TOKversion },
3003	//{ "manifest", TOKmanifest },
3004
3005	// Added after 1.0
3006	{ "__argTypes", TOKargTypes },
3007	{ "ref", TOKref },
3008	{ "macro", TOKmacro },
3009	#if DMDV2
3010	{ "pure", TOKpure },
3011	{ "nothrow", TOKnothrow },
3012	{ "__thread", TOKtls },
3013	{ "__gshared", TOKgshared },
3014	{ "__traits", TOKtraits },
3015	{ "__overloadset", TOKoverloadset },
3016	{ "__FILE__", TOKfile },
3017	{ "__LINE__", TOKline },
3018	{ "shared", TOKshared },
3019	{ "immutable", TOKimmutable },
3020	#endif
3021	};
3022
3023	int Token::isKeyword()
3024	{
3025	for (unsigned u = 0; u < sizeof(keywords) / sizeof(keywords[0]); u++)
3026	{
3027	if (keywords[u].value == value)
3028	return 1;
3029	}
3030	return 0;
3031	}
3032
3033	void Lexer::initKeywords()
3034	{ StringValue *sv;
3035	unsigned u;
3036	enum TOK v;
3037	unsigned nkeywords = sizeof(keywords) / sizeof(keywords[0]);
3038
3039	if (global.params.Dversion == 1)
3040	nkeywords -= 2;
3041
3042	cmtable_init();
3043
3044	for (u = 0; u < nkeywords; u++)
3045	{ const char *s;
3046
3047	//printf("keyword[%d] = '%s'\n",u, keywords[u].name);
3048	s = keywords[u].name;
3049	v = keywords[u].value;
3050	sv = stringtable.insert(s, strlen(s));
3051	sv->ptrvalue = (void *) new Identifier(sv->lstring.string,v);
3052
3053	//printf("tochars[%d] = '%s'\n",v, s);
3054	Token::tochars[v] = s;
3055	}
3056
3057	Token::tochars[TOKeof] = "EOF";
3058	Token::tochars[TOKlcurly] = "{";
3059	Token::tochars[TOKrcurly] = "}";
3060	Token::tochars[TOKlparen] = "(";
3061	Token::tochars[TOKrparen] = ")";
3062	Token::tochars[TOKlbracket] = "[";
3063	Token::tochars[TOKrbracket] = "]";
3064	Token::tochars[TOKsemicolon] = ";";
3065	Token::tochars[TOKcolon] = ":";
3066	Token::tochars[TOKcomma] = ",";
3067	Token::tochars[TOKdot] = ".";
3068	Token::tochars[TOKxor] = "^";
3069	Token::tochars[TOKxorass] = "^=";
3070	Token::tochars[TOKassign] = "=";
3071	Token::tochars[TOKconstruct] = "=";
3072	#if DMDV2
3073	Token::tochars[TOKblit] = "=";
3074	#endif
3075	Token::tochars[TOKlt] = "<";
3076	Token::tochars[TOKgt] = ">";
3077	Token::tochars[TOKle] = "<=";
3078	Token::tochars[TOKge] = ">=";
3079	Token::tochars[TOKequal] = "==";
3080	Token::tochars[TOKnotequal] = "!=";
3081	Token::tochars[TOKnotidentity] = "!is";
3082	Token::tochars[TOKtobool] = "!!";
3083
3084	Token::tochars[TOKunord] = "!<>=";
3085	Token::tochars[TOKue] = "!<>";
3086	Token::tochars[TOKlg] = "<>";
3087	Token::tochars[TOKleg] = "<>=";
3088	Token::tochars[TOKule] = "!>";
3089	Token::tochars[TOKul] = "!>=";
3090	Token::tochars[TOKuge] = "!<";
3091	Token::tochars[TOKug] = "!<=";
3092
3093	Token::tochars[TOKnot] = "!";
3094	Token::tochars[TOKtobool] = "!!";
3095	Token::tochars[TOKshl] = "<<";
3096	Token::tochars[TOKshr] = ">>";
3097	Token::tochars[TOKushr] = ">>>";
3098	Token::tochars[TOKadd] = "+";
3099	Token::tochars[TOKmin] = "-";
3100	Token::tochars[TOKmul] = "*";
3101	Token::tochars[TOKdiv] = "/";
3102	Token::tochars[TOKmod] = "%";
3103	Token::tochars[TOKslice] = "..";
3104	Token::tochars[TOKdotdotdot] = "...";
3105	Token::tochars[TOKand] = "&";
3106	Token::tochars[TOKandand] = "&&";
3107	Token::tochars[TOKor] = "\|";
3108	Token::tochars[TOKoror] = "\|\|";
3109	Token::tochars[TOKarray] = "[]";
3110	Token::tochars[TOKindex] = "[i]";
3111	Token::tochars[TOKaddress] = "&";
3112	Token::tochars[TOKstar] = "*";
3113	Token::tochars[TOKtilde] = "~";
3114	Token::tochars[TOKdollar] = "$";
3115	Token::tochars[TOKcast] = "cast";
3116	Token::tochars[TOKplusplus] = "++";
3117	Token::tochars[TOKminusminus] = "--";
3118	Token::tochars[TOKpreplusplus] = "++";
3119	Token::tochars[TOKpreminusminus] = "--";
3120	Token::tochars[TOKtype] = "type";
3121	Token::tochars[TOKquestion] = "?";
3122	Token::tochars[TOKneg] = "-";
3123	Token::tochars[TOKuadd] = "+";
3124	Token::tochars[TOKvar] = "var";
3125	Token::tochars[TOKaddass] = "+=";
3126	Token::tochars[TOKminass] = "-=";
3127	Token::tochars[TOKmulass] = "*=";
3128	Token::tochars[TOKdivass] = "/=";
3129	Token::tochars[TOKmodass] = "%=";
3130	Token::tochars[TOKshlass] = "<<=";
3131	Token::tochars[TOKshrass] = ">>=";
3132	Token::tochars[TOKushrass] = ">>>=";
3133	Token::tochars[TOKandass] = "&=";
3134	Token::tochars[TOKorass] = "\|=";
3135	Token::tochars[TOKcatass] = "~=";
3136	Token::tochars[TOKcat] = "~";
3137	Token::tochars[TOKcall] = "call";
3138	Token::tochars[TOKidentity] = "is";
3139	Token::tochars[TOKnotidentity] = "!is";
3140
3141	Token::tochars[TOKorass] = "\|=";
3142	Token::tochars[TOKidentifier] = "identifier";
3143	#if DMDV2
3144	Token::tochars[TOKat] = "@";
3145	Token::tochars[TOKpow] = "^^";
3146	Token::tochars[TOKpowass] = "^^=";
3147	#endif
3148
3149	// For debugging
3150	Token::tochars[TOKerror] = "error";
3151	Token::tochars[TOKdotexp] = "dotexp";
3152	Token::tochars[TOKdotti] = "dotti";
3153	Token::tochars[TOKdotvar] = "dotvar";
3154	Token::tochars[TOKdottype] = "dottype";
3155	Token::tochars[TOKsymoff] = "symoff";
3156	Token::tochars[TOKarraylength] = "arraylength";
3157	Token::tochars[TOKarrayliteral] = "arrayliteral";
3158	Token::tochars[TOKassocarrayliteral] = "assocarrayliteral";
3159	Token::tochars[TOKstructliteral] = "structliteral";
3160	Token::tochars[TOKstring] = "string";
3161	Token::tochars[TOKdsymbol] = "symbol";
3162	Token::tochars[TOKtuple] = "tuple";
3163	Token::tochars[TOKdeclaration] = "declaration";
3164	Token::tochars[TOKdottd] = "dottd";
3165	Token::tochars[TOKon_scope_exit] = "scope(exit)";
3166	Token::tochars[TOKon_scope_success] = "scope(success)";
3167	Token::tochars[TOKon_scope_failure] = "scope(failure)";
3168
3169	#if UNITTEST
3170	unittest_lexer();
3171	#endif
3172	}
3173
3174	#if UNITTEST
3175
3176	void unittest_lexer()
3177	{
3178	//printf("unittest_lexer()\n");
3179
3180	/* Not much here, just trying things out.
3181	*/
3182	const unsigned char text[] = "int";
3183	Lexer lex1(NULL, (unsigned char *)text, 0, sizeof(text), 0, 0);
3184	TOK tok;
3185	tok = lex1.nextToken();
3186	//printf("tok == %s, %d, %d\n", Token::toChars(tok), tok, TOKint32);
3187	assert(tok == TOKint32);
3188	tok = lex1.nextToken();
3189	assert(tok == TOKeof);
3190	tok = lex1.nextToken();
3191	assert(tok == TOKeof);
3192	}
3193
3194	#endif

Note: See TracBrowser for help on using the browser.

dmd

root/trunk/src/lexer.c

Download in other formats: