lexer.h

Revision 769, 9.9 kB (checked in by walter, 1 year ago)
belatedly merge in changes from changeset 717
Property svn:eol-style set to `native`

Line
1	// Compiler implementation of the D programming language
2	// Copyright (c) 1999-2010 by Digital Mars
3	// All Rights Reserved
4	// written by Walter Bright
5	// http://www.digitalmars.com
6	// License for redistribution is by either the Artistic License
7	// in artistic.txt, or the GNU General Public License in gnu.txt.
8	// See the included readme.txt for details.
9
10	#ifndef DMD_LEXER_H
11	#define DMD_LEXER_H
12
13	#ifdef __DMC__
14	#pragma once
15	#endif /* __DMC__ */
16
17	#include "root.h"
18	#include "mars.h"
19
20	struct StringTable;
21	struct Identifier;
22	struct Module;
23
24	/* Tokens:
25	( )
26	[ ]
27	{ }
28	< > <= >= == != === !==
29	<< >> <<= >>= >>> >>>=
30	+ - += -=
31	* / % *= /= %=
32	& \| ^ &= \|= ^=
33	= ! ~ @
34	^^ ^^=
35	++ --
36	. -> : ,
37	? && \|\|
38	*/
39
40	enum TOK
41	{
42	TOKreserved,
43
44	// Other
45	TOKlparen, TOKrparen,
46	TOKlbracket, TOKrbracket,
47	TOKlcurly, TOKrcurly,
48	TOKcolon, TOKneg,
49	TOKsemicolon, TOKdotdotdot,
50	TOKeof, TOKcast,
51	TOKnull, TOKassert,
52	TOKtrue, TOKfalse,
53	TOKarray, TOKcall,
54	TOKaddress,
55	TOKtype, TOKthrow,
56	TOKnew, TOKdelete,
57	TOKstar, TOKsymoff,
58	TOKvar, TOKdotvar,
59	TOKdotti, TOKdotexp,
60	TOKdottype, TOKslice,
61	TOKarraylength, TOKversion,
62	TOKmodule, TOKdollar,
63	TOKtemplate, TOKdottd,
64	TOKdeclaration, TOKtypeof,
65	TOKpragma, TOKdsymbol,
66	TOKtypeid, TOKuadd,
67	TOKremove,
68	TOKnewanonclass, TOKcomment,
69	TOKarrayliteral, TOKassocarrayliteral,
70	TOKstructliteral,
71
72	// Operators
73	TOKlt, TOKgt,
74	TOKle, TOKge,
75	TOKequal, TOKnotequal,
76	TOKidentity, TOKnotidentity,
77	TOKindex, TOKis,
78	TOKtobool,
79
80	// 60
81	// NCEG floating point compares
82	// !<>= <> <>= !> !>= !< !<= !<>
83	TOKunord,TOKlg,TOKleg,TOKule,TOKul,TOKuge,TOKug,TOKue,
84
85	TOKshl, TOKshr,
86	TOKshlass, TOKshrass,
87	TOKushr, TOKushrass,
88	TOKcat, TOKcatass, // ~ ~=
89	TOKadd, TOKmin, TOKaddass, TOKminass,
90	TOKmul, TOKdiv, TOKmod,
91	TOKmulass, TOKdivass, TOKmodass,
92	TOKand, TOKor, TOKxor,
93	TOKandass, TOKorass, TOKxorass,
94	TOKassign, TOKnot, TOKtilde,
95	TOKplusplus, TOKminusminus, TOKconstruct, TOKblit,
96	TOKdot, TOKarrow, TOKcomma,
97	TOKquestion, TOKandand, TOKoror,
98	TOKpreplusplus, TOKpreminusminus,
99
100	// 106
101	// Numeric literals
102	TOKint32v, TOKuns32v,
103	TOKint64v, TOKuns64v,
104	TOKfloat32v, TOKfloat64v, TOKfloat80v,
105	TOKimaginary32v, TOKimaginary64v, TOKimaginary80v,
106
107	// Char constants
108	TOKcharv, TOKwcharv, TOKdcharv,
109
110	// Leaf operators
111	TOKidentifier, TOKstring,
112	TOKthis, TOKsuper,
113	TOKhalt, TOKtuple,
114	TOKerror,
115
116	// Basic types
117	TOKvoid,
118	TOKint8, TOKuns8,
119	TOKint16, TOKuns16,
120	TOKint32, TOKuns32,
121	TOKint64, TOKuns64,
122	TOKfloat32, TOKfloat64, TOKfloat80,
123	TOKimaginary32, TOKimaginary64, TOKimaginary80,
124	TOKcomplex32, TOKcomplex64, TOKcomplex80,
125	TOKchar, TOKwchar, TOKdchar, TOKbit, TOKbool,
126	TOKcent, TOKucent,
127
128	// 152
129	// Aggregates
130	TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport,
131	TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction,
132	TOKmixin,
133
134	TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport,
135	TOKstatic, /TOKvirtual,/ TOKfinal, TOKconst, TOKabstract, TOKvolatile,
136	TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout, TOKlazy,
137	TOKauto, TOKpackage, TOKmanifest, TOKimmutable,
138
139	// Statements
140	TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch,
141	TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith,
142	TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally,
143	TOKasm, TOKforeach, TOKforeach_reverse,
144	TOKscope,
145	TOKon_scope_exit, TOKon_scope_failure, TOKon_scope_success,
146
147	// Contracts
148	TOKbody, TOKinvariant,
149
150	// Testing
151	TOKunittest,
152
153	// Added after 1.0
154	TOKargTypes,
155	TOKref,
156	TOKmacro,
157	#if DMDV2
158	TOKtraits,
159	TOKoverloadset,
160	TOKpure,
161	TOKnothrow,
162	TOKtls,
163	TOKgshared,
164	TOKline,
165	TOKfile,
166	TOKshared,
167	TOKat,
168	TOKpow,
169	TOKpowass,
170	#endif
171
172	TOKMAX
173	};
174
175	#define TOKwild TOKinout
176
177	#define BASIC_TYPES \
178	TOKwchar: case TOKdchar: \
179	case TOKbit: case TOKbool: case TOKchar: \
180	case TOKint8: case TOKuns8: \
181	case TOKint16: case TOKuns16: \
182	case TOKint32: case TOKuns32: \
183	case TOKint64: case TOKuns64: \
184	case TOKfloat32: case TOKfloat64: case TOKfloat80: \
185	case TOKimaginary32: case TOKimaginary64: case TOKimaginary80: \
186	case TOKcomplex32: case TOKcomplex64: case TOKcomplex80: \
187	case TOKvoid
188
189	#define BASIC_TYPES_X(t) \
190	TOKvoid: t = Type::tvoid; goto LabelX; \
191	case TOKint8: t = Type::tint8; goto LabelX; \
192	case TOKuns8: t = Type::tuns8; goto LabelX; \
193	case TOKint16: t = Type::tint16; goto LabelX; \
194	case TOKuns16: t = Type::tuns16; goto LabelX; \
195	case TOKint32: t = Type::tint32; goto LabelX; \
196	case TOKuns32: t = Type::tuns32; goto LabelX; \
197	case TOKint64: t = Type::tint64; goto LabelX; \
198	case TOKuns64: t = Type::tuns64; goto LabelX; \
199	case TOKfloat32: t = Type::tfloat32; goto LabelX; \
200	case TOKfloat64: t = Type::tfloat64; goto LabelX; \
201	case TOKfloat80: t = Type::tfloat80; goto LabelX; \
202	case TOKimaginary32: t = Type::timaginary32; goto LabelX; \
203	case TOKimaginary64: t = Type::timaginary64; goto LabelX; \
204	case TOKimaginary80: t = Type::timaginary80; goto LabelX; \
205	case TOKcomplex32: t = Type::tcomplex32; goto LabelX; \
206	case TOKcomplex64: t = Type::tcomplex64; goto LabelX; \
207	case TOKcomplex80: t = Type::tcomplex80; goto LabelX; \
208	case TOKbit: t = Type::tbit; goto LabelX; \
209	case TOKbool: t = Type::tbool; goto LabelX; \
210	case TOKchar: t = Type::tchar; goto LabelX; \
211	case TOKwchar: t = Type::twchar; goto LabelX; \
212	case TOKdchar: t = Type::tdchar; goto LabelX; \
213	LabelX
214
215	struct Token
216	{
217	Token *next;
218	unsigned char *ptr; // pointer to first character of this token within buffer
219	enum TOK value;
220	unsigned char *blockComment; // doc comment string prior to this token
221	unsigned char *lineComment; // doc comment for previous token
222	union
223	{
224	// Integers
225	d_int32 int32value;
226	d_uns32 uns32value;
227	d_int64 int64value;
228	d_uns64 uns64value;
229
230	// Floats
231	#ifdef IN_GCC
232	// real_t float80value; // can't use this in a union!
233	#else
234	d_float80 float80value;
235	#endif
236
237	struct
238	{ unsigned char *ustring; // UTF8 string
239	unsigned len;
240	unsigned char postfix; // 'c', 'w', 'd'
241	};
242
243	Identifier *ident;
244	};
245	#ifdef IN_GCC
246	real_t float80value; // can't use this in a union!
247	#endif
248
249	static const char *tochars[TOKMAX];
250	static void *operator new(size_t sz);
251
252	int isKeyword();
253	void print();
254	const char *toChars();
255	static const char *toChars(enum TOK);
256	};
257
258	struct Lexer
259	{
260	static StringTable stringtable;
261	static OutBuffer stringbuffer;
262	static Token *freelist;
263
264	Loc loc; // for error messages
265
266	unsigned char *base; // pointer to start of buffer
267	unsigned char *end; // past end of buffer
268	unsigned char *p; // current character
269	Token token;
270	Module *mod;
271	int doDocComment; // collect doc comment information
272	int anyToken; // !=0 means seen at least one token
273	int commentToken; // !=0 means comments are TOKcomment's
274
275	Lexer(Module *mod,
276	unsigned char *base, unsigned begoffset, unsigned endoffset,
277	int doDocComment, int commentToken);
278
279	static void initKeywords();
280	static Identifier idPool(const char s);
281	static Identifier uniqueId(const char s);
282	static Identifier uniqueId(const char s, int num);
283
284	TOK nextToken();
285	TOK peekNext();
286	TOK peekNext2();
287	void scan(Token *t);
288	Token peek(Token t);
289	Token peekPastParen(Token t);
290	unsigned escapeSequence();
291	TOK wysiwygStringConstant(Token *t, int tc);
292	TOK hexStringConstant(Token *t);
293	#if DMDV2
294	TOK delimitedStringConstant(Token *t);
295	TOK tokenStringConstant(Token *t);
296	#endif
297	TOK escapeStringConstant(Token *t, int wide);
298	TOK charConstant(Token *t, int wide);
299	void stringPostfix(Token *t);
300	unsigned wchar(unsigned u);
301	TOK number(Token *t);
302	TOK inreal(Token *t);
303	void error(const char *format, ...);
304	void error(Loc loc, const char *format, ...);
305	void pragma();
306	unsigned decodeUTF();
307	void getDocComment(Token *t, unsigned lineComment);
308
309	static int isValidIdentifier(char *p);
310	static unsigned char combineComments(unsigned char c1, unsigned char *c2);
311	};
312
313	#endif /* DMD_LEXER_H */

Note: See TracBrowser for help on using the browser.

dmd

root/trunk/src/lexer.h

Download in other formats: