1 module dfix;
2 
3 import std.experimental.lexer;
4 import dparse.lexer;
5 import dparse.parser;
6 import dparse.ast;
7 import std.stdio;
8 import std.format;
9 import std.file;
10 
11 int main(string[] args)
12 {
13 	import std.getopt : getopt;
14 	import std.parallelism : parallel;
15 
16 	// http://wiki.dlang.org/DIP64
17 	bool dip64;
18 	// http://wiki.dlang.org/DIP65
19 	bool dip65 = true;
20 	//https://github.com/dlang/DIPs/blob/master/DIPs/DIP1003.md
21 	bool dip1003 = true;
22 
23 	bool help;
24 
25 	try
26 	{
27 		getopt(args,
28 			"dip64", &dip64,
29 			"dip65", &dip65,
30 			"dip1003", &dip1003,
31 			"help|h", &help,
32 		);
33 	}
34 	catch (Exception e)
35 	{
36 		stderr.writeln(e.msg);
37 		return 1;
38 	}
39 
40 	if (help)
41 	{
42 		printHelp();
43 		return 0;
44 	}
45 
46 	if (args.length < 2)
47 	{
48 		stderr.writeln("File path is a required argument");
49 		return 1;
50 	}
51 
52 	string[] files;
53 
54 	foreach (arg; args[1 .. $])
55 	{
56 		if (isDir(arg))
57 		{
58 			foreach (f; dirEntries(arg, "*.{d,di}", SpanMode.depth))
59 				files ~= f;
60 		}
61 		else
62 			files ~= arg;
63 	}
64 
65 	foreach (f; parallel(files))
66 	{
67 		try
68 			upgradeFile(f, dip64, dip65, dip1003);
69 		catch (Exception e)
70 			stderr.writeln("Failed to upgrade ", f, ":(", e.file, ":", e.line, ") ", e.msg);
71 	}
72 
73 	return 0;
74 }
75 
76 /**
77  * Prints help message
78  */
79 void printHelp()
80 {
81 	stdout.writeln(`
82 Dfix automatically upgrades D source code to comply with new language changes.
83 Files are modified in place, so have backup copies ready or use a source
84 control system.
85 
86 Usage:
87 
88     dfix [Options] FILES DIRECTORIES
89 
90 Options:
91 
92     --dip64
93         Rewrites attributes to be compliant with DIP64. This defaults to
94         "false". Do not use this feature if you want your code to compile.
95 		It exists as a proof-of-concept for enabling DIP64.
96     --dip65
97         Rewrites catch blocks to be compliant with DIP65. This defaults to
98         "true". Use --dip65=false to disable this fix.
99     --dip1003
100         Rewrites body blocks to be compliant with DIP1003. This defaults to
101         "true". Use --dip1003=false to disable this fix.
102     --help -h
103         Prints this help message
104 `);
105 }
106 
107 /**
108  * Fixes the given file.
109  */
110 void upgradeFile(string fileName, bool dip64, bool dip65, bool dip1003)
111 {
112 	import std.algorithm : filter, canFind;
113 	import std.range : retro;
114 	import std.array : array, uninitializedArray;
115 	import dparse.formatter : Formatter;
116 	import std.exception : enforce;
117 	import dparse.rollback_allocator : RollbackAllocator;
118 	import std.functional : toDelegate;
119 
120 	File input = File(fileName, "rb");
121 	ubyte[] inputBytes = uninitializedArray!(ubyte[])(cast(size_t) input.size);
122 	input.rawRead(inputBytes);
123 	input.close();
124 	StringCache cache = StringCache(StringCache.defaultBucketCount);
125 	LexerConfig config;
126 	config.fileName = fileName;
127 	config.stringBehavior = StringBehavior.source;
128 	auto tokens = byToken(inputBytes, config, &cache).array;
129 	auto parseTokens = tokens.filter!(a => a != tok!"whitespace"
130 		&& a != tok!"comment" && a != tok!"specialTokenSequence").array;
131 
132 	RollbackAllocator allocator;
133 	uint errorCount;
134 	auto mod = parseModule(parseTokens, fileName, &allocator, toDelegate(&reportErrors), &errorCount);
135 	if (errorCount > 0)
136 	{
137 		stderr.writefln("%d parse errors encountered. Aborting upgrade of %s",
138 			errorCount, fileName);
139 		return;
140 	}
141 
142 	File output = File(fileName, "wb");
143 	auto visitor = new DFixVisitor;
144 	visitor.visit(mod);
145 	relocateMarkers(visitor.markers, tokens);
146 
147 	SpecialMarker[] markers = visitor.markers;
148 
149 	auto formatter = new Formatter!(File.LockingTextWriter)(File.LockingTextWriter.init);
150 
151 	void writeType(T)(File output, T tokens, ref size_t i)
152 	{
153 		if (isBasicType(tokens[i].type))
154 		{
155 			writeToken(output, tokens[i]);
156 			i++;
157 		}
158 		else if ((tokens[i] == tok!"const" || tokens[i] == tok!"immutable"
159 				|| tokens[i] == tok!"shared" || tokens[i] == tok!"inout")
160 				&& tokens[i + 1] == tok!"(")
161 		{
162 			writeToken(output, tokens[i]);
163 			i++;
164 			skipAndWrite!("(", ")")(output, tokens, i);
165 		}
166 		else
167 		{
168 			skipIdentifierChain(output, tokens, i, true);
169 			if (i < tokens.length && tokens[i] == tok!"!")
170 			{
171 				writeToken(output, tokens[i]);
172 				i++;
173 				if (i + 1 < tokens.length && tokens[i + 1] == tok!"(")
174 					skipAndWrite!("(", ")")(output, tokens, i);
175 				else if (tokens[i].type == tok!"identifier")
176 					skipIdentifierChain(output, tokens, i, true);
177 				else
178 				{
179 					writeToken(output, tokens[i]);
180 					i++;
181 				}
182 			}
183 		}
184 		skipWhitespace(output, tokens, i);
185 		// print out suffixes
186 		while (i < tokens.length && (tokens[i] == tok!"*" || tokens[i] == tok!"["))
187 		{
188 			if (tokens[i] == tok!"*")
189 			{
190 				writeToken(output, tokens[i]);
191 				i++;
192 			}
193 			else if (tokens[i] == tok!"[")
194 				skipAndWrite!("[", "]")(output, tokens, i);
195 		}
196 	}
197 
198 	for (size_t i = 0; i < tokens.length; i++)
199 	{
200 		markerLoop: foreach (marker; markers)
201 		{
202 			with (SpecialMarkerType) final switch (marker.type)
203 			{
204 			case bodyEnd:
205 				if (tokens[i].index != marker.index)
206 					break;
207 				assert (tokens[i].type == tok!"}", format("%d %s", tokens[i].line, str(tokens[i].type)));
208 				writeToken(output, tokens[i]);
209 				i++;
210 				if (i < tokens.length && tokens[i] == tok!";")
211 					i++;
212 				markers = markers[1 .. $];
213 				break markerLoop;
214 			case functionAttributePrefix:
215 				if (tokens[i].index != marker.index)
216 					break;
217 				// skip over token to be moved
218 				i++;
219 				skipWhitespace(output, tokens, i, false);
220 
221 				// skip over function return type
222 				writeType(output, tokens, i);
223 				skipWhitespace(output, tokens, i);
224 
225 				// skip over function name
226 				skipIdentifierChain(output, tokens, i, true);
227 				skipWhitespace(output, tokens, i, false);
228 
229 				// skip first paramters
230 				skipAndWrite!("(", ")")(output, tokens, i);
231 
232 				immutable bookmark = i;
233 				skipWhitespace(output, tokens, i, false);
234 
235 				// If there is a second set of parameters, go back to the bookmark
236 				// and print out the whitespace
237 				if (i < tokens.length && tokens[i] == tok!"(")
238 				{
239 					i = bookmark;
240 					skipWhitespace(output, tokens, i);
241 					skipAndWrite!("(", ")")(output, tokens, i);
242 					skipWhitespace(output, tokens, i, false);
243 				}
244 				else
245 					i = bookmark;
246 
247 				// write out the attribute being moved
248 				output.write(" ", marker.functionAttribute);
249 
250 				// if there was no whitespace, add it after the moved attribute
251 				if (i < tokens.length && tokens[i] != tok!"whitespace" && tokens[i] != tok!";")
252 					output.write(" ");
253 
254 				markers = markers[1 .. $];
255 				break markerLoop;
256 			case cStyleArray:
257 				if (i != marker.index)
258 					break;
259 				formatter.sink = output.lockingTextWriter();
260 				foreach (node; retro(marker.nodes))
261 					formatter.format(node);
262 				formatter.sink = File.LockingTextWriter.init;
263 				skipWhitespace(output, tokens, i);
264 				writeToken(output, tokens[i]);
265 				i++;
266 				suffixLoop: while (i < tokens.length) switch (tokens[i].type)
267 				{
268 					case tok!"(": skipAndWrite!("(", ")")(output, tokens, i); break;
269 					case tok!"[": skip!("[", "]")(tokens, i); break;
270 					case tok!"*": i++; break;
271 					default: break suffixLoop;
272 				}
273 				markers = markers[1 .. $];
274 				break markerLoop;
275 			}
276 		}
277 
278 		if (i >= tokens.length)
279 			break;
280 
281 		switch (tokens[i].type)
282 		{
283 		case tok!"asm":
284 			skipAsmBlock(output, tokens, i);
285 			goto default;
286 		case tok!"catch":
287 			if (!dip65)
288 				goto default;
289 			size_t j = i + 1;
290 			while (j < tokens.length && (tokens[j] == tok!"whitespace" || tokens[j] == tok!"comment"))
291 				j++;
292 			if (j < tokens.length && tokens[j].type != tok!"(")
293 			{
294 				output.write("catch (Throwable)");
295 				break;
296 			}
297 			else
298 				goto default;
299 		case tok!"deprecated":
300 			if (dip64)
301 				output.write("@");
302 			output.writeToken(tokens[i]);
303 			i++;
304 			if (i < tokens.length && tokens[i] == tok!"(")
305 				skipAndWrite!("(", ")")(output, tokens, i);
306 			if (i < tokens.length)
307 				goto default;
308 			else
309 				break;
310 		case tok!"stringLiteral":
311 			immutable size_t stringBookmark = i;
312 			while (tokens[i] == tok!"stringLiteral")
313 			{
314 				i++;
315 				skipWhitespace(output, tokens, i, false);
316 			}
317 			immutable bool parensNeeded = stringBookmark + 1 != i && tokens[i] == tok!".";
318 			i = stringBookmark;
319 			if (parensNeeded)
320 				output.write("(");
321 			output.writeToken(tokens[i]);
322 			i++;
323 			skipWhitespace(output, tokens, i);
324 			while (tokens[i] == tok!"stringLiteral")
325 			{
326 				output.write("~ ");
327 				output.writeToken(tokens[i]);
328 				i++;
329 				skipWhitespace(output, tokens, i);
330 			}
331 			if (parensNeeded)
332 				output.write(")");
333 			if (i < tokens.length)
334 				goto default;
335 			else
336 				break;
337 		case tok!"override":
338 		case tok!"final":
339 		case tok!"abstract":
340 		case tok!"align":
341 		case tok!"pure":
342 		case tok!"nothrow":
343 			if (!dip64)
344 				goto default;
345 			output.write("@");
346 			output.write(str(tokens[i].type));
347 			break;
348 		case tok!"alias":
349 			bool multipleAliases = false;
350 			bool oldStyle = true;
351 			output.writeToken(tokens[i]); // alias
352 				i++;
353 			size_t j = i + 1;
354 
355 			int depth;
356 			loop: while (j < tokens.length) switch (tokens[j].type)
357 			{
358 			case tok!"(":
359 				depth++;
360 				j++;
361 				break;
362 			case tok!")":
363 				depth--;
364 				if (depth < 0)
365 				{
366 					oldStyle = false;
367 					break loop;
368 				}
369 				j++;
370 				break;
371 			case tok!"=":
372 			case tok!"this":
373 				j++;
374 				oldStyle = false;
375 				break;
376 			case tok!",":
377 				j++;
378 				if (depth == 0)
379 					multipleAliases = true;
380 				break;
381 			case tok!";":
382 				break loop;
383 			default:
384 				j++;
385 				break;
386 			}
387 
388 			if (!oldStyle) foreach (k; i .. j + 1)
389 			{
390 				output.writeToken(tokens[k]);
391 				i = k;
392 			}
393 			else
394 			{
395 				skipWhitespace(output, tokens, i);
396 
397 				size_t beforeStart = i;
398 				size_t beforeEnd = beforeStart;
399 
400 				loop2: while (beforeEnd < tokens.length) switch (tokens[beforeEnd].type)
401 				{
402 				case tok!"bool":
403 				case tok!"byte":
404 				case tok!"ubyte":
405 				case tok!"short":
406 				case tok!"ushort":
407 				case tok!"int":
408 				case tok!"uint":
409 				case tok!"long":
410 				case tok!"ulong":
411 				case tok!"char":
412 				case tok!"wchar":
413 				case tok!"dchar":
414 				case tok!"float":
415 				case tok!"double":
416 				case tok!"real":
417 				case tok!"ifloat":
418 				case tok!"idouble":
419 				case tok!"ireal":
420 				case tok!"cfloat":
421 				case tok!"cdouble":
422 				case tok!"creal":
423 				case tok!"void":
424 					beforeEnd++;
425 					break loop2;
426 				case tok!".":
427 					beforeEnd++;
428 					goto case;
429 				case tok!"identifier":
430 					skipIdentifierChain(output, tokens, beforeEnd);
431 					break loop2;
432 				case tok!"typeof":
433 					beforeEnd++;
434 					skip!("(", ")")(tokens, beforeEnd);
435 					skipWhitespace(output, tokens, beforeEnd, false);
436 					if (tokens[beforeEnd] == tok!".")
437 						skipIdentifierChain(output, tokens, beforeEnd);
438 					break loop2;
439 				case tok!"@":
440 					beforeEnd++;
441 					if (tokens[beforeEnd] == tok!"identifier")
442 						beforeEnd++;
443 					if (tokens[beforeEnd] == tok!"(")
444 						skip!("(", ")")(tokens, beforeEnd);
445 					skipWhitespace(output, tokens, beforeEnd, false);
446 					break;
447 				case tok!"static":
448 				case tok!"const":
449 				case tok!"immutable":
450 				case tok!"inout":
451 				case tok!"shared":
452 				case tok!"extern":
453 				case tok!"nothrow":
454 				case tok!"pure":
455 				case tok!"__vector":
456 					beforeEnd++;
457 					skipWhitespace(output, tokens, beforeEnd, false);
458 					if (tokens[beforeEnd] == tok!"(")
459 						skip!("(", ")")(tokens, beforeEnd);
460 					if (beforeEnd >= tokens.length)
461 						break loop2;
462 					size_t k = beforeEnd;
463 					skipWhitespace(output, tokens, k, false);
464 					if (k + 1 < tokens.length && tokens[k + 1].type == tok!";")
465 						break loop2;
466 					else
467 						beforeEnd = k;
468 					break;
469 				default:
470 					break loop2;
471 				}
472 
473 				i = beforeEnd;
474 
475 				skipWhitespace(output, tokens, i, false);
476 
477 				if (tokens[i] == tok!"*" || tokens[i] == tok!"["
478 					|| tokens[i] == tok!"function" || tokens[i] == tok!"delegate")
479 				{
480 					beforeEnd = i;
481 				}
482 
483 				loop3: while (beforeEnd < tokens.length) switch (tokens[beforeEnd].type)
484 				{
485 				case tok!"*":
486 					beforeEnd++;
487 					size_t m = beforeEnd;
488 					skipWhitespace(output, tokens, m, false);
489 					if (m < tokens.length && (tokens[m] == tok!"*"
490 						|| tokens[m] == tok!"[" || tokens[m] == tok!"function"
491 						|| tokens[m] == tok!"delegate"))
492 					{
493 						beforeEnd = m;
494 					}
495 					break;
496 				case tok!"[":
497 					skip!("[", "]")(tokens, beforeEnd);
498 					size_t m = beforeEnd;
499 					skipWhitespace(output, tokens, m, false);
500 					if (m < tokens.length && (tokens[m] == tok!"*"
501 						|| tokens[m] == tok!"[" || tokens[m] == tok!"function"
502 						|| tokens[m] == tok!"delegate"))
503 					{
504 						beforeEnd = m;
505 					}
506 					break;
507 				case tok!"function":
508 				case tok!"delegate":
509 					beforeEnd++;
510 					skipWhitespace(output, tokens, beforeEnd, false);
511 					skip!("(", ")")(tokens, beforeEnd);
512 					size_t l = beforeEnd;
513 					skipWhitespace(output, tokens, l, false);
514 					loop4: while (l < tokens.length) switch (tokens[l].type)
515 					{
516 					case tok!"const":
517 					case tok!"nothrow":
518 					case tok!"pure":
519 					case tok!"immutable":
520 					case tok!"inout":
521 					case tok!"shared":
522 						beforeEnd = l + 1;
523 						l = beforeEnd;
524 						skipWhitespace(output, tokens, l, false);
525 						if (l < tokens.length && tokens[l].type == tok!"identifier")
526 						{
527 							beforeEnd = l - 1;
528 							break loop4;
529 						}
530 						break;
531 					case tok!"@":
532 						beforeEnd = l + 1;
533 						skipWhitespace(output, tokens, beforeEnd, false);
534 						if (tokens[beforeEnd] == tok!"(")
535 							skip!("(", ")")(tokens, beforeEnd);
536 						else
537 						{
538 							beforeEnd++; // identifier
539 							skipWhitespace(output, tokens, beforeEnd, false);
540 							if (tokens[beforeEnd] == tok!"(")
541 								skip!("(", ")")(tokens, beforeEnd);
542 						}
543 						l = beforeEnd;
544 						skipWhitespace(output, tokens, l, false);
545 						if (l < tokens.length && tokens[l].type == tok!"identifier")
546 						{
547 							beforeEnd = l - 1;
548 							break loop4;
549 						}
550 						break;
551 					default:
552 						break loop4;
553 					}
554 					break;
555 				default:
556 					break loop3;
557 				}
558 
559 				i = beforeEnd;
560 				skipWhitespace(output, tokens, i, false);
561 
562 				output.writeToken(tokens[i]);
563 				output.write(" = ");
564 				foreach (l; beforeStart .. beforeEnd)
565 					output.writeToken(tokens[l]);
566 
567 				if (multipleAliases)
568 				{
569 					i++;
570 					skipWhitespace(output, tokens, i, false);
571 					while (tokens[i] == tok!",")
572 					{
573 						i++; // ,
574 						output.write(", ");
575 						skipWhitespace(output, tokens, i, false);
576 						output.writeToken(tokens[i]);
577 						output.write(" = ");
578 						foreach (l; beforeStart .. beforeEnd)
579 							output.writeToken(tokens[l]);
580 					}
581 				}
582 			}
583 			break;
584 		case tok!"identifier":
585 			if (tokens[i].text == "body")
586 				(dip1003 && tokens.isBodyKw(i)) ? output.write("do") : output.write("body");
587 			else
588 				goto default;
589 			break;
590 		default:
591 			output.writeToken(tokens[i]);
592 			break;
593 		}
594 	}
595 }
596 
597 /**
598  * The types of special token ranges identified by the parsing pass
599  */
600 enum SpecialMarkerType
601 {
602 	/// Function declarations such as "const int foo();"
603 	functionAttributePrefix,
604 	/// Variable and parameter declarations such as "int bar[]"
605 	cStyleArray,
606 	/// The location of a closing brace for an interface, class, struct, union,
607 	/// or enum.
608 	bodyEnd
609 }
610 
611 /**
612  * Identifies ranges of tokens in the source tokens that need to be rewritten
613  */
614 struct SpecialMarker
615 {
616 	/// Range type
617 	SpecialMarkerType type;
618 
619 	/// Begin byte position (before relocateMarkers) or token index
620 	/// (after relocateMarkers)
621 	size_t index;
622 
623 	/// The type suffix AST nodes that should be moved
624 	const(TypeSuffix[]) nodes;
625 
626 	/// The function attribute such as const, immutable, or inout to move
627 	string functionAttribute;
628 }
629 
630 /**
631  * Scans a module's parsed AST and looks for C-style array variables and
632  * parameters, storing the locations in the markers array.
633  */
634 class DFixVisitor : ASTVisitor
635 {
636 	// C-style arrays variables
637 	override void visit(const VariableDeclaration varDec)
638 	{
639 		if (varDec.declarators.length == 0)
640 			return;
641 		markers ~= SpecialMarker(SpecialMarkerType.cStyleArray,
642 			varDec.declarators[0].name.index, varDec.declarators[0].cstyle);
643 	}
644 
645 	// C-style array parameters
646 	override void visit(const Parameter param)
647 	{
648 		if (param.cstyle.length > 0)
649 			markers ~= SpecialMarker(SpecialMarkerType.cStyleArray, param.name.index,
650 				param.cstyle);
651 		param.accept(this);
652 	}
653 
654 	// interface, union, class, struct body closing braces
655 	override void visit(const StructBody structBody)
656 	{
657 		structBody.accept(this);
658 		markers ~= SpecialMarker(SpecialMarkerType.bodyEnd, structBody.endLocation);
659 	}
660 
661 	// enum body closing braces
662 	override void visit(const EnumBody enumBody)
663 	{
664 		enumBody.accept(this);
665 		// skip over enums whose body is a single semicolon
666 		if (enumBody.endLocation == 0 && enumBody.startLocation == 0)
667 			return;
668 		markers ~= SpecialMarker(SpecialMarkerType.bodyEnd, enumBody.endLocation);
669 	}
670 
671 	// Confusing placement of function attributes
672 	override void visit(const Declaration dec)
673 	{
674 		if (dec.functionDeclaration is null)
675 			goto end;
676 		if (dec.attributes.length == 0)
677 			goto end;
678 		foreach (attr; dec.attributes)
679 		{
680 			if (attr.attribute == tok!"")
681 				continue;
682 			if (attr.attribute == tok!"const"
683 				|| attr.attribute == tok!"inout"
684 				|| attr.attribute == tok!"immutable")
685 			{
686 				markers ~= SpecialMarker(SpecialMarkerType.functionAttributePrefix,
687 					attr.attribute.index, null, str(attr.attribute.type));
688 			}
689 		}
690 	end:
691 		dec.accept(this);
692 	}
693 
694 	alias visit = ASTVisitor.visit;
695 
696 	/// Parts of the source file identified as needing a rewrite
697 	SpecialMarker[] markers;
698 }
699 
700 /**
701  * Converts the marker index from a byte index into the source code to an index
702  * into the tokens array.
703  */
704 void relocateMarkers(SpecialMarker[] markers, const(Token)[] tokens) pure nothrow @nogc
705 {
706 	foreach (ref marker; markers)
707 	{
708 		if (marker.type != SpecialMarkerType.cStyleArray)
709 			continue;
710 		size_t index = 0;
711 		while (tokens[index].index != marker.index)
712 			index++;
713 		marker.index = index - 1;
714 	}
715 }
716 
717 /**
718  * Writes a token to the output file.
719  */
720 void writeToken(File output, ref const(Token) token)
721 {
722 	output.write(token.text is null ? str(token.type) : token.text);
723 }
724 
725 void skipAndWrite(alias Open, alias Close)(File output, const(Token)[] tokens, ref size_t index)
726 {
727 	int depth = 1;
728 	writeToken(output, tokens[index]);
729 	index++;
730 	while (index < tokens.length && depth > 0) switch (tokens[index].type)
731 	{
732 	case tok!Open:
733 		depth++;
734 		writeToken(output, tokens[index]);
735 		index++;
736 		break;
737 	case tok!Close:
738 		depth--;
739 		writeToken(output, tokens[index]);
740 		index++;
741 		break;
742 	default:
743 		writeToken(output, tokens[index]);
744 		index++;
745 		break;
746 	}
747 }
748 
749 /**
750  * Returns true if `body` is a keyword and false if it's an identifier.
751  */
752 bool isBodyKw(const(Token)[] tokens, size_t index)
753 {
754 	assert(index);
755 	index -= 1;
756 	L0: while (index--) switch (tokens[index].type)
757 	{
758 		// `in {} body {}`
759 		case tok!"}":
760 			return true;
761 		case tok!"comment":
762 			continue;
763 		// `void foo () return {}` or `return body;`
764 		case tok!"return":
765 			continue;
766 		// `void foo () @safe pure body {}`
767 		case tok!")":
768 		case tok!"const":
769 		case tok!"immutable":
770 		case tok!"inout":
771 		case tok!"shared":
772 		case tok!"@":
773 		case tok!"pure":
774 		case tok!"nothrow":
775 		case tok!"scope":
776 			return true;
777 		default:
778 			break L0;
779 	}
780 	return false;
781 }
782 
783 /**
784  * Skips balanced parens, braces, or brackets. index will be incremented to
785  * index tokens just after the balanced closing token.
786  */
787 void skip(alias Open, alias Close)(const(Token)[] tokens, ref size_t index)
788 {
789 	int depth = 1;
790 	index++;
791 	while (index < tokens.length && depth > 0) switch (tokens[index].type)
792 	{
793 	case tok!Open: depth++;  index++; break;
794 	case tok!Close: depth--; index++; break;
795 	default:                 index++; break;
796 	}
797 }
798 
799 /**
800  * Skips whitespace tokens, incrementing index until it indexes tokens at a
801  * non-whitespace token.
802  */
803 void skipWhitespace(File output, const(Token)[] tokens, ref size_t index, bool print = true)
804 {
805 	while (index < tokens.length && (tokens[index] == tok!"whitespace" || tokens[index] == tok!"comment"))
806 	{
807 		if (print) output.writeToken(tokens[index]);
808 		index++;
809 	}
810 }
811 
812 /**
813  * Advances index until it indexs the token just after an identifier or template
814  * chain.
815  */
816 void skipIdentifierChain(File output, const(Token)[] tokens, ref size_t index, bool print = false)
817 {
818 	loop: while (index < tokens.length) switch (tokens[index].type)
819 	{
820 	case tok!".":
821 		if (print)
822 			writeToken(output, tokens[index]);
823 		index++;
824 		skipWhitespace(output, tokens, index, false);
825 		break;
826 	case tok!"identifier":
827 		if (print)
828 			writeToken(output, tokens[index]);
829 		index++;
830 		size_t i = index;
831 		skipWhitespace(output, tokens, i, false);
832 		if (tokens[i] == tok!"!")
833 		{
834 			i++;
835 			if (print)
836 				writeToken(output, tokens[index]);
837 			index++;
838 			skipWhitespace(output, tokens, i, false);
839 			if (tokens[i] == tok!"(")
840 			{
841 				if (print)
842 					skipAndWrite!("(", ")")(output, tokens, i);
843 				else
844 					skip!("(", ")")(tokens, i);
845 				index = i;
846 			}
847 			else
848 			{
849 				i++;
850 				if (print)
851 					writeToken(output, tokens[index]);
852 				index++;
853 			}
854 		}
855 		if (tokens[i] != tok!".")
856 			break loop;
857 		break;
858 	case tok!"whitespace":
859 		index++;
860 		break;
861 	default:
862 		break loop;
863 	}
864 }
865 
866 /**
867  * Skips over an attribute
868  */
869 void skipAttribute(File output, const(Token)[] tokens, ref size_t i)
870 {
871 	switch (tokens[i].type)
872 	{
873 	case tok!"@":
874 		output.writeToken(tokens[i]);
875 		i++; // @
876 		skipWhitespace(output, tokens, i, true);
877 		switch (tokens[i].type)
878 		{
879 		case tok!"identifier":
880 			output.writeToken(tokens[i]);
881 			i++; // identifier
882 			skipWhitespace(output, tokens, i, true);
883 			if (tokens[i].type == tok!"(")
884 				goto case tok!"(";
885 			break;
886 		case tok!"(":
887 			int depth = 1;
888 			output.writeToken(tokens[i]);
889 			i++;
890 			while (i < tokens.length && depth > 0) switch (tokens[i].type)
891 			{
892 			case tok!"(": depth++; output.writeToken(tokens[i]); i++; break;
893 			case tok!")": depth--; output.writeToken(tokens[i]); i++; break;
894 			default:               output.writeToken(tokens[i]); i++; break;
895 			}
896 			break;
897 		default:
898 			break;
899 		}
900 		break;
901 	case tok!"nothrow":
902 	case tok!"pure":
903 		output.writeToken(tokens[i]);
904 		i++;
905 		break;
906 	default:
907 		break;
908 	}
909 }
910 
911 /**
912  * Skips over (and prints) an asm block
913  */
914 void skipAsmBlock(File output, const(Token)[] tokens, ref size_t i)
915 {
916 	import std.exception : enforce;
917 
918 	output.write("asm");
919 	i++; // asm
920 	skipWhitespace(output, tokens, i);
921 	loop: while (true) switch (tokens[i].type)
922 	{
923 	case tok!"@":
924 	case tok!"nothrow":
925 	case tok!"pure":
926 		skipAttribute(output, tokens, i);
927 		skipWhitespace(output, tokens, i);
928 		break;
929 	case tok!"{":
930 		break loop;
931 	default:
932 		break loop;
933 	}
934 	enforce(tokens[i].type == tok!"{");
935 	output.write("{");
936 	i++; // {
937 	int depth = 1;
938 	while (depth > 0 && i < tokens.length) switch (tokens[i].type)
939 	{
940 	case tok!"{": depth++; goto default;
941 	case tok!"}": depth--; goto default;
942 	default: writeToken(output, tokens[i]); i++; break;
943 	}
944 }
945 
946 /**
947  * Dummy message output function for the lexer/parser
948  */
949 void reportErrors(string fileName, size_t lineNumber, size_t columnNumber,
950 	string message, bool isError)
951 {
952 	import std.stdio : stderr;
953 
954 	if (!isError)
955 		return;
956 	stderr.writefln("%s(%d:%d)[error]: %s", fileName, lineNumber, columnNumber, message);
957 }