Fixed lexing/parsing bugs

main.d

@@ -40,6 +40,7 @@ int main(string[] args)
 	bool imports;
 	bool muffin;
 	bool outline;
+	bool tokenDump;
 
 	try
 	{
@@ -47,7 +48,7 @@ int main(string[] args)
 			"ctags|c", &ctags, "recursive|r|R", &recursive, "help|h", &help,
 			"tokenCount|t", &tokenCount, "syntaxCheck|s", &syntaxCheck,
 			"ast|xml", &ast, "imports|i", &imports, "outline|o", &outline,
-			"muffinButton", &muffin);
+			"tokenDump", &tokenDump, "muffinButton", &muffin);
 	}
 	catch (Exception e)
 	{
@@ -76,7 +77,7 @@ int main(string[] args)
 	}
 
 	auto optionCount = count!"a"([sloc, highlight, ctags, tokenCount,
-		syntaxCheck, ast, imports, outline]);
+		syntaxCheck, ast, imports, outline, tokenDump]);
 	if (optionCount > 1)
 	{
 		stderr.writeln("Too many options specified");
@@ -96,6 +97,17 @@ int main(string[] args)
 		highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]);
 		return 0;
 	}
+	else if (tokenDump)
+	{
+		bool usingStdin = args.length == 1;
+		ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
+		auto tokens = byToken!(ubyte[], false, false)(bytes);
+		foreach (ref token; tokens)
+		{
+			writeln("«", token.text is null ? str(token.type) : token.text,
+				" ", token.index, " ", token.line, " ", token.column, "»");
+		}
+	}
 	else if (ctags)
 	{
 		stdout.printCtags(expandArgs(args, recursive));

stdx/d/lexer.d

@@ -482,10 +482,10 @@ public struct DLexer(R)
 	Token lexNumber() pure nothrow
 	{
 		auto mark = range.mark();
-		auto lookahead = range.lookahead(1);
+		auto lookahead = range.lookahead(2);
-		if (range.front == '0' && lookahead.length == 1)
+		if (range.front == '0' && lookahead.length == 2)
 		{
-			switch (lookahead[0])
+			switch (lookahead[1])
 			{
 			case 'x':
 			case 'X':
@@ -1040,11 +1040,19 @@ public struct DLexer(R)
 			else
 				app.put(t.text);
 			if (t.type == tok!"}")
+			{
 				depth--;
+				if (depth > 0)
+				popFront();
+			}
 			else if (t.type == tok!"{")
+			{
 				depth++;
 				popFront();
 			}
+			else
+				popFront();
+		}
 		IdType type = tok!"stringLiteral";
 		lexStringSuffix(type);
 		return Token(type, app.data, range.line, range.column, range.index);

stdx/d/parser.d

@@ -12,7 +12,7 @@ import std.string : format;
 
 // Uncomment this if you want ALL THE OUTPUT
 // Caution: generates 180 megabytes of logging for std.datetime
-version = std_parser_verbose;
+//version = std_parser_verbose;
 
 /**
  * Params:
@@ -1248,7 +1248,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         case tok!"!<=":
             node.relExpression = parseRelExpression(shift);
             break;
-        case tok!"=":
+        case tok!"==":
         case tok!"!=":
             node.equalExpression = parseEqualExpression(shift);
             break;
@@ -6547,6 +6547,7 @@ protected:
         case tok!"stringLiteral":
         case tok!"wstringLiteral":
         case tok!"dstringLiteral":
+        case tok!"characterLiteral":
     };
     enum string SPECIAL_CASES = q{
         case tok!"__DATE__":