Merge branch 'NewLexer'

Conflicts:
	stdx/d/parser.d

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "datapicked"]
+	path = datapicked
+	url = ./datapicked/

astprinter.d

@@ -21,7 +21,7 @@ class XMLPrinter : ASTVisitor
 {
 	override void visit(AddExpression addExpression)
 	{
-		output.writeln("<addExpression operator=\"", getTokenValue(addExpression.operator) ,"\">");
+		output.writeln("<addExpression operator=\"", str(addExpression.operator) ,"\">");
 		output.writeln("<left>");
 		addExpression.left.accept(this);
 		output.writeln("</left>");
@@ -51,7 +51,7 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(AlignAttribute alignAttribute)
 	{
-		output.writeln("<alignAttribute align=\"", alignAttribute.intLiteral.value, "\">");
+		output.writeln("<alignAttribute align=\"", alignAttribute.intLiteral.text, "\">");
 	}
 
 	override void visit(AndAndExpression andAndExpression)
@@ -130,7 +130,7 @@ class XMLPrinter : ASTVisitor
 			output.writeln("<assignExpression>");
 		else
 			output.writeln("<assignExpression operator=\"",
-				getTokenValue(assignExpression.operator), "\">");
+				str(assignExpression.operator), "\">");
 		assignExpression.accept(this);
 		output.writeln("</assignExpression>");
 	}
@@ -143,20 +143,20 @@ class XMLPrinter : ASTVisitor
 	override void visit(AtAttribute atAttribute)
 	{
 		output.writeln("<atAttribute>");
-		if (atAttribute.identifier.type == TokenType.invalid)
+		if (atAttribute.identifier.type == tok!"")
 			atAttribute.accept(this);
 		else
-			output.writeln("<identifier>", atAttribute.identifier.value, "</identifier>");
+			output.writeln("<identifier>", atAttribute.identifier.text, "</identifier>");
 		output.writeln("</atAttribute>");
 	}
 
 	override void visit(Attribute attribute)
 	{
 		output.writeln("<attribute>");
-		if (attribute.attribute == TokenType.invalid)
+		if (attribute.attribute == tok!"")
 			attribute.accept(this);
 		else
-			output.writeln(getTokenValue(attribute.attribute));
+			output.writeln(str(attribute.attribute));
 		output.writeln("</attribute>");
 	}
 
@@ -173,7 +173,7 @@ class XMLPrinter : ASTVisitor
 		{
 			output.writeln("<item>");
 			output.writeln("<name line=\"", autoDec.identifiers[i].line, "\">",
-				autoDec.identifiers[i].value, "</name>");
+				autoDec.identifiers[i].text, "</name>");
 			visit(autoDec.initializers[i]);
 			output.writeln("</item>");
 		}
@@ -196,10 +196,10 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(BreakStatement breakStatement)
 	{
-		if (breakStatement.label.type == TokenType.invalid)
+		if (breakStatement.label.type == tok!"")
 			output.writeln("<breakStatement/>");
 		else
-			output.writeln("<breakStatement label=\"", breakStatement.label.value, "\"/>");
+			output.writeln("<breakStatement label=\"", breakStatement.label.text, "\"/>");
 	}
 
 	override void visit(BaseClass baseClass)
@@ -256,7 +256,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(ClassDeclaration classDec)
 	{
 		output.writeln("<classDeclaration line=\"", classDec.name.line, "\">");
-		output.writeln("<name>", classDec.name.value, "</name>");
+		output.writeln("<name>", classDec.name.text, "</name>");
 		classDec.accept(this);
 		output.writeln("</classDeclaration>");
 	}
@@ -318,29 +318,29 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(ContinueStatement continueStatement)
 	{
-		if (continueStatement.label.type == TokenType.invalid)
+		if (continueStatement.label.type == tok!"")
 			output.writeln("<continueStatement/>");
 		else
 			output.writeln("<continueStatement label=\"",
-				continueStatement.label.value, "\"/>");
+				continueStatement.label.text, "\"/>");
 	}
 
 	override void visit(DebugCondition debugCondition)
 	{
-		if (debugCondition.identifierOrInteger.type == TokenType.invalid)
+		if (debugCondition.identifierOrInteger.type == tok!"")
 			output.writeln("<debugCondition/>");
 		else
 			output.writeln("<debugCondition condition=\"",
-				debugCondition.identifierOrInteger.value, "\"/>");
+				debugCondition.identifierOrInteger.text, "\"/>");
 	}
 
 	override void visit(DebugSpecification debugSpecification)
 	{
-		if (debugSpecification.identifierOrInteger.type == TokenType.invalid)
+		if (debugSpecification.identifierOrInteger.type == tok!"")
 			output.writeln("<debugSpecification/>");
 		else
 			output.writeln("<debugSpecification condition=\"",
-				debugSpecification.identifierOrInteger.value, "\"/>");
+				debugSpecification.identifierOrInteger.text, "\"/>");
 	}
 
 	override void visit(Declaration declaration)
@@ -361,7 +361,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(Declarator declarator)
 	{
 		output.writeln("<declarator line=\"", declarator.name.line, "\">");
-		output.writeln("<name>", declarator.name.value, "</name>");
+		output.writeln("<name>", declarator.name.text, "</name>");
 		declarator.accept(this);
 		output.writeln("</declarator>");
 	}
@@ -411,8 +411,9 @@ class XMLPrinter : ASTVisitor
 	override void visit(EnumDeclaration enumDec)
 	{
 		output.writeln("<enumDeclaration line=\"", enumDec.name.line, "\">");
-		if (enumDec.name.type == TokenType.identifier)
+        writeDdoc(enumDec.comment);
-			output.writeln("<name>", enumDec.name.value, "</name>");
+		if (enumDec.name.type == tok!"identifier")
+			output.writeln("<name>", enumDec.name.text, "</name>");
 		enumDec.accept(this);
 		output.writeln("</enumDeclaration>");
 	}
@@ -420,13 +421,14 @@ class XMLPrinter : ASTVisitor
 	override void visit(EnumMember enumMem)
 	{
 		output.writeln("<enumMember line=\"", enumMem.name.line, "\">");
+        writeDdoc(enumMem.comment);
 		enumMem.accept(this);
 		output.writeln("</enumMember>");
 	}
 
 	override void visit(EqualExpression equalExpression)
 	{
-		output.writeln("<enumMember operator=\"", getTokenValue(equalExpression.operator), "\">");
+		output.writeln("<enumMember operator=\"", str(equalExpression.operator), "\">");
 		output.writeln("<left>");
 		visit(equalExpression.left);
 		output.writeln("</left>");
@@ -469,9 +471,9 @@ class XMLPrinter : ASTVisitor
 		output.writeln("<forStatement>");
 		if (forStatement.declarationOrStatement !is null)
 		{
-			output.writeln("<initialize>");
+			output.writeln("<initialization>");
-			visit(forStatement.declarationOrStatement);
+			visit(forStatement.initialization);
-			output.writeln("</initialize>");
+			output.writeln("</initialization>");
 		}
 		if (forStatement.test !is null)
 		{
@@ -491,7 +493,7 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(ForeachStatement foreachStatement)
 	{
-		output.writeln("<foreachStatement type=\"", getTokenValue(
+		output.writeln("<foreachStatement type=\"", str(
 			foreachStatement.type), "\">");
 		if (foreachStatement.foreachType !is null)
 			visit(foreachStatement.foreachType);
@@ -515,7 +517,7 @@ class XMLPrinter : ASTVisitor
 		output.writeln("<foreachType>");
 		foreach (constructor; foreachType.typeConstructors)
 		{
-			output.writeln("<typeConstructor>", getTokenValue(constructor), "</typeConstructor>");
+			output.writeln("<typeConstructor>", str(constructor), "</typeConstructor>");
 		}
 		if (foreachType.type !is null)
 			visit(foreachType.type);
@@ -552,7 +554,8 @@ class XMLPrinter : ASTVisitor
 	override void visit(FunctionDeclaration functionDec)
 	{
 		output.writeln("<functionDeclaration line=\"", functionDec.name.line, "\">");
-		output.writeln("<name>", functionDec.name.value, "</name>");
+		output.writeln("<name>", functionDec.name.text, "</name>");
+        writeDdoc(functionDec.comment);
 		if (functionDec.hasAuto)
 			output.writeln("<auto/>");
 		if (functionDec.hasRef)
@@ -564,17 +567,17 @@ class XMLPrinter : ASTVisitor
 	override void visit(FunctionLiteralExpression functionLiteralExpression)
 	{
 		output.writeln("<functionLiteralExpression type=\"",
-			getTokenValue(functionLiteralExpression.functionOrDelegate), "\">");
+			str(functionLiteralExpression.functionOrDelegate), "\">");
 		functionLiteralExpression.accept(this);
 		output.writeln("</functionLiteralExpression>");
 	}
 
 	override void visit(GotoStatement gotoStatement)
 	{
-		if (gotoStatement.label.type == TokenType.default_)
+		if (gotoStatement.label.type == tok!"default")
 			output.writeln("<gotoStatement default=\"true\"/>");
-		else if (gotoStatement.label.type == TokenType.identifier)
+		else if (gotoStatement.label.type == tok!"identifier")
-			output.writeln("<gotoStatement label=\"", gotoStatement.label.value, "\"/>");
+			output.writeln("<gotoStatement label=\"", gotoStatement.label.text, "\"/>");
 		else
 		{
 			output.writeln("<gotoStatement>");
@@ -625,7 +628,7 @@ class XMLPrinter : ASTVisitor
 		output.writeln("<ifStatement>");
 
 		output.writeln("<condition>");
-		if (ifStatement.identifier.type != TokenType.invalid)
+		if (ifStatement.identifier.type != tok!"")
 		{
 			if (ifStatement.type is null)
 				output.writeln("<auto/>");
@@ -651,11 +654,11 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(ImportBind importBind)
 	{
-		if (importBind.right.type == TokenType.invalid)
+		if (importBind.right.type == tok!"")
-			output.writeln("<importBind symbol=\"", importBind.left.value, "\">");
+			output.writeln("<importBind symbol=\"", importBind.left.text, "\">");
 		else
-			output.writeln("<importBind symbol=\"", importBind.right.value,
+			output.writeln("<importBind symbol=\"", importBind.right.text,
-				"\" rename=\"", importBind.left.value, "\">");
+				"\" rename=\"", importBind.left.text, "\">");
 	}
 
 	override void visit(ImportBindings importBindings)
@@ -725,7 +728,8 @@ class XMLPrinter : ASTVisitor
 	override void visit(InterfaceDeclaration interfaceDec)
 	{
 		output.writeln("<interfaceDeclaration line=\"", interfaceDec.name.line, "\">");
-		output.writeln("<name>", interfaceDec.name.value, "</name>");
+		output.writeln("<name>", interfaceDec.name.text, "</name>");
+        writeDdoc(interfaceDec.comment);
 		interfaceDec.accept(this);
 		output.writeln("</interfaceDeclaration>");
 	}
@@ -733,6 +737,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(Invariant invariant_)
 	{
 		output.writeln("<invariant>");
+        writeDdoc(invariant_.comment);
 		invariant_.accept(this);
 		output.writeln("</invariant>");
 	}
@@ -741,11 +746,11 @@ class XMLPrinter : ASTVisitor
 	{
 		output.writeln("<isExpression>");
 		visit(isExpression.type);
-		if (isExpression.identifier.type != TokenType.invalid)
+		if (isExpression.identifier.type != tok!"")
 			visit(isExpression.identifier);
 		if (isExpression.typeSpecialization !is null)
 		{
-			if (isExpression.equalsOrColon == TokenType.colon)
+			if (isExpression.equalsOrColon == tok!":")
 				output.writeln("<colon/>");
 			else
 				output.writeln("<equals/>");
@@ -776,7 +781,7 @@ class XMLPrinter : ASTVisitor
 	override void visit (LabeledStatement labeledStatement)
 	{
 		output.writeln("<labeledStatement label=\"",
-			labeledStatement.identifier.value ,"\">");
+			labeledStatement.identifier.text ,"\">");
 		visit(labeledStatement.declarationOrStatement);
 		output.writeln("</labeledStatement>");
 	}
@@ -784,9 +789,9 @@ class XMLPrinter : ASTVisitor
 	override void visit(LambdaExpression lambdaExpression)
 	{
 		output.writeln("<lambdaExpression>");
-		if (lambdaExpression.functionType == TokenType.function_)
+		if (lambdaExpression.functionType == tok!"function")
 			output.writeln("<function/>");
-		if (lambdaExpression.functionType == TokenType.delegate_)
+		if (lambdaExpression.functionType == tok!"delegate")
 			output.writeln("<delegate/>");
 		lambdaExpression.accept(this);
 		output.writeln("</lambdaExpression>");
@@ -803,14 +808,14 @@ class XMLPrinter : ASTVisitor
 			output.writeln("<linkageAttribute linkage=\"c++\"/>");
 		else
 			output.writeln("<linkageAttribute linkage=\"",
-				linkageAttribute.identifier.value, "\"/>");
+				linkageAttribute.identifier.text, "\"/>");
 	}
 
 	override void visit(MemberFunctionAttribute memberFunctionAttribute)
 	{
 		output.writeln("<memberFunctionAttribute>");
 		if (memberFunctionAttribute.atAttribute is null)
-			output.writeln(getTokenValue(memberFunctionAttribute.tokenType));
+			output.writeln(str(memberFunctionAttribute.tokenType));
 		else
 			memberFunctionAttribute.accept(this);
 		output.writeln("</memberFunctionAttribute>");
@@ -851,7 +856,7 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(MulExpression mulExpression)
 	{
-		output.writeln("<mulExpression operator=\"", getTokenValue(mulExpression.operator) ,"\">");
+		output.writeln("<mulExpression operator=\"", str(mulExpression.operator) ,"\">");
 		output.writeln("<left>");
 		mulExpression.left.accept(this);
 		output.writeln("</left>");
@@ -902,11 +907,11 @@ class XMLPrinter : ASTVisitor
 	override void visit(Parameter param)
 	{
 		output.writeln("<parameter>");
-		if (param.name.type == TokenType.identifier)
+		if (param.name.type == tok!"identifier")
-			output.writeln("<name>", param.name.value, "</name>");
+			output.writeln("<name>", param.name.text, "</name>");
 		foreach (attribute; param.parameterAttributes)
 		{
-			output.writeln("<parameterAttribute>", getTokenValue(attribute), "</parameterAttribute>");
+			output.writeln("<parameterAttribute>", str(attribute), "</parameterAttribute>");
 		}
 		param.accept(this);
 		if (param.vararg)
@@ -927,7 +932,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(PostIncDecExpression postIncDecExpression)
 	{
 		output.writeln("<postIncDecExpression operator=\"",
-			getTokenValue(postIncDecExpression.operator), "\">");
+			str(postIncDecExpression.operator), "\">");
 		postIncDecExpression.accept(this);
 		output.writeln("</postIncDecExpression>");
 	}
@@ -960,7 +965,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(PreIncDecExpression preIncDecExpression)
 	{
 		output.writeln("<preIncDecExpression operator=\"",
-			getTokenValue(preIncDecExpression.operator), "\">");
+			str(preIncDecExpression.operator), "\">");
 		preIncDecExpression.accept(this);
 		output.writeln("</preIncDecExpression>");
 	}
@@ -975,7 +980,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(RelExpression relExpression)
 	{
 		output.writeln("<relExpression operator=\"",
-			xmlEscape(getTokenValue(relExpression.operator)), "\">");
+			xmlEscape(str(relExpression.operator)), "\">");
 		output.writeln("<left>");
 		visit(relExpression.left);
 		output.writeln("</left>");
@@ -1015,7 +1020,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(ShiftExpression shiftExpression)
 	{
 		output.writeln("<shiftExpression operator=\"",
-			xmlEscape(getTokenValue(shiftExpression.operator)), "\">");
+			xmlEscape(str(shiftExpression.operator)), "\">");
 		output.writeln("<left>");
 		visit(shiftExpression.left);
 		output.writeln("</left>");
@@ -1027,10 +1032,10 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(SingleImport singleImport)
 	{
-		if (singleImport.rename.type == TokenType.invalid)
+		if (singleImport.rename.type == tok!"")
 			output.writeln("<singleImport>");
 		else
-			output.writeln("<singleImport rename=\"", singleImport.rename.value, "\">");
+			output.writeln("<singleImport rename=\"", singleImport.rename.text, "\">");
 		visit(singleImport.identifierChain);
 		output.writeln("</singleImport>");
 	}
@@ -1095,7 +1100,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(StructDeclaration structDec)
 	{
 		output.writeln("<structDeclaration line=\"", structDec.name.line, "\">");
-		output.writeln("<name>", structDec.name.value, "</name>");
+		output.writeln("<name>", structDec.name.text, "</name>");
 		structDec.accept(this);
 		output.writeln("</structDeclaration>");
 	}
@@ -1194,10 +1199,10 @@ class XMLPrinter : ASTVisitor
 			output.writeln("</templateDeclaration>");
 			return;
 		}
-
+        writeDdoc(templateDeclaration.comment);
 		output.writeln("<templateDeclaration line=\"",
 			templateDeclaration.name.line, "\">");
-		output.writeln("<name>", templateDeclaration.name.value, "</name>");
+		output.writeln("<name>", templateDeclaration.name.text, "</name>");
 		visit(templateDeclaration.templateParameters);
 		if (templateDeclaration.constraint !is null)
 			visit(templateDeclaration.constraint);
@@ -1276,28 +1281,28 @@ class XMLPrinter : ASTVisitor
 	override void visit(Token token)
 	{
 		string tagName;
-		with (TokenType) switch (token.type)
+		switch (token.type)
 		{
-		case invalid: return;
+		case tok!"": return;
-		case identifier: tagName = "identifier"; break;
+		case tok!"identifier": tagName = "identifier"; break;
-		case doubleLiteral: tagName = "doubleLiteral"; break;
+		case tok!"doubleLiteral": tagName = "doubleLiteral"; break;
-		case idoubleLiteral: tagName = "idoubleLiteral"; break;
+		case tok!"idoubleLiteral": tagName = "idoubleLiteral"; break;
-		case floatLiteral: tagName = "floatLiteral"; break;
+		case tok!"floatLiteral": tagName = "floatLiteral"; break;
-		case ifloatLiteral: tagName = "ifloatLiteral"; break;
+		case tok!"ifloatLiteral": tagName = "ifloatLiteral"; break;
-		case intLiteral: tagName = "intLiteral"; break;
+		case tok!"intLiteral": tagName = "intLiteral"; break;
-		case uintLiteral: tagName = "uintLiteral"; break;
+		case tok!"uintLiteral": tagName = "uintLiteral"; break;
-		case longLiteral: tagName = "longLiteral"; break;
+		case tok!"longLiteral": tagName = "longLiteral"; break;
-		case ulongLiteral: tagName = "ulongLiteral"; break;
+		case tok!"ulongLiteral": tagName = "ulongLiteral"; break;
-		case realLiteral: tagName = "realLiteral"; break;
+		case tok!"realLiteral": tagName = "realLiteral"; break;
-		case irealLiteral: tagName = "irealLiteral"; break;
+		case tok!"irealLiteral": tagName = "irealLiteral"; break;
-		case characterLiteral: tagName = "characterLiteral"; break;
+		case tok!"characterLiteral": tagName = "characterLiteral"; break;
-		case stringLiteral: tagName = "stringLiteral"; break;
+		case tok!"stringLiteral": tagName = "stringLiteral"; break;
-		case dstringLiteral: tagName = "dstringLiteral"; break;
+		case tok!"dstringLiteral": tagName = "dstringLiteral"; break;
-		case wstringLiteral: tagName = "wstringLiteral"; break;
+		case tok!"wstringLiteral": tagName = "wstringLiteral"; break;
-		case dollar: output.writeln("<dollar/>"); return;
+		case tok!"$": output.writeln("<dollar/>"); return;
-		default: output.writeln("<", getTokenValue(token.type), "/>"); return;
+		default: output.writeln("<", str(token.type), "/>"); return;
 		}
-		output.writeln("<", tagName, ">", xmlEscape(token.value), "</", tagName, ">");
+		output.writeln("<", tagName, ">", xmlEscape(token.text), "</", tagName, ">");
 	}
 
 	override void visit(TraitsExpression traitsExpression)
@@ -1323,8 +1328,8 @@ class XMLPrinter : ASTVisitor
 	override void visit(Type2 type2)
 	{
 		output.writeln("<type2>");
-		if (type2.builtinType != TokenType.invalid)
+		if (type2.builtinType != tok!"")
-			output.writeln(getTokenValue(type2.builtinType));
+			output.writeln(str(type2.builtinType));
 		else
 			type2.accept(this);
 		output.writeln("</type2>");
@@ -1393,16 +1398,16 @@ class XMLPrinter : ASTVisitor
 	override void visit(UnaryExpression unaryExpression)
 	{
 		output.writeln("<unaryExpression>");
-		if (unaryExpression.prefix != TokenType.invalid)
+		if (unaryExpression.prefix != tok!"")
 		{
-			output.writeln("<prefix>", xmlEscape(unaryExpression.prefix.value),
+			output.writeln("<prefix>", xmlEscape(unaryExpression.prefix.text),
 				"</prefix>");
 			visit(unaryExpression.unaryExpression);
 		}
-		if (unaryExpression.suffix != TokenType.invalid)
+		if (unaryExpression.suffix != tok!"")
 		{
 			visit(unaryExpression.unaryExpression);
-			output.writeln("<suffix>", unaryExpression.suffix.value,
+			output.writeln("<suffix>", unaryExpression.suffix.text,
 				"</suffix>");
 		}
 		else
@@ -1413,8 +1418,8 @@ class XMLPrinter : ASTVisitor
 	override void visit(UnionDeclaration unionDeclaration)
 	{
 		output.writeln("<unionDeclaration line=\"", unionDeclaration.name.line, "\">");
-		if (unionDeclaration.name != TokenType.invalid)
+		if (unionDeclaration.name != tok!"")
-			output.writeln("<name>", unionDeclaration.name.value, "</name>");
+			output.writeln("<name>", unionDeclaration.name.text, "</name>");
 		if (unionDeclaration.templateParameters !is null)
 			visit(unionDeclaration.templateParameters);
 		if (unionDeclaration.constraint !is null)
@@ -1433,7 +1438,10 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(VariableDeclaration variableDeclaration)
 	{
-		mixin (tagAndAccept!"variableDeclaration");
+        output.writeln("<variableDeclaration>");
+        writeDdoc(variableDeclaration.comment);
+        variableDeclaration.accept(this);
+        output.writeln("</variableDeclaration>");
 	}
 
 	override void visit(Vector vector)
@@ -1478,10 +1486,16 @@ class XMLPrinter : ASTVisitor
 
 	alias ASTVisitor.visit visit;
 
-	private string xmlEscape(string s)
+	private static string xmlEscape(string s)
 	{
 		return s.translate(['<' : "&lt;", '>' : "&gt;", '&' : "&amp;"]);
 	}
 
+    private void writeDdoc(string comment)
+    {
+        if (comment is null) return;
+        output.writeln("<ddoc>", xmlEscape(comment), "</ddoc>");
+    }
+
 	File output;
 }

build.bat

@@ -1 +1 @@
-dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d outliner.d formatter.d stdx/d/lexer.d stdx/d/parser.d stdx/d/entities.d stdx/d/ast.d -wi -ofdscanner
+dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d outliner.d formatter.d stdx/d/lexer.d stdx/d/parser.d stdx/d/entities.d stdx/d/ast.d stdx/lexer.d -wi -ofdscanner

build.sh

@@ -1,5 +1,19 @@
 #dmd *.d stdx/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd
-dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/d/*.d -g -m64 -wi -ofdscanner
+dmd\
-#ldc2 -O3 *.d stdx/d/*.d -of=dscanner-ldc -release -m64
+	main.d\
+	stats.d\
+	imports.d\
+	highlighter.d\
+	ctags.d\
+	astprinter.d\
+	formatter.d\
+	outliner.d\
+	style.d\
+	stdx/*.d\
+	stdx/d/*.d\
+	datapicked/dpick/buffer/*.d\
+	-Idatapicked\
+	-g -m64 -wi -ofdscanner
+#ldc2 main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -of=dscanner-ldc -m64 -oq
 #ldc2 *.d stdx/d/*.d -of=dscanner -unittest -m64 -g
 #/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d stdx/d/*.d

ctags.d

@@ -14,7 +14,7 @@ import std.stdio;
 import std.array;
 import std.conv;
 
-void doNothing(string, int, int, string) {}
+void doNothing(string, size_t, size_t, string) {}
 
 void printCtags(File output, string[] fileNames)
 {
@@ -24,9 +24,8 @@ void printCtags(File output, string[] fileNames)
 		File f = File(fileName);
 		auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
 		f.rawRead(bytes);
-		LexerConfig config;
+		auto tokens = byToken(bytes);
-		auto tokens = byToken(bytes, config);
+		Module m = parseModule(tokens.array, fileName, &doNothing);
-		Module m = parseModule(tokens.array(), fileName, &doNothing);
 		auto printer = new CTagsPrinter;
 		printer.fileName = fileName;
 		printer.visit(m);
@@ -41,88 +40,85 @@ void printCtags(File output, string[] fileNames)
 
 class CTagsPrinter : ASTVisitor
 {
-
-	alias ASTVisitor.visit visit;
-
 	override void visit(ClassDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\tc%s\n".format(dec.name.value, fileName, dec.name.line, context);
+		tagLines ~= "%s\t%s\t%d;\"\tc%s\n".format(dec.name.text, fileName, dec.name.line, context);
 		auto c = context;
-		context = "\tclass:" ~ dec.name.value;
+		context = "\tclass:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(StructDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\ts%s\n".format(dec.name.value, fileName, dec.name.line, context);
+		tagLines ~= "%s\t%s\t%d;\"\ts%s\n".format(dec.name.text, fileName, dec.name.line, context);
 		auto c = context;
-		context = "\tstruct:" ~ dec.name.value;
+		context = "\tstruct:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(InterfaceDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\ti%s\n".format(dec.name.value, fileName, dec.name.line, context);
+		tagLines ~= "%s\t%s\t%d;\"\ti%s\n".format(dec.name.text, fileName, dec.name.line, context);
 		auto c = context;
-		context = "\tclass:" ~ dec.name.value;
+		context = "\tclass:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(TemplateDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.value, fileName, dec.name.line, context);
+		tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.text, fileName, dec.name.line, context);
 		auto c = context;
-		context = "\ttemplate:" ~ dec.name.value;
+		context = "\ttemplate:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(FunctionDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d%s\n".format(dec.name.value, fileName,
+		tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d%s\n".format(dec.name.text, fileName,
 			dec.name.line, dec.parameters.parameters.length, context);
 		auto c = context;
-		context = "\tfunction:" ~ dec.name.value;
+		context = "\tfunction:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(EnumDeclaration dec)
 	{
-		if (dec.name == TokenType.invalid)
+		if (dec.name == tok!"")
 		{
 			dec.accept(this);
 			return;
 		}
-		tagLines ~= "%s\t%s\t%d;\"\tg%s\n".format(dec.name.value, fileName,
+		tagLines ~= "%s\t%s\t%d;\"\tg%s\n".format(dec.name.text, fileName,
 			dec.name.line, context);
 		auto c = context;
-		context = "\tenum:" ~ dec.name.value;
+		context = "\tenum:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(UnionDeclaration dec)
 	{
-		if (dec.name == TokenType.invalid)
+		if (dec.name == tok!"")
 		{
 			dec.accept(this);
 			return;
 		}
-		tagLines ~= "%s\t%s\t%d;\"\tu%s\n".format(dec.name.value, fileName,
+		tagLines ~= "%s\t%s\t%d;\"\tu%s\n".format(dec.name.text, fileName,
 			dec.name.line, context);
 		auto c = context;
-		context = "\tunion:" ~ dec.name.value;
+		context = "\tunion:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(EnumMember mem)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\te%s\n".format(mem.name.value, fileName,
+		tagLines ~= "%s\t%s\t%d;\"\te%s\n".format(mem.name.text, fileName,
 			mem.name.line, context);
 	}
 
@@ -130,11 +126,13 @@ class CTagsPrinter : ASTVisitor
 	{
 		foreach (d; dec.declarators)
 		{
-			tagLines ~= "%s\t%s\t%d;\"\tv%s\n".format(d.name.value, fileName,
+			tagLines ~= "%s\t%s\t%d;\"\tv%s\n".format(d.name.text, fileName,
 				d.name.line, context);
 		}
 		dec.accept(this);
 	}
+	
+	alias ASTVisitor.visit visit;
 
 	string fileName;
 	string[] tagLines;

datapicked

@@ -0,0 +1 @@
+Subproject commit f63a843e9c0ce8db7fd897684fe323697255d87d

formatter.d

@@ -70,7 +70,7 @@ class Formatter(Sink)
 		if (addExpression.right is null)
 			return;
 		sink.put(" ");
-		sink.put(getTokenValue(addExpression.operator));
+		sink.put(str(addExpression.operator));
 		sink.put(" ");
 		format(addExpression.right);
 	}
@@ -333,8 +333,8 @@ class Formatter(Sink)
 	void format(const GotoStatement gotoStatement)
 	{
 		sink.put("goto ");
-		if (gotoStatement.label != TokenType.invalid)
+		if (gotoStatement.label != tok!"")
-			sink.put(gotoStatement.label.value);
+			sink.put(gotoStatement.label.text);
 		else
 			format(gotoStatement.expression);
 		sink.put(";");
@@ -348,7 +348,7 @@ class Formatter(Sink)
 			if (!first)
 				sink.put(".");
 			first = false;
-			sink.put(ident.value);
+			sink.put(ident.text);
 		}
 	}
 
@@ -372,7 +372,7 @@ class Formatter(Sink)
 		if (identifierOrTemplateInstance.templateInstance !is null)
 			format(identifierOrTemplateInstance.templateInstance);
 		else
-			sink.put(identifierOrTemplateInstance.identifier.value);
+			sink.put(identifierOrTemplateInstance.identifier.text);
 
 	}
 
@@ -488,14 +488,14 @@ class Formatter(Sink)
 	{
 		foreach (attribute; parameter.parameterAttributes)
 		{
-			sink.put(getTokenValue(attribute));
+			sink.put(str(attribute));
 		}
 		if (parameter.type !is null)
 			format(parameter.type);
-		if (parameter.name.type != TokenType.invalid)
+		if (parameter.name.type != tok!"")
 		{
 			sink.put(" ");
-			sink.put(parameter.name.value);
+			sink.put(parameter.name.text);
 		}
 		if (parameter.vararg)
 			sink.put(" ...");
@@ -607,7 +607,7 @@ class Formatter(Sink)
 
 	void format(const Symbol symbol)
 	{
-        if (symbol.dot != TokenType.invalid)
+        if (symbol.dot != tok!"")
             sink.put(".");
 		format(symbol.identifierOrTemplateChain);
 	}
@@ -671,7 +671,7 @@ class Formatter(Sink)
 
 	void format(const Token token)
 	{
-		sink.put(token.value);
+		sink.put(token.text);
 	}
 
 	void format(const TraitsExpression traitsExpression)
@@ -688,7 +688,7 @@ class Formatter(Sink)
             if (first)
                 sink.put(" ");
             first = false;
-            sink.put(getTokenValue(constructor));
+            sink.put(str(constructor));
         }
         if (type.typeConstructors.length > 0)
             sink.put(" ");
@@ -711,16 +711,16 @@ class Formatter(Sink)
             format(type2.typeofExpression);
 			return;
         }
-        else if (type2.typeConstructor != TokenType.invalid)
+        else if (type2.typeConstructor != tok!"")
         {
-			sink.put(getTokenValue(type2.typeConstructor));
+			sink.put(str(type2.typeConstructor));
 			sink.put("(");
 			format(type2.type);
 			sink.put(")");
 			return;
         }
         else
-            sink.put(getTokenValue(type2.builtinType));
+            sink.put(str(type2.builtinType));
 	}
 
 	void format(const TypeSpecialization typeSpecialization)

highlighter.d

@@ -11,7 +11,7 @@ import std.array;
 import stdx.d.lexer;
 
 // http://ethanschoonover.com/solarized
-void highlight(R)(TokenRange!R tokens, string fileName)
+void highlight(R)(ref R tokens, string fileName)
 {
 	stdout.writeln(q"[
 <!DOCTYPE html>
@@ -33,30 +33,33 @@ html  { background-color: #fdf6e3; color: #002b36; }
 </style>
 <pre>]");
 
-	foreach (Token t; tokens)
+	while (!tokens.empty)
 	{
+		auto t = tokens.front;
+		tokens.popFront();
 		if (isBasicType(t.type))
-			writeSpan("type", t.value);
+			writeSpan("type", str(t.type));
 		else if (isKeyword(t.type))
-			writeSpan("kwrd", t.value);
+			writeSpan("kwrd", str(t.type));
-		else if (t.type == TokenType.comment)
+		else if (t.type == tok!"comment")
-			writeSpan("com", t.value);
+			writeSpan("com", t.text);
-		else if (isStringLiteral(t.type) || t.type == TokenType.characterLiteral)
+		else if (isStringLiteral(t.type) || t.type == tok!"characterLiteral")
-			writeSpan("str", t.value);
+			writeSpan("str", t.text);
 		else if (isNumberLiteral(t.type))
-			writeSpan("num", t.value);
+			writeSpan("num", t.text);
 		else if (isOperator(t.type))
-			writeSpan("op", t.value);
+			writeSpan("op", str(t.type));
 		else
 		{
 			version(Windows)
 			{
-				// Stupid Windows automatically does a LF → CRLF, so CRLF → CRCRLF, which is obviously wrong.
+				// Stupid Windows automatically does a LF → CRLF, so
+				// CRLF → CRCRLF, which is obviously wrong.
 				// Strip out the CR characters here to avoid this.
-				stdout.write(t.value.replace("<", "&lt;").replace("\r", ""));
+				stdout.write(t.text.replace("<", "&lt;").replace("\r", ""));
 			}
 			else
-				stdout.write(t.value.replace("<", "&lt;"));
+				stdout.write(t.text.replace("<", "&lt;"));
 		}
 
 	}
@@ -70,13 +73,3 @@ void writeSpan(string cssClass, string value)
 	else
 		stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);
 }
-
-/+void main(string[] args)
-{
-	LexerConfig config;
-	config.tokenStyle = TokenStyle.source;
-	config.iterStyle = IterationStyle.everything;
-	config.fileName = args[1];
-	auto f = File(args[1]);
-	(cast(ubyte[]) f.byLine(KeepTerminator.yes).join()).byToken(config).highlight();
-}+/

imports.d

@@ -26,7 +26,7 @@ class ImportPrinter : ASTVisitor
 		{
 			if (!first)
 				write(".");
-			write(ident.value);
+			write(ident.text);
 			first = false;
 		}
 	}

main.d

@@ -17,6 +17,7 @@ import std.stdio;
 import std.range;
 import stdx.d.lexer;
 import stdx.d.parser;
+import dpick.buffer.buffer;
 
 import highlighter;
 import stats;
@@ -24,6 +25,7 @@ import ctags;
 import astprinter;
 import imports;
 import outliner;
+import style;
 
 int main(string[] args)
 {
@@ -39,6 +41,8 @@ int main(string[] args)
 	bool imports;
 	bool muffin;
 	bool outline;
+	bool tokenDump;
+	bool styleCheck;
 
 	try
 	{
@@ -46,6 +50,7 @@ int main(string[] args)
 			"ctags|c", &ctags, "recursive|r|R", &recursive, "help|h", &help,
 			"tokenCount|t", &tokenCount, "syntaxCheck|s", &syntaxCheck,
 			"ast|xml", &ast, "imports|i", &imports, "outline|o", &outline,
+			"tokenDump", &tokenDump, "styleCheck", &styleCheck,
 			"muffinButton", &muffin);
 	}
 	catch (Exception e)
@@ -75,7 +80,7 @@ int main(string[] args)
 	}
 
 	auto optionCount = count!"a"([sloc, highlight, ctags, tokenCount,
-		syntaxCheck, ast, imports, outline]);
+		syntaxCheck, ast, imports, outline, tokenDump, styleCheck]);
 	if (optionCount > 1)
 	{
 		stderr.writeln("Too many options specified");
@@ -89,27 +94,51 @@ int main(string[] args)
 
 	if (highlight)
 	{
-		LexerConfig config;
-		config.iterStyle = IterationStyle.everything;
-		config.tokenStyle = TokenStyle.source;
 		bool usingStdin = args.length == 1;
 		ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
-		highlighter.highlight(byToken(bytes, config),
+        LexerConfig config;
-			args.length == 1 ? "stdin" : args[1]);
+        config.whitespaceBehavior = WhitespaceBehavior.include;
+        config.stringBehavior = StringBehavior.source;
+        config.commentBehavior = CommentBehavior.include;
+		auto tokens = byToken(bytes, config);
+		highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]);
 		return 0;
 	}
+	else if (tokenDump)
+	{
+		bool usingStdin = args.length == 1;
+		ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
+		LexerConfig config;
+        config.whitespaceBehavior = WhitespaceBehavior.skip;
+        config.stringBehavior = StringBehavior.source;
+        config.commentBehavior = CommentBehavior.attach;
+		auto tokens = byToken(bytes, config);
+		foreach (ref token; tokens)
+		{
+			writeln("«", token.text is null ? str(token.type) : token.text,
+				" ", token.index, " ", token.line, " ", token.column, " ",
+                token.comment, "»");
+		}
+	}
 	else if (ctags)
 	{
 		stdout.printCtags(expandArgs(args, recursive));
 	}
+	else if (styleCheck)
+	{
+		stdout.styleCheck(expandArgs(args, recursive));
+	}
 	else
 	{
-		LexerConfig config;
 		bool usingStdin = args.length == 1;
 		if (sloc || tokenCount)
 		{
 			if (usingStdin)
 			{
+                LexerConfig config;
+                config.whitespaceBehavior = WhitespaceBehavior.include;
+                config.stringBehavior = StringBehavior.source;
+                config.commentBehavior = CommentBehavior.include;
 				auto tokens = byToken(readStdin(), config);
 				if (tokenCount)
 					printTokenCount(stdout, "stdin", tokens);
@@ -121,7 +150,7 @@ int main(string[] args)
 				ulong count;
 				foreach (f; expandArgs(args, recursive))
 				{
-					auto tokens = byToken(readFile(f), config);
+					auto tokens = byToken!(ubyte[])(readFile(f));
 					if (tokenCount)
 						count += printTokenCount(stdout, f, tokens);
 					else
@@ -132,48 +161,28 @@ int main(string[] args)
 		}
 		else if (syntaxCheck)
 		{
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
+			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
-				config);
+			parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
-			if (usingStdin)
-				config.fileName = "stdin";
-			else
-				config.fileName = args[1];
-			parseModule(tokens.array(), config.fileName);
 		}
 		else if (imports)
 		{
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
+			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
-				config);
+			auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
-			if (usingStdin)
-				config.fileName = "stdin";
-			else
-				config.fileName = args[1];
-			auto mod = parseModule(tokens.array(), config.fileName);
 			auto visitor = new ImportPrinter;
 			visitor.visit(mod);
 		}
 		else if (ast)
 		{
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
+			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
-				config);
+			auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
-			if (usingStdin)
-				config.fileName = "stdin";
-			else
-				config.fileName = args[1];
-			auto mod = parseModule(tokens.array(), config.fileName);
 			auto printer = new XMLPrinter;
 			printer.output = stdout;
 			printer.visit(mod);
 		}
 		else if (outline)
 		{
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
+			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
-				config);
+			auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
-			if (usingStdin)
-				config.fileName = "stdin";
-			else
-				config.fileName = args[1];
-			auto mod = parseModule(tokens.array(), config.fileName);
 			auto outliner = new Outliner(stdout);
 			outliner.visit(mod);
 		}
@@ -245,7 +254,7 @@ options:
         Prints the number of logical lines of code in the given
         source files. If no files are specified, input is read from stdin.
 
-    --tokenCount | t [sourceFiles]
+    --tokenCount | -t [sourceFiles]
         Prints the number of tokens in the given source files. If no files are
         specified, input is read from stdin.
 
@@ -262,6 +271,10 @@ options:
         syntax errors to stdout. One error or warning is printed per line.
         If no files are specified, input is read from stdin.
 
+    --styleCheck [sourceFiles]
+        Lexes and parses sourceFiles, printing the line and column number of any
+        style guideline violations to stdout.
+
     --ctags | -c sourceFile
         Generates ctags information from the given source code file. Note that
         ctags information requires a filename, so stdin cannot be used in place

outliner.d

@@ -21,7 +21,7 @@ class Outliner : ASTVisitor
 	override void visit(ClassDeclaration classDec)
 	{
 		printIndentation();
-		output.writeln("class ", classDec.name.value, " : ", classDec.name.line);
+		output.writeln("class ", classDec.name.text, " : ", classDec.name.line);
 		indent();
 		classDec.accept(this);
 		outdent();
@@ -31,7 +31,7 @@ class Outliner : ASTVisitor
 	override void visit(EnumDeclaration enumDec)
 	{
 		printIndentation();
-		output.writeln("enum ", enumDec.name.value, " : ", enumDec.name.line);
+		output.writeln("enum ", enumDec.name.text, " : ", enumDec.name.line);
 		indent();
 		enumDec.accept(this);
 		outdent();
@@ -41,7 +41,7 @@ class Outliner : ASTVisitor
 	override void visit(EnumMember enumMem)
 	{
 		printIndentation();
-		output.writeln(enumMem.name.value, " : ", enumMem.name.line);
+		output.writeln(enumMem.name.text, " : ", enumMem.name.line);
 		finish();
 	}
 
@@ -57,7 +57,7 @@ class Outliner : ASTVisitor
 		if (functionDec.returnType !is null)
 			f.format(functionDec.returnType);
 		app.put(" ");
-		app.put(functionDec.name.value);
+		app.put(functionDec.name.text);
 		f.format(functionDec.parameters);
 		app.put(" : ");
 		app.put(to!string(functionDec.name.line));
@@ -68,7 +68,7 @@ class Outliner : ASTVisitor
 	override void visit(InterfaceDeclaration interfaceDec)
 	{
 		printIndentation();
-		output.writeln("interface ", interfaceDec.name.value, " : ",
+		output.writeln("interface ", interfaceDec.name.text, " : ",
 			interfaceDec.name.line);
 		indent();
 		interfaceDec.accept(this);
@@ -79,7 +79,7 @@ class Outliner : ASTVisitor
 	override void visit(StructDeclaration structDec)
 	{
 		printIndentation();
-		output.writeln("struct ", structDec.name.value, " : ",
+		output.writeln("struct ", structDec.name.text, " : ",
 			structDec.name.line);
 		indent();
 		structDec.accept(this);
@@ -90,7 +90,7 @@ class Outliner : ASTVisitor
 	override void visit(TemplateDeclaration templateDeclaration)
 	{
 		printIndentation();
-		output.writeln("template", templateDeclaration.name.value, " : ",
+		output.writeln("template", templateDeclaration.name.text, " : ",
 			templateDeclaration.name.line);
 		finish();
 	}
@@ -105,7 +105,7 @@ class Outliner : ASTVisitor
 	override void visit(UnionDeclaration unionDeclaration)
 	{
 		printIndentation();
-		output.writeln("union ", unionDeclaration.name.value, " : ",
+		output.writeln("union ", unionDeclaration.name.text, " : ",
 			unionDeclaration.name.line);
 		indent();
 		unionDeclaration.accept(this);
@@ -125,7 +125,7 @@ class Outliner : ASTVisitor
 				f.format(variableDeclaration.type);
 			}
 			app.put(" ");
-			app.put(d.name.value);
+			app.put(d.name.text);
 			app.put(" : ");
 			app.put(to!string(d.name.line));
 			output.writeln(app.data);

stats.d

@@ -9,21 +9,21 @@ import std.stdio;
 import std.algorithm;
 import stdx.d.lexer;
 
-pure nothrow bool isLineOfCode(TokenType t)
+pure nothrow bool isLineOfCode(IdType t)
 {
-	with (TokenType) switch(t)
+	switch(t)
 	{
-	case semicolon:
+	case tok!";":
-	case while_:
+	case tok!"while":
-	case if_:
+	case tok!"if":
-	case do_:
+	case tok!"do":
-	case else_:
+	case tok!"else":
-	case switch_:
+	case tok!"switch":
-	case for_:
+	case tok!"for":
-	case foreach_:
+	case tok!"foreach":
-	case foreach_reverse_:
+	case tok!"foreach_reverse":
-	case default_:
+	case tok!"default":
-	case case_:
+	case tok!"case":
 		return true;
 	default:
 		return false;

stdx/d/ast.d

@@ -275,7 +275,7 @@ public:
     {
         mixin (visitIfNotNull!(left, right));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -290,6 +290,7 @@ public:
     /** */ Type type;
     /** */ Token name;
     /** */ AliasInitializer[] initializers;
+    /** */ string comment;
 }
 
 ///
@@ -410,7 +411,7 @@ class AsmAddExp : ExpressionNode
 {
 public:
     mixin (DEFAULT_ACCEPT);
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -483,7 +484,7 @@ class AsmMulExp : ExpressionNode
 {
 public:
     mixin (DEFAULT_ACCEPT);
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 
 }
@@ -583,7 +584,7 @@ public:
     }
     /** */ ExpressionNode ternaryExpression;
     /** */ ExpressionNode assignExpression;
-    /** */ TokenType operator;
+    /** */ IdType operator;
 }
 
 ///
@@ -623,7 +624,7 @@ public:
     /** */ AlignAttribute alignAttribute;
     /** */ PragmaExpression pragmaExpression;
     /** */ StorageClass storageClass;
-    /** */ TokenType attribute;
+    /** */ IdType attribute;
 }
 
 ///
@@ -808,6 +809,7 @@ public:
     /** */ Constraint constraint;
     /** */ BaseClassList baseClassList;
     /** */ StructBody structBody;
+    /** */ string comment;
 }
 
 ///
@@ -891,6 +893,7 @@ public:
     /** */ MemberFunctionAttribute[] memberFunctionAttributes;
     /** */ TemplateParameters templateParameters;
     /** */ size_t location;
+    /** */ string comment;
 }
 
 ///
@@ -943,7 +946,7 @@ public:
             destructor, staticConstructor, staticDestructor,
             sharedStaticDestructor, sharedStaticConstructor,
             conditionalDeclaration, pragmaDeclaration, versionSpecification,
-            declarations));
+			invariant_, postblit, declarations));
     }
 
     /** */ Attribute[] attributes;
@@ -1067,6 +1070,7 @@ public:
     }
     /** */ FunctionBody functionBody;
     /** */ size_t location;
+    /** */ string comment;
 }
 
 ///
@@ -1113,6 +1117,7 @@ public:
     /** */ Token name;
     /** */ Type type;
     /** */ EnumBody enumBody;
+    /** */ string comment;
 }
 
 ///
@@ -1126,6 +1131,7 @@ public:
     /** */ Token name;
     /** */ Type type;
     /** */ AssignExpression assignExpression;
+    /** */ string comment;
 }
 
 ///
@@ -1149,7 +1155,7 @@ public:
     {
         mixin (visitIfNotNull!(left, right));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -1222,7 +1228,7 @@ public:
         mixin (visitIfNotNull!(foreachType, foreachTypeList, low, high,
             declarationOrStatement));
     }
-    /** */ TokenType type;
+    /** */ IdType type;
     /** */ ForeachTypeList foreachTypeList;
 	/** */ ForeachType foreachType;
     /** */ Expression low;
@@ -1239,7 +1245,7 @@ public:
     {
         mixin (visitIfNotNull!(type, identifier));
     }
-    /** */ TokenType[] typeConstructors;
+    /** */ IdType[] typeConstructors;
     /** */ Type type;
     /** */ Token identifier;
 }
@@ -1325,6 +1331,7 @@ public:
     /** */ Constraint constraint;
     /** */ FunctionBody functionBody;
     /** */ MemberFunctionAttribute[] memberFunctionAttributes;
+    /** */ string comment;
 }
 
 ///
@@ -1336,7 +1343,7 @@ public:
         mixin (visitIfNotNull!(type, parameters, functionAttributes,
             functionBody));
     }
-    /** */ TokenType functionOrDelegate;
+    /** */ IdType functionOrDelegate;
     /** */ Type type;
     /** */ Parameters parameters;
     /** */ FunctionAttribute[] functionAttributes;
@@ -1549,6 +1556,7 @@ public:
     /** */ Constraint constraint;
     /** */ BaseClassList baseClassList;
     /** */ StructBody structBody;
+    /** */ string comment;
 }
 
 ///
@@ -1560,6 +1568,7 @@ public:
         mixin (visitIfNotNull!(blockStatement));
     }
     /** */ BlockStatement blockStatement;
+    /** */ string comment;
 }
 
 ///
@@ -1575,7 +1584,7 @@ public:
     /** */ Token identifier;
     /** */ TypeSpecialization typeSpecialization;
     /** */ TemplateParameterList templateParameterList;
-    /** */ TokenType equalsOrColon;
+    /** */ IdType equalsOrColon;
 }
 
 ///
@@ -1622,7 +1631,7 @@ public:
         mixin (visitIfNotNull!(identifier, parameters, functionAttributes,
             assignExpression));
     }
-    /** */ TokenType functionType;
+    /** */ IdType functionType;
     /** */ Token identifier;
     /** */ Parameters parameters;
     /** */ FunctionAttribute[] functionAttributes;
@@ -1660,7 +1669,7 @@ public:
     {
         mixin (visitIfNotNull!(atAttribute));
     }
-    /** */ TokenType tokenType;
+    /** */ IdType tokenType;
     /** */ AtAttribute atAttribute;
 }
 
@@ -1743,7 +1752,7 @@ public:
     {
         mixin (visitIfNotNull!(left, right));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -1893,7 +1902,7 @@ public:
         mixin (visitIfNotNull!(type, name, default_));
     }
 
-    /** */ TokenType[] parameterAttributes;
+    /** */ IdType[] parameterAttributes;
     /** */ Type type;
     /** */ Token name;
     /** */ bool vararg;
@@ -1932,7 +1941,7 @@ public:
     {
         mixin (visitIfNotNull!(unaryExpression));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     /** */ UnaryExpression unaryExpression;
 }
 
@@ -1978,7 +1987,7 @@ public:
     {
         mixin (visitIfNotNull!(unaryExpression));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     /** */ UnaryExpression unaryExpression;
 }
 
@@ -2030,7 +2039,7 @@ public:
     {
         mixin (visitIfNotNull!(left, right));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -2067,6 +2076,7 @@ public:
     }
     /** */ FunctionBody functionBody;
     /** */ size_t location;
+    /** */ string comment;
 }
 
 ///
@@ -2079,6 +2089,7 @@ public:
     }
     /** */ FunctionBody functionBody;
     /** */ size_t location;
+    /** */ string comment;
 }
 
 ///
@@ -2089,7 +2100,7 @@ public:
     {
         mixin (visitIfNotNull!(left, right));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -2236,6 +2247,7 @@ public:
     /** */ TemplateParameters templateParameters;
     /** */ Constraint constraint;
     /** */ StructBody structBody;
+    /** */ string comment;
 }
 
 ///
@@ -2376,6 +2388,7 @@ public:
     /** */ Constraint constraint;
     /** */ Declaration[] declarations;
     /** */ EponymousTemplateDeclaration eponymousTemplateDeclaration;
+    /** */ string comment;
 }
 
 ///
@@ -2573,7 +2586,7 @@ public:
         mixin (visitIfNotNull!(type2, typeSuffixes));
     }
 
-    /** */ TokenType[] typeConstructors;
+    /** */ IdType[] typeConstructors;
     /** */ TypeSuffix[] typeSuffixes;
     /** */ Type2 type2;
 }
@@ -2588,11 +2601,11 @@ public:
             identifierOrTemplateChain, type));
     }
 
-    /** */ TokenType builtinType;
+    /** */ IdType builtinType;
     /** */ Symbol symbol;
     /** */ TypeofExpression typeofExpression;
     /** */ IdentifierOrTemplateChain identifierOrTemplateChain;
-    /** */ TokenType typeConstructor;
+    /** */ IdType typeConstructor;
     /** */ Type type;
 }
 
@@ -2694,6 +2707,7 @@ public:
     /** */ TemplateParameters templateParameters;
     /** */ Constraint constraint;
     /** */ StructBody structBody;
+    /** */ string comment;
 }
 
 ///
@@ -2705,6 +2719,7 @@ public:
         mixin (visitIfNotNull!(blockStatement));
     }
     /** */ BlockStatement blockStatement;
+    /** */ string comment;
 }
 
 ///
@@ -2719,6 +2734,7 @@ public:
     /** */ Declarator[] declarators;
     /** */ StorageClass storageClass;
     /** */ AutoDeclaration autoDeclaration;
+    /** */ string comment;
 }
 
 ///

stdx/d/entities.d

@@ -6,7 +6,7 @@
  * Copyright: Brian Schott 2013
  * License: <a href="http://www.boost.org/LICENSE_1_0.txt">Boost License 1.0</a>.
  * Authors: Brian Schott
- * Source: $(PHOBOSSRC std/d/_lexer.d)
+ * Source: $(PHOBOSSRC std/d/_entities.d)
  */
 
 module stdx.d.entities;
@@ -18,7 +18,7 @@ module stdx.d.entities;
 struct HtmlEntity
 {
     string name, value;
-} 
+}
 
 immutable HtmlEntity[] characterEntities = [
         HtmlEntity("AElig", "\u00C6"),

stdx/lexer.d

@@ -0,0 +1,300 @@
+// Written in the D programming language
+
+/**
+ * This module contains a range-based _lexer generator.
+ *
+ * Copyright: Brian Schott 2013
+ * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0)
+ * Authors: Brian Schott, with ideas shamelessly stolen from Andrei Alexandrescu
+ * Source: $(PHOBOSSRC std/_lexer.d)
+ */
+
+module stdx.lexer;
+
+import std.typecons;
+import std.algorithm;
+import std.range;
+import std.traits;
+import std.conv;
+import std.math;
+import dpick.buffer.buffer;
+import dpick.buffer.traits;
+
+template TokenIdType(alias staticTokens, alias dynamicTokens,
+	alias possibleDefaultTokens)
+{
+  static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= ubyte.max)
+		alias TokenIdType = ubyte;
+	else static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= ushort.max)
+		alias TokenIdType = ushort;
+	else static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= uint.max)
+		alias TokenIdType = uint;
+	else
+		static assert (false);
+}
+
+string tokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens)(IdType type) @property
+{
+	if (type == 0)
+		return "!ERROR!";
+	else if (type < staticTokens.length + 1)
+		return staticTokens[type - 1];
+	else if (type < staticTokens.length + possibleDefaultTokens.length + 1)
+		return possibleDefaultTokens[type - staticTokens.length - 1];
+    else if (type < staticTokens.length + possibleDefaultTokens.length + dynamicTokens.length + 1)
+		return dynamicTokens[type - staticTokens.length - possibleDefaultTokens.length - 1];
+	else
+		return null;
+}
+
+template TokenId(IdType, alias staticTokens, alias dynamicTokens,
+	alias possibleDefaultTokens, string symbol)
+{
+	static if (symbol == "")
+	{
+	  enum id = 0;
+		alias id TokenId;
+	}
+	else static if (symbol == "\0")
+	{
+		enum id = 1 + staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length;
+		alias id TokenId;
+	}
+	else
+	{
+		enum i = staticTokens.countUntil(symbol);
+		static if (i >= 0)
+		{
+			enum id = i + 1;
+			alias id TokenId;
+		}
+		else
+		{
+			enum ii = possibleDefaultTokens.countUntil(symbol);
+			static if (ii >= 0)
+			{
+				enum id = ii + staticTokens.length + 1;
+				static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol);
+				alias id TokenId;
+			}
+			else
+			{
+				enum dynamicId = dynamicTokens.countUntil(symbol);
+				enum id = dynamicId >= 0
+					? i + staticTokens.length + possibleDefaultTokens.length + dynamicId + 1
+					: -1;
+				static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol);
+				alias id TokenId;
+			}
+		}
+	}
+}
+
+struct TokenStructure(IDType, string extraFields = "")
+{
+	bool opEquals(IDType type) const pure nothrow @safe
+	{
+		return this.type == type;
+	}
+
+	this(IDType type)
+	{
+		this.type = type;
+	}
+
+	this(IDType type, string text, size_t line, size_t column, size_t index)
+	{
+		this.text = text;
+		this.line = line;
+		this.column = column;
+		this.type = type;
+		this.index = index;
+	}
+
+	string text;
+	size_t line;
+	size_t column;
+	size_t index;
+	IDType type;
+	mixin (extraFields);
+}
+
+mixin template Lexer(R, IDType, Token, alias defaultTokenFunction,
+	alias staticTokens, alias dynamicTokens, alias pseudoTokens,
+	alias pseudoTokenHandlers, alias possibleDefaultTokens)
+{
+	static string generateCaseStatements(string[] tokens, size_t offset = 0)
+	{
+		string code;
+		for (size_t i = 0; i < tokens.length; i++)
+		{
+			auto indent = "";
+			foreach (k; 0 .. offset)
+				indent ~= "    ";
+			size_t j = i + 1;
+
+			if (offset < tokens[i].length)
+			{
+				while (j < tokens.length && offset < tokens[j].length
+					&& tokens[i][offset] == tokens[j][offset]) j++;
+				code ~= indent ~ "case " ~ text(cast(ubyte) tokens[i][offset]) ~ ":\n";
+				if (i + 1 >= j)
+				{
+					if (offset + 1 == tokens[i].length)
+						code ~= generateLeaf(tokens[i], indent ~ "    ");
+					else
+					{
+						code ~= indent ~ "    if (range.lookahead(" ~ text(tokens[i].length) ~ ").length == 0)\n";
+						code ~= indent ~ "        goto outer_default;\n";
+						code ~= indent ~ "    if (range.lookahead(" ~ text(tokens[i].length) ~ ") == \"" ~ escape(tokens[i]) ~ "\")\n";
+						code ~= indent ~ "    {\n";
+						code ~= generateLeaf(tokens[i], indent ~ "        ");
+						code ~= indent ~ "    }\n";
+						code ~= indent ~ "    else\n";
+						code ~= indent ~ "        goto outer_default;\n";
+					}
+				}
+				else
+				{
+					code ~= indent ~ "    if (range.lookahead(" ~ text(offset + 2) ~ ").length == 0)\n";
+					code ~= indent ~ "    {\n";
+					code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ "        ");
+					code ~= indent ~ "    }\n";
+					code ~= indent ~ "    switch (range.lookahead(" ~ text(offset + 2) ~ ")[" ~ text(offset + 1) ~ "])\n";
+					code ~= indent ~ "    {\n";
+					code ~= generateCaseStatements(tokens[i .. j], offset + 1);
+					code ~= indent ~ "    default:\n";
+					code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ "        ");
+					code ~= indent ~ "    }\n";
+				}
+			}
+			i = j - 1;
+		}
+		return code;
+	}
+
+	static string generateLeaf(string token, string indent)
+	{
+		static assert (pseudoTokenHandlers.length % 2 == 0,
+			"Each pseudo-token must have a matching function name.");
+		string code;
+		if (staticTokens.countUntil(token) >= 0)
+		{
+			if (token.length == 1)
+				code ~= indent ~ "range.popFront();\n";
+			else
+				code ~= indent ~ "range.popFrontN(" ~ text(token.length) ~ ");\n";
+			code ~= indent ~ "return Token(tok!\"" ~ escape(token) ~ "\", null, range.line, range.column, range.index);\n";
+		}
+		else if (pseudoTokens.countUntil(token) >= 0)
+			code ~= indent ~ "return " ~ pseudoTokenHandlers[pseudoTokenHandlers.countUntil(token) + 1] ~ "();\n";
+		else if (possibleDefaultTokens.countUntil(token) >= 0)
+		{
+			code ~= indent ~ "if (range.lookahead(" ~ text(token.length + 1) ~ ").length == 0 || isSeparating(range.lookahead(" ~ text(token.length + 1) ~ ")[" ~ text(token.length) ~ "]))\n";
+			code ~= indent ~ "{\n";
+			if (token.length == 1)
+				code ~= indent ~ "    range.popFront();\n";
+			else
+				code ~= indent ~ "    range.popFrontN(" ~ text(token.length) ~ ");\n";
+			code ~= indent ~ "    return Token(tok!\"" ~ escape(token) ~"\", null, range.line, range.column, range.index);\n";
+			code ~= indent ~ "}\n";
+			code ~= indent ~ "else\n";
+			code ~= indent ~ "    goto outer_default;\n";
+		}
+		else
+			code ~= indent ~ "goto outer_default;\n";
+		return code;
+	}
+
+	const(Token) front() pure nothrow const @property
+	{
+		return _front;
+	}
+
+	void _popFront() pure
+	{
+		_front = advance();
+	}
+
+	bool empty() pure const nothrow @property
+	{
+		return _front.type == tok!"\0";
+	}
+
+	static string escape(string input)
+	{
+		string rVal;
+		foreach (ubyte c; cast(ubyte[]) input)
+		{
+			switch (c)
+			{
+			case '\\': rVal ~= `\\`; break;
+			case '"': rVal ~= `\"`; break;
+			case '\'': rVal ~= `\'`; break;
+			case '\t': rVal ~= `\t`; break;
+			case '\n': rVal ~= `\n`; break;
+			case '\r': rVal ~= `\r`; break;
+			default: rVal ~= c; break;
+			}
+		}
+		return rVal;
+	}
+
+	Token advance() pure
+	{
+		if (range.empty)
+			return Token(tok!"\0");
+		lexerLoop: switch (range.front)
+		{
+		mixin(generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens))));
+//		pragma(msg, generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens))));
+		outer_default:
+		default:
+			return defaultTokenFunction();
+		}
+	}
+
+	/**
+	 * This only exists because the real array() can't be called at compile-time
+	 */
+	static T[] stupidToArray(R, T = ElementType!R)(R range)
+	{
+		T[] rVal;
+		foreach (v; range)
+			rVal ~= v;
+		return rVal;
+	}
+
+	LexerRange!(typeof(buffer(R.init))) range;
+	Token _front;
+}
+
+struct LexerRange(BufferType) if (isBuffer!BufferType)
+{
+	this(BufferType r)
+	{
+		this.range = r;
+		index = 0;
+		column = 1;
+		line = 1;
+	}
+
+	void popFront() pure
+	{
+		index++;
+		column++;
+		range.popFront();
+	}
+
+	void incrementLine() pure nothrow
+	{
+		column = 1;
+		line++;
+	}
+
+	BufferType range;
+	alias range this;
+	size_t index;
+	size_t column;
+	size_t line;
+}

style.d

@@ -0,0 +1,102 @@
+//          Copyright Brian Schott (Sir Alaran) 2014.
+// Distributed under the Boost Software License, Version 1.0.
+//    (See accompanying file LICENSE_1_0.txt or copy at
+//          http://www.boost.org/LICENSE_1_0.txt)
+
+module style;
+
+import stdx.d.ast;
+import stdx.d.lexer;
+import stdx.d.parser;
+import std.stdio;
+import std.regex;
+import std.array;
+import std.conv;
+
+void doNothing(string, size_t, size_t, string) {}
+
+void styleCheck(File output, string[] fileNames)
+{
+	foreach (fileName; fileNames)
+	{
+		File f = File(fileName);
+		auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
+		f.rawRead(bytes);
+		auto tokens = byToken(bytes);
+		Module m = parseModule(tokens.array, fileName, &doNothing);
+		auto checker = new StyleChecker;
+		checker.fileName = fileName;
+		checker.visit(m);
+	}
+}
+
+class StyleChecker : ASTVisitor
+{
+	enum varFunNameRegex = `^([\p{Ll}_][_\w\d]*|[\p{Lu}\d_]+)$`;
+	enum aggregateNameRegex = `^\p{Lu}[\w\d]*$`;
+	enum moduleNameRegex = `^\p{Ll}+$`;
+	
+	override void visit(ModuleDeclaration dec)
+	{
+		foreach (part; dec.moduleName.identifiers)
+		{
+			if (part.text.matchFirst(moduleNameRegex).length == 0)
+				writeln(fileName, "(", part.line, ":", part.column, ") ",
+					"Module/package name ", part.text, " does not match style guidelines");
+		}
+	}
+	
+	override void visit(Declarator dec)
+	{
+		checkLowercaseName("Variable", dec.name);
+	}
+	
+	override void visit(FunctionDeclaration dec)
+	{
+		checkLowercaseName("Function", dec.name);
+	}
+	
+	void checkLowercaseName(string type, ref Token name)
+	{
+		if (name.text.matchFirst(varFunNameRegex).length == 0)
+			writeln(fileName, "(", name.line, ":", name.column, ") ",
+				type, " name ", name.text, " does not match style guidelines");
+	}
+	
+	override void visit(ClassDeclaration dec)
+	{
+		checkAggregateName("Class", dec.name);
+		dec.accept(this);
+	}
+	
+	override void visit(InterfaceDeclaration dec)
+	{
+		checkAggregateName("Interface", dec.name);
+		dec.accept(this);
+	}
+	
+	override void visit(EnumDeclaration dec)
+	{
+		if (dec.name.text is null || dec.name.text.length == 0)
+			return;
+		checkAggregateName("Enum", dec.name);
+		dec.accept(this);
+	}
+	
+	override void visit(StructDeclaration dec)
+	{
+		checkAggregateName("Struct", dec.name);
+		dec.accept(this);
+	}
+	
+	void checkAggregateName(string aggregateType, ref Token name)
+	{
+		if (name.text.matchFirst(aggregateNameRegex).length == 0)
+			writeln(fileName, "(", name.line, ":", name.column, ") ",
+				aggregateType, " name ", name.text,
+				" does not match style guidelines");
+	}
+
+	alias ASTVisitor.visit visit;
+	string fileName;
+}