forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed lexing/parsing bugs

This commit is contained in:
Hackerpilot 2014-01-11 17:10:11 +00:00
parent 9d8ad4679d
commit 844b626ed5
3 changed files with 29 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
main.d
View File

@ -40,6 +40,7 @@ int main(string[] args)
bool imports; bool imports;
bool muffin; bool muffin;
bool outline; bool outline;
bool tokenDump;
try try
{ {
@ -47,7 +48,7 @@ int main(string[] args)
"ctags|c", &ctags, "recursive|r|R", &recursive, "help|h", &help, "ctags|c", &ctags, "recursive|r|R", &recursive, "help|h", &help,
"tokenCount|t", &tokenCount, "syntaxCheck|s", &syntaxCheck, "tokenCount|t", &tokenCount, "syntaxCheck|s", &syntaxCheck,
"ast|xml", &ast, "imports|i", &imports, "outline|o", &outline, "ast|xml", &ast, "imports|i", &imports, "outline|o", &outline,
"muffinButton", &muffin); "tokenDump", &tokenDump, "muffinButton", &muffin);
} }
catch (Exception e) catch (Exception e)
{ {
@ -76,7 +77,7 @@ int main(string[] args)
} }
auto optionCount = count!"a"([sloc, highlight, ctags, tokenCount, auto optionCount = count!"a"([sloc, highlight, ctags, tokenCount,
syntaxCheck, ast, imports, outline]); syntaxCheck, ast, imports, outline, tokenDump]);
if (optionCount > 1) if (optionCount > 1)
{ {
stderr.writeln("Too many options specified"); stderr.writeln("Too many options specified");
@ -96,6 +97,17 @@ int main(string[] args)
highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]); highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]);
return 0; return 0;
} }
else if (tokenDump)
{
bool usingStdin = args.length == 1;
ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
auto tokens = byToken!(ubyte[], false, false)(bytes);
foreach (ref token; tokens)
{
writeln("«", token.text is null ? str(token.type) : token.text,
" ", token.index, " ", token.line, " ", token.column, "»");
}
}
else if (ctags) else if (ctags)
{ {
stdout.printCtags(expandArgs(args, recursive)); stdout.printCtags(expandArgs(args, recursive));

16
stdx/d/lexer.d
View File

@ -482,10 +482,10 @@ public struct DLexer(R)
Token lexNumber() pure nothrow Token lexNumber() pure nothrow
{ {
auto mark = range.mark(); auto mark = range.mark();
auto lookahead = range.lookahead(1); auto lookahead = range.lookahead(2);
if (range.front == '0' && lookahead.length == 1) if (range.front == '0' && lookahead.length == 2)
{ {
switch (lookahead[0]) switch (lookahead[1])
{ {
case 'x': case 'x':
case 'X': case 'X':
@ -1040,10 +1040,18 @@ public struct DLexer(R)
else else
app.put(t.text); app.put(t.text);
if (t.type == tok!"}") if (t.type == tok!"}")
{
depth--; depth--;
if (depth > 0)
popFront();
}
else if (t.type == tok!"{") else if (t.type == tok!"{")
{
depth++; depth++;
popFront(); popFront();
}
else
popFront();
} }
IdType type = tok!"stringLiteral"; IdType type = tok!"stringLiteral";
lexStringSuffix(type); lexStringSuffix(type);

5
stdx/d/parser.d
View File

@ -12,7 +12,7 @@ import std.string : format;
// Uncomment this if you want ALL THE OUTPUT // Uncomment this if you want ALL THE OUTPUT
// Caution: generates 180 megabytes of logging for std.datetime // Caution: generates 180 megabytes of logging for std.datetime
version = std_parser_verbose; //version = std_parser_verbose;
/** /**
* Params: * Params:
@ -1248,7 +1248,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
case tok!"!<=": case tok!"!<=":
node.relExpression = parseRelExpression(shift); node.relExpression = parseRelExpression(shift);
break; break;
case tok!"=": case tok!"==":
case tok!"!=": case tok!"!=":
node.equalExpression = parseEqualExpression(shift); node.equalExpression = parseEqualExpression(shift);
break; break;
@ -6547,6 +6547,7 @@ protected:
case tok!"stringLiteral": case tok!"stringLiteral":
case tok!"wstringLiteral": case tok!"wstringLiteral":
case tok!"dstringLiteral": case tok!"dstringLiteral":
case tok!"characterLiteral":
}; };
enum string SPECIAL_CASES = q{ enum string SPECIAL_CASES = q{
case tok!"__DATE__": case tok!"__DATE__":