diff --git a/highlighter.d b/highlighter.d
index 1478a1c..032c3f4 100644
--- a/highlighter.d
+++ b/highlighter.d
@@ -24,12 +24,11 @@ void highlight(R)(R tokens)
]");
@@ -39,22 +38,22 @@ html { background-color: #111; color: #ccc; }
switch (t.type)
{
case TokenType.KEYWORDS_BEGIN: .. case TokenType.KEYWORDS_END:
- writeSpan("keyword", t.value);
+ writeSpan("kwrd", t.value);
break;
case TokenType.TYPES_BEGIN: .. case TokenType.TYPES_END:
writeSpan("type", t.value);
break;
case TokenType.Comment:
- writeSpan("comment", t.value);
+ writeSpan("com", t.value);
break;
case TokenType.STRINGS_BEGIN: .. case TokenType.STRINGS_END:
- writeSpan("string", t.value);
+ writeSpan("str", t.value);
break;
case TokenType.NUMBERS_BEGIN: .. case TokenType.NUMBERS_END:
- writeSpan("number", t.value);
+ writeSpan("num", t.value);
break;
case TokenType.OPERATORS_BEGIN: .. case TokenType.OPERATORS_END:
- writeSpan("operator", t.value);
+ writeSpan("op", t.value);
break;
default:
stdout.write(t.value.replace("<", "<"));
diff --git a/tokenizer.d b/tokenizer.d
index dae129e..3dea8ad 100644
--- a/tokenizer.d
+++ b/tokenizer.d
@@ -517,6 +517,7 @@ body
t.startIndex = index;
t.type = TokenType.StringLiteral;
auto app = appender!(char[])();
+ bool isWysiwyg = input.front == 'r' || input.front == '`';
if (input.front == 'r')
{
if (style & StringStyle.IncludeQuotes)
@@ -542,7 +543,7 @@ body
{
auto r = input.save();
r.popFront();
- if (r.front == quote)
+ if (r.front == quote && !isWysiwyg)
{
app.put('\\');
app.put(quote);
@@ -550,7 +551,7 @@ body
input.popFront();
index += 2;
}
- else if (r.front == '\\')
+ else if (r.front == '\\' && !isWysiwyg)
{
app.put('\\');
app.put('\\');
@@ -1394,14 +1395,14 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
current = lexString(range, index, lineNumber, stringStyle);
break;
case 'q':
- auto r = range.save;
+ /+auto r = range.save;
r.popFront();
if (!r.isEoF() && r.front == '{')
{
writeln("ParseTokenString");
break;
}
- else
+ else+/
goto default;
case '/':
auto r = range.save();
@@ -1410,6 +1411,8 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
{
current.type = TokenType.Div;
current.value = "/";
+ range.popFront();
+ ++index;
break;
}
switch (r.front)
@@ -1422,19 +1425,23 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
case '=':
current.type = TokenType.DivEquals;
current.value = "/=";
+ range.popFront();
+ range.popFront();
+ index += 2;
break outer;
default:
current.type = TokenType.Div;
current.value = "/";
- break;
+ ++index;
+ range.popFront();
+ break outer;
}
- break;
case 'r':
auto r = range.save();
r.popFront();
if (!r.isEoF() && r.front == '"')
{
- current = lexString(range, index, lineNumber, StringStyle.NotEscaped);
+ current = lexString(range, index, lineNumber, stringStyle);
break;
}
else
@@ -1455,6 +1462,7 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
{
app.put(range.front);
range.popFront();
+ ++index;
}
current.value = to!string(app.data);
current.type = lookupTokenTypeOptimized(current.value);
@@ -1475,7 +1483,7 @@ private:
unittest
{
- auto c = "rust r\"\\ntest\" r`eh?`";
- foreach (t; byToken(c))
- writeln(t);
+ auto c = `r"d:\path\foo.bat"`;
+ foreach (t; byToken(c, IterationStyle.CodeOnly, StringStyle.Source))
+ writeln(t.type, ": {", t.value, "}");
}