diff --git a/build.sh b/build.sh
index f0f544f..bbf0ff4 100755
--- a/build.sh
+++ b/build.sh
@@ -1,2 +1,2 @@
-dmd *.d std/d/*.d -release -noboundscheck -O -w -wi -m64 -property -ofdscanner -L-lsqlite3 #-inline
-#dmd *.d std/d/*.d -g -m64 -w -wi -property -ofdscanner -L-lsqlite3 #-unittest
+#dmd *.d std/d/*.d -release -noboundscheck -O -w -wi -m64 -property -ofdscanner -L-lsqlite3 #-inline
+dmd *.d std/d/*.d -g -m64 -w -wi -property -ofdscanner -L-lsqlite3 #-unittest
diff --git a/circularbuffer.d b/circularbuffer.d
index 9c45128..11e1053 100644
--- a/circularbuffer.d
+++ b/circularbuffer.d
@@ -14,28 +14,28 @@ class CircularBuffer(T) : InputRange!(T)
{
public:
- this (size_t size, InputRange!(T) range)
- {
- this.range = range;
- this.margin = size;
- data = new T[(margin * 2) + 1];
- if (range.empty())
- {
- _empty = true;
- return;
- }
- for (size_t i = 0; i <= margin && !this.range.empty(); ++i)
- {
- data[i] = this.range.front();
- this.range.popFront();
+ this (size_t size, InputRange!(T) range)
+ {
+ this.range = range;
+ this.margin = size;
+ data = new T[(margin * 2) + 1];
+ if (range.empty())
+ {
+ _empty = true;
+ return;
+ }
+ for (size_t i = 0; i <= margin && !this.range.empty(); ++i)
+ {
+ data[i] = this.range.front();
+ this.range.popFront();
end++;
- }
- }
+ }
+ }
- override T front() const @property
- {
- return data[index];
- }
+ override T front() @property
+ {
+ return data[index];
+ }
T peek(int offset = 1)
in
@@ -52,32 +52,32 @@ public:
return abs(offset) <= margin && sourceIndex + offset >= 0;
}
- override void popFront()
+ override void popFront()
in
{
assert (!_empty);
}
body
- {
+ {
index = (index + 1) % data.length;
++sourceIndex;
- if (range.empty())
+ if (range.empty())
{
if (index == end)
_empty = true;
}
- else
- {
+ else
+ {
data[end] = range.front();
end = (end + 1) % data.length;
range.popFront();
- }
- }
+ }
+ }
- bool empty() const @property
- {
- return _empty;
- }
+ bool empty() const @property
+ {
+ return _empty;
+ }
override T moveFront()
{
@@ -94,6 +94,7 @@ public:
result = dg(front);
if (result)
break;
+ popFront();
}
return result;
}
@@ -107,18 +108,19 @@ public:
result = dg(i, front);
if (result)
break;
+ popFront();
}
return result;
}
private:
- InputRange!(T) range;
- immutable size_t margin;
- T[] data;
+ InputRange!(T) range;
+ immutable size_t margin;
+ T[] data;
size_t sourceIndex;
size_t end;
- size_t index;
- bool _empty;
+ size_t index;
+ bool _empty;
}
unittest
@@ -131,10 +133,10 @@ unittest
unittest
{
- int[] arr = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
- auto buf = CircularBuffer!(int, int[])(2, arr);
- assert (buf.data.length == 5);
- auto iterated = array(buf);
+ int[] arr = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+ auto buf = CircularBuffer!(int, int[])(2, arr);
+ assert (buf.data.length == 5);
+ auto iterated = array(buf);
assert (iterated == arr);
}
diff --git a/highlighter.d b/highlighter.d
index 324712d..6bd4482 100644
--- a/highlighter.d
+++ b/highlighter.d
@@ -10,8 +10,6 @@ import std.stdio;
import std.array;
import std.d.lexer;
-import langutils;
-
void writeSpan(string cssClass, string value)
{
stdout.write(``, value.replace("&", "&").replace("<", "<"), ``);
diff --git a/langutils.d b/langutils.d
index cf0c995..00bd391 100644
--- a/langutils.d
+++ b/langutils.d
@@ -6,6 +6,7 @@
module langutils;
import std.array;
+import std.algorithm;
import std.d.lexer;
@@ -43,7 +44,7 @@ string combineTokens(ref const Token[] tokens)
return app.data;
}
-pure string getTypeFromToken(const Token t)
+pure nothrow string getTypeFromToken(const Token t)
{
switch (t.type)
{
@@ -73,8 +74,14 @@ pure string getTypeFromToken(const Token t)
}
}
-pure bool isIdentifierOrType(inout Token t)
+pure bool isIdentifierOrType(const Token t)
{
return t.type == TokenType.Identifier || (t.type > TokenType.TYPES_BEGIN
&& TokenType.TYPES_END);
}
+
+pure bool isDocComment(ref const Token t)
+{
+ return t.value.startsWith("///") || t.value.startsWith("/**")
+ || t.value.startsWith("/++");
+}
diff --git a/main.d b/main.d
index 848cd0a..0e77918 100644
--- a/main.d
+++ b/main.d
@@ -15,6 +15,7 @@ import std.parallelism;
import std.path;
import std.regex;
import std.stdio;
+import std.range;
import std.d.lexer;
import autocomplete;
@@ -28,7 +29,7 @@ import circularbuffer;
immutable size_t CIRC_BUFF_SIZE = 4;
-pure bool isLineOfCode(TokenType t)
+pure nothrow bool isLineOfCode(TokenType t)
{
switch(t)
{
@@ -138,36 +139,21 @@ int main(string[] args)
{
if (args.length == 1)
{
- auto f = appender!string();
- char[] buf;
- while (stdin.readln(buf))
- f.put(buf);
- writeln(f.data.byToken().count!(a => isLineOfCode(a.type))());
+ writeln(stdin.byLine(KeepTerminator.yes).join().byToken().count!(a => isLineOfCode(a.type))());
}
else
{
- writeln(args[1..$].map!(a => a.readText().byToken())().joiner()
- .count!(a => isLineOfCode(a.type))());
+ writeln(args[1..$].map!(a => File(a).byLine(KeepTerminator.yes).join().byToken())()
+ .joiner().count!(a => isLineOfCode(a.type))());
}
return 0;
}
if (highlight)
{
- if (args.length == 1)
- {
- auto f = appender!string();
- char[] buf;
- while (stdin.readln(buf))
- f.put(buf);
- highlighter.highlight(f.data.byToken(IterationStyle.Everything,
- StringStyle.Source));
- }
- else
- {
- highlighter.highlight(args[1].readText().byToken(
- IterationStyle.Everything, StringStyle.Source));
- }
+ File f = args.length == 1 ? stdin : File(args[1]);
+ highlighter.highlight(f.byLine(KeepTerminator.yes).join().byToken(
+ IterationStyle.Everything, StringStyle.Source));
return 0;
}
@@ -213,20 +199,9 @@ int main(string[] args)
if (json)
{
CircularBuffer!(Token) tokens;
- if (args.length == 1)
- {
- // Read from stdin
- auto f = appender!string();
- char[] buf;
- while (stdin.readln(buf))
- f.put(buf);
- tokens = new CircularBuffer!(Token)(CIRC_BUFF_SIZE, byToken!string(f.data));
- }
- else
- {
- // read given file
- tokens = new CircularBuffer!(Token)(CIRC_BUFF_SIZE, byToken!string(readText(args[1])));
- }
+ File f = args.length == 1 ? stdin : File(args[1]);
+ tokens = new CircularBuffer!(Token)(CIRC_BUFF_SIZE,
+ f.byLine(KeepTerminator.yes).join().byToken!(char[])());
auto mod = parseModule(tokens);
mod.writeJSONTo(stdout);
return 0;
diff --git a/parser.d b/parser.d
index 70498a8..55f94ba 100644
--- a/parser.d
+++ b/parser.d
@@ -26,6 +26,7 @@ public:
this(InputRange!Token tokens, TokenType open, TokenType close)
{
super(0, tokens);
+ this.range = tokens;
this.open = open;
this.close = close;
}
@@ -35,7 +36,7 @@ public:
return _empty;
}
- override Token front() const @property
+ override Token front() @property
{
return range.front;
}
@@ -47,14 +48,20 @@ public:
++depth;
else if (range.front == close)
--depth;
- _empty = depth == 0;
+ _empty = depth == 0 || range.empty;
}
+ invariant()
+ {
+ assert (range);
+ assert (depth >= 0);
+ }
+
private:
int depth;
TokenType open;
TokenType close;
- TokenBuffer range;
+ InputRange!(Token) range;
bool _empty;
}
diff --git a/std/d/lexer.d b/std/d/lexer.d
index 840b76e..1ac8019 100644
--- a/std/d/lexer.d
+++ b/std/d/lexer.d
@@ -1,71 +1,71 @@
// Written in the D programming language
/**
- * This module contains a range-based lexer for the D programming language.
- *
- * Examples:
- *
- * Generate HTML markup of D code.
- * ---
- * import std.stdio;
- * import std.array;
- * import std.file;
- * import std.d.lexer;
- *
- * void writeSpan(string cssClass, string value)
- * {
- * stdout.write(``, value.replace("&", "&").replace("<", "<"), ``);
- * }
- *
- * void highlight(R)(R tokens)
- * {
- * stdout.writeln(q"[
- *
- *
- *
- *
- *
- * ]");
- *
- * foreach (Token t; tokens)
- * {
- * if (t.type > TokenType.TYPES_BEGIN && t.type < TokenType.TYPES_END)
- * writeSpan("type", t.value);
- * else if (t.type > TokenType.KEYWORDS_BEGIN && t.type < TokenType.KEYWORDS_END)
- * writeSpan("kwrd", t.value);
- * else if (t.type == TokenType.Comment)
- * writeSpan("com", t.value);
- * else if (t.type > TokenType.STRINGS_BEGIN && t.type < TokenType.STRINGS_END)
- * writeSpan("str", t.value);
- * else if (t.type > TokenType.NUMBERS_BEGIN && t.type < TokenType.NUMBERS_END)
- * writeSpan("num", t.value);
- * else if (t.type > TokenType.OPERATORS_BEGIN && t.type < TokenType.OPERATORS_END)
- * writeSpan("op", t.value);
- * else
- * stdout.write(t.value.replace("<", "<"));
- * }
- * stdout.writeln("\n");
- * }
- *
- * void main(string[] args)
- * {
- * args[1].readText().byToken(IterationStyle.Everything, StringStyle.Source).highlight();
- * }
- * ---
- *
- * Copyright: Brian Schott 2013
- * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0)
- * Authors: Brian Schott
- * Source: $(PHOBOSSRC std/d/_lexer.d)
- */
+* This module contains a range-based lexer for the D programming language.
+*
+* Examples:
+*
+* Generate HTML markup of D code.
+* ---
+* import std.stdio;
+* import std.array;
+* import std.file;
+* import std.d.lexer;
+*
+* void writeSpan(string cssClass, string value)
+* {
+* stdout.write(``, value.replace("&", "&").replace("<", "<"), ``);
+* }
+*
+* void highlight(R)(R tokens)
+* {
+* stdout.writeln(q"[
+*
+*
+*
+*
+*
+* ]");
+*
+* foreach (Token t; tokens)
+* {
+* if (t.type > TokenType.TYPES_BEGIN && t.type < TokenType.TYPES_END)
+* writeSpan("type", t.value);
+* else if (t.type > TokenType.KEYWORDS_BEGIN && t.type < TokenType.KEYWORDS_END)
+* writeSpan("kwrd", t.value);
+* else if (t.type == TokenType.Comment)
+* writeSpan("com", t.value);
+* else if (t.type > TokenType.STRINGS_BEGIN && t.type < TokenType.STRINGS_END)
+* writeSpan("str", t.value);
+* else if (t.type > TokenType.NUMBERS_BEGIN && t.type < TokenType.NUMBERS_END)
+* writeSpan("num", t.value);
+* else if (t.type > TokenType.OPERATORS_BEGIN && t.type < TokenType.OPERATORS_END)
+* writeSpan("op", t.value);
+* else
+* stdout.write(t.value.replace("<", "<"));
+* }
+* stdout.writeln("\n");
+* }
+*
+* void main(string[] args)
+* {
+* args[1].readText().byToken(IterationStyle.Everything, StringStyle.Source).highlight();
+* }
+* ---
+*
+* Copyright: Brian Schott 2013
+* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0)
+* Authors: Brian Schott
+* Source: $(PHOBOSSRC std/d/_lexer.d)
+*/
module std.d.lexer;
@@ -81,8 +81,8 @@ import std.d.entities;
public:
/**
- * Represents a D token
- */
+* Represents a D token
+*/
struct Token
{
/// The token type.
@@ -98,28 +98,28 @@ struct Token
uint startIndex;
/**
- * Check to see if the token is of the same type and has the same string
- * representation as the given token.
- */
+ * Check to see if the token is of the same type and has the same string
+ * representation as the given token.
+ */
bool opEquals(ref const(Token) other) const
{
return other.type == type && other.value == value;
}
/**
- * Checks to see if the token's string representation is equal to the given
- * string.
- */
+ * Checks to see if the token's string representation is equal to the given
+ * string.
+ */
bool opEquals(string value) const { return this.value == value; }
/**
- * Checks to see if the token is of the given type.
- */
+ * Checks to see if the token is of the given type.
+ */
bool opEquals(TokenType type) const { return type == type; }
/**
- * Comparison operator orders tokens by start index.
- */
+ * Comparison operator orders tokens by start index.
+ */
int opCmp(size_t i) const
{
if (startIndex < i) return -1;
@@ -129,8 +129,8 @@ struct Token
}
/**
- * Configure the behavior of the byToken() function
- */
+* Configure the behavior of the byToken() function
+*/
enum IterationStyle
{
/// Only include code, not whitespace or comments
@@ -139,58 +139,59 @@ enum IterationStyle
IncludeComments = 0b0001,
/// Includes whitespace
IncludeWhitespace = 0b0010,
- /// Include $(LINK2 http://dlang.org/lex.html#specialtokens, special tokens)
- IncludeSpecialTokens = 0b0100,
+ /// Include $(LINK2 http://dlang.org/lex.html#specialtokens, special tokens)
+ IncludeSpecialTokens = 0b0100,
/// Include everything
Everything = IncludeComments | IncludeWhitespace
}
/**
- * Configuration of the string lexing style
- */
+* Configuration of the string lexing style
+*/
enum StringStyle : uint
{
/**
- * Escape sequences will be replaced with their equivalent characters,
- * enclosing quote characters will not be included. Useful for creating a
- * compiler or interpreter.
- */
+ * Escape sequences will be replaced with their equivalent characters,
+ * enclosing quote characters will not be included. Useful for creating a
+ * compiler or interpreter.
+ */
Default = 0b0000,
/**
- * Escape sequences will not be processed. An escaped quote character will
- * not terminate string lexing, but it will not be replaced with the quote
- * character in the token.
- */
+ * Escape sequences will not be processed. An escaped quote character will
+ * not terminate string lexing, but it will not be replaced with the quote
+ * character in the token.
+ */
NotEscaped = 0b0001,
/**
- * Strings will include their opening and closing quote characters as well
- * as any prefixes or suffixes $(LPAREN)e.g.: $(D_STRING "abcde"w) will
- * include the $(D_STRING 'w') character as well as the opening and closing
- * quotes$(RPAREN)
- */
+ * Strings will include their opening and closing quote characters as well
+ * as any prefixes or suffixes $(LPAREN)e.g.: $(D_STRING "abcde"w) will
+ * include the $(D_STRING 'w') character as well as the opening and closing
+ * quotes$(RPAREN)
+ */
IncludeQuotes = 0x0010,
/**
- * Strings will be read exactly as they appeared in the source, including
- * their opening and closing quote characters. Useful for syntax
- * highlighting.
- */
+ * Strings will be read exactly as they appeared in the source, including
+ * their opening and closing quote characters. Useful for syntax
+ * highlighting.
+ */
Source = NotEscaped | IncludeQuotes,
}
/**
- * Iterate over the given range of characters by D tokens.
- * Params:
- * range = the range of characters
- * iterationStyle = See IterationStyle
- * stringStyle = see StringStyle
- * Returns:
- * an input range of tokens
- */
+* Iterate over the given range of characters by D tokens.
+* Params:
+* range = the range of characters
+* iterationStyle = See IterationStyle
+* stringStyle = see StringStyle
+* Returns:
+* an input range of tokens
+*/
TokenRange!(R) byToken(R)(R range, const IterationStyle iterationStyle = IterationStyle.CodeOnly,
- const StringStyle stringStyle = StringStyle.Default) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
+ const StringStyle stringStyle = StringStyle.Default)
+ if (isForwardRange!(R) && (isSomeChar!(ElementType!(R)) || is (ElementType!(R) == ubyte)))
{
auto r = new TokenRange!(R)(range);
r.stringStyle = stringStyle;
@@ -201,9 +202,9 @@ TokenRange!(R) byToken(R)(R range, const IterationStyle iterationStyle = Iterati
}
/**
- * Range of tokens. Avoid creating instances of this manually. Use
- * $(DDOC_PSYMBOL byToken$(LPAREN)$(RPAREN)) instead, as it does some initialization work.
- */
+* Range of tokens. Avoid creating instances of this manually. Use
+* $(DDOC_PSYMBOL byToken$(LPAREN)$(RPAREN)) instead, as it does some initialization work.
+*/
class TokenRange(R) : InputRange!(Token)
{
this(ref R range)
@@ -212,16 +213,16 @@ class TokenRange(R) : InputRange!(Token)
}
/**
- * Returns: true if the range is empty
- */
+ * Returns: true if the range is empty
+ */
override bool empty() const @property
{
return _empty;
}
/**
- * Returns: the current token
- */
+ * Returns: the current token
+ */
override Token front() const @property
{
enforce(!_empty, "Cannot call front() on empty token range");
@@ -229,8 +230,8 @@ class TokenRange(R) : InputRange!(Token)
}
/**
- * Returns the current token and then removes it from the range
- */
+ * Returns the current token and then removes it from the range
+ */
override Token moveFront()
{
auto r = front();
@@ -265,38 +266,38 @@ class TokenRange(R) : InputRange!(Token)
return result;
}
- override void popFront()
- {
- // Filter out tokens we don't care about
- loop: do
- {
- advance();
- switch (current.type)
- {
- case TokenType.Comment:
- if (iterStyle & IterationStyle.IncludeComments)
- break loop;
- break;
- case TokenType.Whitespace:
- if (iterStyle & IterationStyle.IncludeWhitespace)
- break loop;
- break;
- case TokenType.SpecialTokenSequence:
- if (iterStyle & IterationStyle.IncludeSpecialTokens)
- break loop;
- break;
- default:
- break loop;
- }
- }
- while (!empty());
- }
+ override void popFront()
+ {
+ // Filter out tokens we don't care about
+ loop: do
+ {
+ advance();
+ switch (current.type)
+ {
+ case TokenType.Comment:
+ if (iterStyle & IterationStyle.IncludeComments)
+ break loop;
+ break;
+ case TokenType.Whitespace:
+ if (iterStyle & IterationStyle.IncludeWhitespace)
+ break loop;
+ break;
+ case TokenType.SpecialTokenSequence:
+ if (iterStyle & IterationStyle.IncludeSpecialTokens)
+ break loop;
+ break;
+ default:
+ break loop;
+ }
+ }
+ while (!empty());
+ }
private:
/*
- * Advances the range to the next token
- */
+ * Advances the range to the next token
+ */
void advance()
{
if (range.empty)
@@ -311,8 +312,8 @@ private:
if (std.uni.isWhite(range.front))
{
- current = lexWhitespace(range, index, lineNumber);
- return;
+ current = lexWhitespace(range, index, lineNumber);
+ return;
}
outer: switch (range.front)
{
@@ -456,22 +457,22 @@ private:
}
else
goto default;
- case '#':
- string special = lexSpecialTokenSequence(range, index, lineNumber);
- if (special)
- {
- current.type = TokenType.SpecialTokenSequence;
- current.value = special;
- }
- else
- {
- current.type = TokenType.Hash;
- current.value = "#";
- range.popFront();
+ case '#':
+ string special = lexSpecialTokenSequence(range, index, lineNumber);
+ if (special)
+ {
+ current.type = TokenType.SpecialTokenSequence;
+ current.value = special;
+ }
+ else
+ {
+ current.type = TokenType.Hash;
+ current.value = "#";
+ range.popFront();
++index;
break;
- }
- break;
+ }
+ break;
default:
auto app = appender!(ElementType!(R)[])();
while(!range.isEoF() && !isSeparating(range.front))
@@ -497,38 +498,38 @@ private:
unittest
{
- import std.stdio;
- auto a = "/**comment*/\n#lin #line 10 \"test.d\"\nint a;//test\n";
- foreach (t; byToken(a))
- writeln(t);
+ import std.stdio;
+ auto a = "/**comment*/\n#lin #line 10 \"test.d\"\nint a;//test\n";
+ foreach (t; byToken(a))
+ writeln(t);
}
/**
- * Listing of all the tokens in the D language.
- *
- * Token types are arranged so that it is easy to group tokens while iterating
- * over them. For example:
- * ---
- * assert(TokenType.Increment < TokenType.OPERATORS_END);
- * assert(TokenType.Increment > TokenType.OPERATORS_BEGIN);
- * ---
- * The non-token values are documented below:
- *
- * $(BOOKTABLE ,
- * $(TR $(TH Begin) $(TH End) $(TH Content) $(TH Examples))
- * $(TR $(TD OPERATORS_BEGIN) $(TD OPERATORS_END) $(TD operatiors) $(TD +, -, <<=))
- * $(TR $(TD TYPES_BEGIN) $(TD TYPES_END) $(TD types) $(TD bool, char, double))
- * $(TR $(TD KEYWORDS_BEGIN) $(TD KEYWORDS) $(TD keywords) $(TD class, if, assert))
- * $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD attributes) $(TD override synchronized, __gshared))
- * $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD protection) $(TD public, protected))
- * $(TR $(TD CONSTANTS_BEGIN) $(TD CONSTANTS_END) $(TD compile-time constants) $(TD __FILE__, __TIME__))
- * $(TR $(TD LITERALS_BEGIN) $(TD LITERALS_END) $(TD string and numeric literals) $(TD "str", 123))
- * $(TR $(TD NUMBERS_BEGIN) $(TD NUMBERS_END) $(TD numeric literals) $(TD 0x123p+9, 0b0110))
- * $(TR $(TD STRINGS_BEGIN) $(TD STRINGS_END) $(TD string literals) $(TD `123`c, q{tokens;}, "abcde"))
- * $(TR $(TD MISC_BEGIN) $(TD MISC_END) $(TD anything else) $(TD whitespace, comments, identifiers))
- * )
- * Note that several of the above ranges overlap.
- */
+* Listing of all the tokens in the D language.
+*
+* Token types are arranged so that it is easy to group tokens while iterating
+* over them. For example:
+* ---
+* assert(TokenType.Increment < TokenType.OPERATORS_END);
+* assert(TokenType.Increment > TokenType.OPERATORS_BEGIN);
+* ---
+* The non-token values are documented below:
+*
+* $(BOOKTABLE ,
+* $(TR $(TH Begin) $(TH End) $(TH Content) $(TH Examples))
+* $(TR $(TD OPERATORS_BEGIN) $(TD OPERATORS_END) $(TD operatiors) $(TD +, -, <<=))
+* $(TR $(TD TYPES_BEGIN) $(TD TYPES_END) $(TD types) $(TD bool, char, double))
+* $(TR $(TD KEYWORDS_BEGIN) $(TD KEYWORDS) $(TD keywords) $(TD class, if, assert))
+* $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD attributes) $(TD override synchronized, __gshared))
+* $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD protection) $(TD public, protected))
+* $(TR $(TD CONSTANTS_BEGIN) $(TD CONSTANTS_END) $(TD compile-time constants) $(TD __FILE__, __TIME__))
+* $(TR $(TD LITERALS_BEGIN) $(TD LITERALS_END) $(TD string and numeric literals) $(TD "str", 123))
+* $(TR $(TD NUMBERS_BEGIN) $(TD NUMBERS_END) $(TD numeric literals) $(TD 0x123p+9, 0b0110))
+* $(TR $(TD STRINGS_BEGIN) $(TD STRINGS_END) $(TD string literals) $(TD `123`c, q{tokens;}, "abcde"))
+* $(TR $(TD MISC_BEGIN) $(TD MISC_END) $(TD anything else) $(TD whitespace, comments, identifiers))
+* )
+* Note that several of the above ranges overlap.
+*/
enum TokenType: uint
{
// Operators
@@ -599,38 +600,38 @@ enum TokenType: uint
OPERATORS_END, ///
- // Keywords
+ // Keywords
KEYWORDS_BEGIN, ///
- TYPES_BEGIN, ///
- Bool, /// $(D_KEYWORD bool)
- Byte, /// $(D_KEYWORD byte)
- Cdouble, /// $(D_KEYWORD cdouble)
- Cent, /// $(D_KEYWORD cent)
- Cfloat, /// $(D_KEYWORD cfloat)
- Char, /// $(D_KEYWORD char)
- Creal, /// $(D_KEYWORD creal)
- Dchar, /// $(D_KEYWORD dchar)
- Double, /// $(D_KEYWORD double)
- DString, /// $(D_KEYWORD dstring)
- Float, /// $(D_KEYWORD float)
- Function, /// $(D_KEYWORD function)
- Idouble, /// $(D_KEYWORD idouble)
- Ifloat, /// $(D_KEYWORD ifloat)
- Int, /// $(D_KEYWORD int)
- Ireal, /// $(D_KEYWORD ireal)
- Long, /// $(D_KEYWORD long)
- Real, /// $(D_KEYWORD real)
- Short, /// $(D_KEYWORD short)
- String, /// $(D_KEYWORD string)
- Ubyte, /// $(D_KEYWORD ubyte)
- Ucent, /// $(D_KEYWORD ucent)
- Uint, /// $(D_KEYWORD uint)
- Ulong, /// $(D_KEYWORD ulong)
- Ushort, /// $(D_KEYWORD ushort)
- Void, /// $(D_KEYWORD void)
- Wchar, /// $(D_KEYWORD wchar)
- WString, /// $(D_KEYWORD wstring)
- TYPES_END, ///
+ TYPES_BEGIN, ///
+ Bool, /// $(D_KEYWORD bool)
+ Byte, /// $(D_KEYWORD byte)
+ Cdouble, /// $(D_KEYWORD cdouble)
+ Cent, /// $(D_KEYWORD cent)
+ Cfloat, /// $(D_KEYWORD cfloat)
+ Char, /// $(D_KEYWORD char)
+ Creal, /// $(D_KEYWORD creal)
+ Dchar, /// $(D_KEYWORD dchar)
+ Double, /// $(D_KEYWORD double)
+ DString, /// $(D_KEYWORD dstring)
+ Float, /// $(D_KEYWORD float)
+ Function, /// $(D_KEYWORD function)
+ Idouble, /// $(D_KEYWORD idouble)
+ Ifloat, /// $(D_KEYWORD ifloat)
+ Int, /// $(D_KEYWORD int)
+ Ireal, /// $(D_KEYWORD ireal)
+ Long, /// $(D_KEYWORD long)
+ Real, /// $(D_KEYWORD real)
+ Short, /// $(D_KEYWORD short)
+ String, /// $(D_KEYWORD string)
+ Ubyte, /// $(D_KEYWORD ubyte)
+ Ucent, /// $(D_KEYWORD ucent)
+ Uint, /// $(D_KEYWORD uint)
+ Ulong, /// $(D_KEYWORD ulong)
+ Ushort, /// $(D_KEYWORD ushort)
+ Void, /// $(D_KEYWORD void)
+ Wchar, /// $(D_KEYWORD wchar)
+ WString, /// $(D_KEYWORD wstring)
+ TYPES_END, ///
ATTRIBUTES_BEGIN, ///
Align, /// $(D_KEYWORD align)
Deprecated, /// $(D_KEYWORD deprecated)
@@ -699,7 +700,7 @@ enum TokenType: uint
Struct, /// $(D_KEYWORD struct)
Super, /// $(D_KEYWORD super)
Switch, /// $(D_KEYWORD switch)
- Template, /// $(D_KEYWORD template)
+ Template, /// $(D_KEYWORD template)
This, /// $(D_KEYWORD this)
Throw, /// $(D_KEYWORD throw)
True, /// $(D_KEYWORD true)
@@ -729,7 +730,7 @@ enum TokenType: uint
Identifier, /// anything else
ScriptLine, // Line at the beginning of source file that starts from #!
Whitespace, /// whitespace
- SpecialTokenSequence, /// #line 10 "file.d"
+ SpecialTokenSequence, /// #line 10 "file.d"
MISC_END, ///
// Literals
@@ -1122,53 +1123,53 @@ Token lexHexString(R, C = ElementType!R)(ref R input, ref uint index, ref uint l
const StringStyle style = StringStyle.Default)
in
{
- assert (input.front == 'x');
+ assert (input.front == 'x');
}
body
{
- Token t;
- t.lineNumber = lineNumber;
- t.startIndex = index;
- t.type = TokenType.StringLiteral;
- auto app = appender!(C[])();
- if (style & StringStyle.IncludeQuotes)
- app.put("x\"");
- input.popFront();
- input.popFront();
- index += 2;
- while (!input.isEoF())
- {
- if (isNewline(input))
- {
- app.put(popNewline(input, index));
- ++lineNumber;
- }
- else if (isHexDigit(input.front))
- {
- app.put(input.front);
- input.popFront();
- ++index;
- }
- else if (std.uni.isWhite(input.front) && (style & StringStyle.NotEscaped))
- {
- app.put(input.front);
- input.popFront();
- ++index;
- }
- else if (input.front == '"')
- {
- if (style & StringStyle.IncludeQuotes)
- app.put('"');
- input.popFront();
- ++index;
- break;
- }
- else
- {
- // This is an error
- }
- }
- if (!input.isEoF())
+ Token t;
+ t.lineNumber = lineNumber;
+ t.startIndex = index;
+ t.type = TokenType.StringLiteral;
+ auto app = appender!(C[])();
+ if (style & StringStyle.IncludeQuotes)
+ app.put("x\"");
+ input.popFront();
+ input.popFront();
+ index += 2;
+ while (!input.isEoF())
+ {
+ if (isNewline(input))
+ {
+ app.put(popNewline(input, index));
+ ++lineNumber;
+ }
+ else if (isHexDigit(input.front))
+ {
+ app.put(input.front);
+ input.popFront();
+ ++index;
+ }
+ else if (std.uni.isWhite(input.front) && (style & StringStyle.NotEscaped))
+ {
+ app.put(input.front);
+ input.popFront();
+ ++index;
+ }
+ else if (input.front == '"')
+ {
+ if (style & StringStyle.IncludeQuotes)
+ app.put('"');
+ input.popFront();
+ ++index;
+ break;
+ }
+ else
+ {
+ // This is an error
+ }
+ }
+ if (!input.isEoF())
{
switch (input.front)
{
@@ -1188,43 +1189,43 @@ body
break;
}
}
- if (style & StringStyle.NotEscaped)
+ if (style & StringStyle.NotEscaped)
t.value = to!string(app.data);
else
- {
- auto a = appender!(char[])();
- foreach (b; std.range.chunks(app.data, 2))
- a.put(to!string(cast(dchar) parse!uint(b, 16)));
- t.value = to!string(a.data);
- }
+ {
+ auto a = appender!(char[])();
+ foreach (b; std.range.chunks(app.data, 2))
+ a.put(to!string(cast(dchar) parse!uint(b, 16)));
+ t.value = to!string(a.data);
+ }
- return t;
+ return t;
}
unittest
{
- uint i;
- uint l;
+ uint i;
+ uint l;
- auto a = `x"204041"`;
- auto ar = lexHexString(a, i, l);
- assert (ar == " @A");
- assert (ar == TokenType.StringLiteral);
+ auto a = `x"204041"`;
+ auto ar = lexHexString(a, i, l);
+ assert (ar == " @A");
+ assert (ar == TokenType.StringLiteral);
- auto b = `x"20"w`;
- auto br = lexHexString(b, i, l);
- assert (br == " ");
- assert (br == TokenType.WStringLiteral);
+ auto b = `x"20"w`;
+ auto br = lexHexString(b, i, l);
+ assert (br == " ");
+ assert (br == TokenType.WStringLiteral);
- auto c = `x"6d"`;
- auto cr = lexHexString(c, i, l, StringStyle.NotEscaped);
- assert (cr == "6d");
+ auto c = `x"6d"`;
+ auto cr = lexHexString(c, i, l, StringStyle.NotEscaped);
+ assert (cr == "6d");
- auto d = `x"5e5f"d`;
- auto dr = lexHexString(d, i, l, StringStyle.NotEscaped | StringStyle.IncludeQuotes);
- assert (dr == `x"5e5f"d`);
- assert (dr == TokenType.DStringLiteral);
+ auto d = `x"5e5f"d`;
+ auto dr = lexHexString(d, i, l, StringStyle.NotEscaped | StringStyle.IncludeQuotes);
+ assert (dr == `x"5e5f"d`);
+ assert (dr == TokenType.DStringLiteral);
}
Token lexString(R)(ref R input, ref uint index, ref uint lineNumber,
@@ -1582,7 +1583,7 @@ body
unittest
{
- import std.stdio;
+ import std.stdio;
uint i;
uint l;
auto a = "q{import std.stdio;} abcd";
@@ -2178,106 +2179,106 @@ unittest
}
string lexSpecialTokenSequence(R)(ref R input, ref uint index,
- ref uint lineNumber)
+ ref uint lineNumber)
in
{
- assert (input.front == '#');
+ assert (input.front == '#');
}
body
{
- auto i = index;
- auto r = input.save;
- auto l = lineNumber;
- r.popFront();
- ++i;
- auto app = appender!(ElementType!(R)[])();
- app.put('#');
+ auto i = index;
+ auto r = input.save;
+ auto l = lineNumber;
+ r.popFront();
+ ++i;
+ auto app = appender!(ElementType!(R)[])();
+ app.put('#');
- auto specialType = appender!(ElementType!(R)[])();
+ auto specialType = appender!(ElementType!(R)[])();
- while (!r.empty && !isSeparating(r.front))
- {
- specialType.put(r.front);
- ++i;
- r.popFront();
- }
+ while (!r.empty && !isSeparating(r.front))
+ {
+ specialType.put(r.front);
+ ++i;
+ r.popFront();
+ }
- if (to!string(specialType.data) != "line")
- return null;
- app.put(specialType.data);
+ if (to!string(specialType.data) != "line")
+ return null;
+ app.put(specialType.data);
- if (std.uni.isWhite(r.front))
- app.put(lexWhitespace(r, i, l).value);
+ if (std.uni.isWhite(r.front))
+ app.put(lexWhitespace(r, i, l).value);
- if (!isDigit(r.front))
- return null;
+ if (!isDigit(r.front))
+ return null;
- auto t = lexNumber(r, i, l);
- if (t != TokenType.IntLiteral)
- return null;
+ auto t = lexNumber(r, i, l);
+ if (t != TokenType.IntLiteral)
+ return null;
- app.put(t.value);
- l = to!uint(t.value);
+ app.put(t.value);
+ l = to!uint(t.value);
- if (!isNewline(r))
- {
- if (!r.empty && std.uni.isWhite(r.front))
- app.put(lexWhitespace(r, i, l).value);
+ if (!isNewline(r))
+ {
+ if (!r.empty && std.uni.isWhite(r.front))
+ app.put(lexWhitespace(r, i, l).value);
- if (!r.empty && r.front == '"')
- {
- auto fSpecApp = appender!(ElementType!(R)[])();
- fSpecApp.put(r.front);
- r.popFront();
- ++i;
- while (!r.empty)
- {
- if (r.front == '"')
- {
- fSpecApp.put('"');
- ++i;
- r.popFront();
- break;
- }
- ++i;
- fSpecApp.put(r.front);
- r.popFront();
- }
- app.put(fSpecApp.data);
- }
- else
- return null;
- }
+ if (!r.empty && r.front == '"')
+ {
+ auto fSpecApp = appender!(ElementType!(R)[])();
+ fSpecApp.put(r.front);
+ r.popFront();
+ ++i;
+ while (!r.empty)
+ {
+ if (r.front == '"')
+ {
+ fSpecApp.put('"');
+ ++i;
+ r.popFront();
+ break;
+ }
+ ++i;
+ fSpecApp.put(r.front);
+ r.popFront();
+ }
+ app.put(fSpecApp.data);
+ }
+ else
+ return null;
+ }
- app.put(popNewline(r, i));
- input.popFrontN(i - index);
- index = i;
- lineNumber = l;
- return to!string(app.data);
+ app.put(popNewline(r, i));
+ input.popFrontN(i - index);
+ index = i;
+ lineNumber = l;
+ return to!string(app.data);
}
unittest
{
- uint i;
- uint l;
- auto a = "#line 10\n";
- auto ar = lexSpecialTokenSequence(a, i, l);
- assert (ar == "#line 10\n");
- assert (a == "");
- assert (l == 10);
+ uint i;
+ uint l;
+ auto a = "#line 10\n";
+ auto ar = lexSpecialTokenSequence(a, i, l);
+ assert (ar == "#line 10\n");
+ assert (a == "");
+ assert (l == 10);
- auto b = "#line 9201 \"test.d\"\n";
- auto br = lexSpecialTokenSequence(b, i, l);
- assert (l == 9201);
- assert (br == "#line 9201 \"test.d\"\n");
- assert (b == "");
+ auto b = "#line 9201 \"test.d\"\n";
+ auto br = lexSpecialTokenSequence(b, i, l);
+ assert (l == 9201);
+ assert (br == "#line 9201 \"test.d\"\n");
+ assert (b == "");
- auto c = `#lin`;
- auto cr = lexSpecialTokenSequence(c, i, l);
- assert (l == 9201);
- assert (cr is null);
- assert (c == `#lin`);
+ auto c = `#lin`;
+ auto cr = lexSpecialTokenSequence(c, i, l);
+ assert (l == 9201);
+ assert (cr is null);
+ assert (c == `#lin`);
}
pure nothrow bool isSeparating(C)(C ch) if (isSomeChar!C)
@@ -2477,8 +2478,8 @@ pure nothrow TokenType lookupTokenType(const string input)
class Trie(K, V) if (isInputRange!K): TrieNode!(K, V)
{
/**
- * Adds the given value to the trie with the given key
- */
+ * Adds the given value to the trie with the given key
+ */
void add(K key, V value) pure
{
TrieNode!(K,V) current = this;