forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/Hackerpilot/Dscanner

This commit is contained in:
Hackerpilot 2014-05-08 02:45:07 -07:00
parent 22bacd83ce d97a2c5f34
commit 017b90e31f
31 changed files with 1036 additions and 423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@ -19,7 +19,7 @@ The following examples assume that we are analyzing a simple file called hellowo
### Token Count ### Token Count
The "--tokenCount" or "-t" option prints the number of tokens in the given file The "--tokenCount" or "-t" option prints the number of tokens in the given file
$ dscanner --tokencount helloworld.d $ dscanner --tokenCount helloworld.d
20 20
### Import Listing ### Import Listing
@ -52,13 +52,15 @@ given source files.
* Left side of a *foreach* or *foreach\_reverse* range expression is larger than the right. * Left side of a *foreach* or *foreach\_reverse* range expression is larger than the right.
* Left side of a slice expression is larger than the right * Left side of a slice expression is larger than the right
* Variable, struct, class, union, module, package, and interface names that do not comply with Phobos style guidelines * Variable, struct, class, union, module, package, and interface names that do not comply with Phobos style guidelines
* Struct constructors that have a single parameter that has a default argument.
* Assign expressions where the left side of the '=' operator is the same as the right
* 'if' statements where the 'else' block is the same as the 'if' block.
#### Wishlish #### Wishlish
* Assigning to foreach variables that are not "ref". * Assigning to foreach variables that are not "ref".
* Unused variables. * Unused variables.
* Unused imports. * Unused imports.
* Unused parameters (check is skipped if function is marked "override") * Unused parameters (check is skipped if function is marked "override")
* Struct constructors that have a single parameter that has a default argument.
* Variables that are never modified and not declared immutable. * Variables that are never modified and not declared immutable.
* Public declarations not documented * Public declarations not documented
* Declaring opEquals without toHash * Declaring opEquals without toHash
@ -196,6 +198,6 @@ but not yet implemented.
# Useful code # Useful code
The source code for DScanner has a complete lexer, parser, and abstact syntax The source code for DScanner has a complete lexer, parser, and abstact syntax
tree library for D code under the stdx/d/ directory. It is intended that these tree library for D code under the std/d/ directory. It is intended that these
modules eventually end up in Phobos, so feel free to use them for your own D modules eventually end up in Phobos, so feel free to use them for your own D
tools. tools.

2
analysis/base.d
View File

@ -2,7 +2,7 @@ module analysis.base;
import std.container; import std.container;
import std.string; import std.string;
import stdx.d.ast; import std.d.ast;
import std.array; import std.array;
struct Message struct Message

86
analysis/constructors.d Normal file
View File

@ -0,0 +1,86 @@
module analysis.constructors;
import std.d.ast;
import std.d.lexer;
import analysis.base;
class ConstructorCheck : BaseAnalyzer
{
alias visit = BaseAnalyzer.visit;
this(string fileName)
{
super(fileName);
}
override void visit(const ClassDeclaration classDeclaration)
{
bool oldHasDefault = hasDefaultArgConstructor;
bool oldHasNoArg = hasNoArgConstructor;
hasNoArgConstructor = false;
hasDefaultArgConstructor = false;
State prev = state;
state = State.inClass;
classDeclaration.accept(this);
if (hasNoArgConstructor && hasDefaultArgConstructor)
{
addErrorMessage(classDeclaration.name.line,
classDeclaration.name.column, "This class has a zero-argument"
~ " constructor as well as a constructor with one default"
~ " argument. This can be confusing");
}
hasDefaultArgConstructor = oldHasDefault;
hasNoArgConstructor = oldHasNoArg;
state = prev;
}
override void visit(const StructDeclaration structDeclaration)
{
State prev = state;
state = State.inStruct;
structDeclaration.accept(this);
state = prev;
}
override void visit(const Constructor constructor)
{
final switch (state)
{
case State.inStruct:
if (constructor.parameters.parameters.length == 1
&& constructor.parameters.parameters[0].default_ !is null)
{
addErrorMessage(constructor.line, constructor.column,
"This struct constructor can never be called with its "
~ "default argument.");
}
break;
case State.inClass:
if (constructor.parameters.parameters.length == 1
&& constructor.parameters.parameters[0].default_ !is null)
{
hasDefaultArgConstructor = true;
}
else if (constructor.parameters.parameters.length == 0)
hasNoArgConstructor = true;
break;
case State.ignoring:
break;
}
}
private:
enum State: ubyte
{
ignoring,
inClass,
inStruct
}
State state;
bool hasNoArgConstructor;
bool hasDefaultArgConstructor;
}

4
analysis/del.d
View File

@ -5,8 +5,8 @@
module analysis.del; module analysis.del;
import stdx.d.ast; import std.d.ast;
import stdx.d.lexer; import std.d.lexer;
import analysis.base; import analysis.base;
/** /**

4
analysis/enumarrayliteral.d
View File

@ -5,8 +5,8 @@
module analysis.enumarrayliteral; module analysis.enumarrayliteral;
import stdx.d.ast; import std.d.ast;
import stdx.d.lexer; import std.d.lexer;
import analysis.base; import analysis.base;
void doNothing(string, size_t, size_t, string, bool) {} void doNothing(string, size_t, size_t, string, bool) {}

5
analysis/fish.d
View File

@ -5,8 +5,8 @@
module analysis.fish; module analysis.fish;
import stdx.d.ast; import std.d.ast;
import stdx.d.lexer; import std.d.lexer;
import analysis.base; import analysis.base;
/** /**
@ -24,6 +24,7 @@ class FloatOperatorCheck : BaseAnalyzer
override void visit(const RelExpression r) override void visit(const RelExpression r)
{ {
if (r.operator == tok!"<>" if (r.operator == tok!"<>"
|| r.operator == tok!"<>="
|| r.operator == tok!"!<>" || r.operator == tok!"!<>"
|| r.operator == tok!"!>" || r.operator == tok!"!>"
|| r.operator == tok!"!<" || r.operator == tok!"!<"

43
analysis/ifelsesame.d Normal file
View File

@ -0,0 +1,43 @@
// Copyright Brian Schott (Sir Alaran) 2014.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
module analysis.ifelsesame;
import std.d.ast;
import std.d.lexer;
import analysis.base;
/**
* Checks for if statements whose "then" block is the same as the "else" block
*/
class IfElseSameCheck : BaseAnalyzer
{
alias visit = BaseAnalyzer.visit;
this(string fileName)
{
super(fileName);
}
override void visit(const IfStatement ifStatement)
{
if (ifStatement.thenStatement == ifStatement.elseStatement)
addErrorMessage(ifStatement.line, ifStatement.column,
"\"Else\" branch is identical to \"Then\" branch.");
ifStatement.accept(this);
}
override void visit(const AssignExpression assignExpression)
{
const AssignExpression e = cast(const AssignExpression) assignExpression.assignExpression;
if (e !is null && assignExpression.operator == tok!"="
&& e.ternaryExpression == assignExpression.ternaryExpression)
{
addErrorMessage(assignExpression.line, assignExpression.column,
"Left side of assignment operatior is identical to the right side");
}
assignExpression.accept(this);
}
}

1
analysis/linespan.d
View File

@ -5,7 +5,6 @@
module analysis.linespan; module analysis.linespan;
/** /**
* Used for determining which lines to include as context in the generated HTML * Used for determining which lines to include as context in the generated HTML
* report. * report.

4
analysis/numbers.d
View File

@ -6,8 +6,8 @@
module analysis.numbers; module analysis.numbers;
import std.regex; import std.regex;
import stdx.d.ast; import std.d.ast;
import stdx.d.lexer; import std.d.lexer;
import analysis.base; import analysis.base;
/** /**

13
analysis/objectconst.d
View File

@ -6,12 +6,13 @@
module analysis.objectconst; module analysis.objectconst;
import std.regex; import std.regex;
import stdx.d.ast; import std.d.ast;
import stdx.d.lexer; import std.d.lexer;
import analysis.base; import analysis.base;
/** /**
* Checks for use of the deprecated "delete" keyword * Checks that opEquals, opCmp, toHash, and toString are either const,
* immutable, or inout.
*/ */
class ObjectConstCheck : BaseAnalyzer class ObjectConstCheck : BaseAnalyzer
{ {
@ -35,7 +36,7 @@ class ObjectConstCheck : BaseAnalyzer
&& !hasConst(d.functionDeclaration.memberFunctionAttributes))) && !hasConst(d.functionDeclaration.memberFunctionAttributes)))
{ {
addErrorMessage(d.functionDeclaration.name.line, addErrorMessage(d.functionDeclaration.name.line,
d.functionDeclaration.name.column, "opCmp, ToHash, opEquals," d.functionDeclaration.name.column, "opCmp, toHash, opEquals,"
~ " and toString should be declared const"); ~ " and toString should be declared const");
} }
d.accept(this); d.accept(this);
@ -51,7 +52,9 @@ class ObjectConstCheck : BaseAnalyzer
private static bool hasConst(const MemberFunctionAttribute[] attributes) private static bool hasConst(const MemberFunctionAttribute[] attributes)
{ {
import std.algorithm; import std.algorithm;
return attributes.any!(a => a.tokenType == tok!"const"); return attributes.any!(a => a.tokenType == tok!"const"
|| a.tokenType == tok!"immutable"
|| a.tokenType == tok!"inout");
} }
private static bool isInteresting(string name) private static bool isInteresting(string name)

4
analysis/pokemon.d
View File

@ -5,8 +5,8 @@
module analysis.pokemon; module analysis.pokemon;
import stdx.d.ast; import std.d.ast;
import stdx.d.lexer; import std.d.lexer;
import analysis.base; import analysis.base;
/** /**

25
analysis/range.d
View File

@ -5,8 +5,8 @@
module analysis.range; module analysis.range;
import stdx.d.ast; import std.d.ast;
import stdx.d.lexer; import std.d.lexer;
import analysis.base; import analysis.base;
/** /**
@ -64,24 +64,37 @@ class BackwardsRangeCheck : BaseAnalyzer
override void visit(const PrimaryExpression primary) override void visit(const PrimaryExpression primary)
{ {
import std.conv;
import std.string;
if (state == State.ignore || !isNumberLiteral(primary.primary.type)) if (state == State.ignore || !isNumberLiteral(primary.primary.type))
return; return;
if (state == State.left) if (state == State.left)
{ {
line = primary.primary.line; line = primary.primary.line;
this.column = primary.primary.column; this.column = primary.primary.column;
left = to!long(primary.primary.text.removechars("_uUlL")); left = parseNumber(primary.primary.text);
hasLeft = true; hasLeft = true;
} }
else else
{ {
right = to!long(primary.primary.text.removechars("_uUlL")); right = parseNumber(primary.primary.text);
hasRight = true; hasRight = true;
} }
} }
long parseNumber(string te)
{
import std.conv;
import std.string;
string t = te.removechars("_uUlL");
if (t.length > 2)
{
if (t[1] == 'x' || t[1] == 'X')
return to!long(t[2..$], 16);
if (t[1] == 'b' || t[1] == 'B')
return to!long(t[2..$], 2);
}
return to!long(t);
}
override void visit(const SliceExpression sliceExpression) override void visit(const SliceExpression sliceExpression)
{ {
if (sliceExpression.lower !is null && sliceExpression.upper !is null) if (sliceExpression.lower !is null && sliceExpression.upper !is null)

11
analysis/run.d
View File

@ -6,10 +6,9 @@ import std.conv;
import std.algorithm; import std.algorithm;
import std.range; import std.range;
import std.array; import std.array;
import std.d.lexer;
import stdx.d.lexer; import std.d.parser;
import stdx.d.parser; import std.d.ast;
import stdx.d.ast;
import analysis.base; import analysis.base;
import analysis.style; import analysis.style;
@ -20,6 +19,8 @@ import analysis.fish;
import analysis.numbers; import analysis.numbers;
import analysis.objectconst; import analysis.objectconst;
import analysis.range; import analysis.range;
import analysis.constructors;
import analysis.ifelsesame;
void messageFunction(string fileName, size_t line, size_t column, string message, void messageFunction(string fileName, size_t line, size_t column, string message,
bool isError) bool isError)
@ -70,6 +71,8 @@ void analyze(File output, string[] fileNames, bool staticAnalyze = true)
checks ~= new NumberStyleCheck(fileName); checks ~= new NumberStyleCheck(fileName);
checks ~= new ObjectConstCheck(fileName); checks ~= new ObjectConstCheck(fileName);
checks ~= new BackwardsRangeCheck(fileName); checks ~= new BackwardsRangeCheck(fileName);
checks ~= new IfElseSameCheck(fileName);
checks ~= new ConstructorCheck(fileName);
foreach (check; checks) foreach (check; checks)
{ {

4
analysis/style.d
View File

@ -5,8 +5,8 @@
module analysis.style; module analysis.style;
import stdx.d.ast; import std.d.ast;
import stdx.d.lexer; import std.d.lexer;
import std.regex; import std.regex;
import std.array; import std.array;
import std.conv; import std.conv;

19
astprinter.d
View File

@ -3,8 +3,8 @@
// (See accompanying file LICENSE_1_0.txt or copy at // (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt) // http://www.boost.org/LICENSE_1_0.txt)
import stdx.d.lexer; import std.d.lexer;
import stdx.d.ast; import std.d.ast;
import std.stdio; import std.stdio;
import std.string; import std.string;
import std.array; import std.array;
@ -133,7 +133,7 @@ class XMLPrinter : ASTVisitor
output.writeln("<assignExpression>"); output.writeln("<assignExpression>");
else else
output.writeln("<assignExpression operator=\"", output.writeln("<assignExpression operator=\"",
str(assignExpression.operator), "\">"); xmlEscape(str(assignExpression.operator)), "\">");
assignExpression.accept(this); assignExpression.accept(this);
output.writeln("</assignExpression>"); output.writeln("</assignExpression>");
} }
@ -479,7 +479,7 @@ class XMLPrinter : ASTVisitor
override void visit(const ForStatement forStatement) override void visit(const ForStatement forStatement)
{ {
output.writeln("<forStatement>"); output.writeln("<forStatement>");
if (forStatement.declarationOrStatement !is null) if (forStatement.initialization !is null)
{ {
output.writeln("<initialization>"); output.writeln("<initialization>");
visit(forStatement.initialization); visit(forStatement.initialization);
@ -497,6 +497,7 @@ class XMLPrinter : ASTVisitor
visit(forStatement.increment); visit(forStatement.increment);
output.writeln("</increment>"); output.writeln("</increment>");
} }
if (forStatement.declarationOrStatement !is null)
visit(forStatement.declarationOrStatement); visit(forStatement.declarationOrStatement);
output.writeln("</forStatement>"); output.writeln("</forStatement>");
} }
@ -592,6 +593,7 @@ class XMLPrinter : ASTVisitor
{ {
output.writeln("<gotoStatement>"); output.writeln("<gotoStatement>");
output.writeln("<case>"); output.writeln("<case>");
if (gotoStatement.expression)
visit(gotoStatement.expression); visit(gotoStatement.expression);
output.writeln("</case>"); output.writeln("</case>");
output.writeln("</gotoStatement>"); output.writeln("</gotoStatement>");
@ -665,10 +667,10 @@ class XMLPrinter : ASTVisitor
override void visit(const ImportBind importBind) override void visit(const ImportBind importBind)
{ {
if (importBind.right.type == tok!"") if (importBind.right.type == tok!"")
output.writeln("<importBind symbol=\"", importBind.left.text, "\">"); output.writeln("<importBind symbol=\"", importBind.left.text, "\"/>");
else else
output.writeln("<importBind symbol=\"", importBind.right.text, output.writeln("<importBind symbol=\"", importBind.right.text,
"\" rename=\"", importBind.left.text, "\">"); "\" rename=\"", importBind.left.text, "\"/>");
} }
override void visit(const ImportBindings importBindings) override void visit(const ImportBindings importBindings)
@ -779,7 +781,7 @@ class XMLPrinter : ASTVisitor
output.writeln("</key>"); output.writeln("</key>");
output.writeln("<value>"); output.writeln("<value>");
visit(keyValuePair.value); visit(keyValuePair.value);
output.writeln("<value>"); output.writeln("</value>");
output.writeln("</keyValuePair>"); output.writeln("</keyValuePair>");
} }
@ -1312,8 +1314,9 @@ class XMLPrinter : ASTVisitor
case tok!"stringLiteral": tagName = "stringLiteral"; break; case tok!"stringLiteral": tagName = "stringLiteral"; break;
case tok!"dstringLiteral": tagName = "dstringLiteral"; break; case tok!"dstringLiteral": tagName = "dstringLiteral"; break;
case tok!"wstringLiteral": tagName = "wstringLiteral"; break; case tok!"wstringLiteral": tagName = "wstringLiteral"; break;
case tok!"scriptLine": tagName = "scriptLine"; break;
case tok!"$": output.writeln("<dollar/>"); return; case tok!"$": output.writeln("<dollar/>"); return;
default: output.writeln("<", str(token.type), "/>"); return; default: tagName = "token"; break;
} }
output.writeln("<", tagName, ">", xmlEscape(token.text), "</", tagName, ">"); output.writeln("<", tagName, ">", xmlEscape(token.text), "</", tagName, ">");
} }

2
build.bat
View File

@ -1 +1 @@
dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/allocator.d stdx/lexer.d stdx/d/ast.d stdx/d/parser.d stdx/d/lexer.d analysis/base.d analysis/del.d analysis/enumarrayliteral.d analysis/fish.d analysis/numbers.d analysis/objectconst.d analysis/package.d analysis/pokemon.d analysis/range.d analysis/run.d analysis/style.d -ofdscanner.exe -O -release -noboundscheck -inline dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d std/allocator.d std/lexer.d std/d/ast.d std/d/parser.d std/d/lexer.d analysis/base.d analysis/del.d analysis/enumarrayliteral.d analysis/constructors.d analysis/ifelsesame.d analysis/fish.d analysis/numbers.d analysis/objectconst.d analysis/package.d analysis/pokemon.d analysis/range.d analysis/run.d analysis/style.d -ofdscanner.exe -version=DIP61 -O -release -noboundscheck -inline

17
build.sh
View File

@ -7,12 +7,13 @@ dmd\
astprinter.d\ astprinter.d\
formatter.d\ formatter.d\
outliner.d\ outliner.d\
stdx/*.d\ std/*.d\
stdx/d/*.d\ std/d/*.d\
analysis/*.d\ analysis/*.d\
-version=DIP61\
-ofdscanner\ -ofdscanner\
-m64 -g\ -g\
-O -release -noboundscheck -inline -O -release
#gdc\ #gdc\
# main.d\ # main.d\
@ -23,8 +24,8 @@ dmd\
# astprinter.d\ # astprinter.d\
# formatter.d\ # formatter.d\
# outliner.d\ # outliner.d\
# stdx/*.d\ # std/*.d\
# stdx/d/*.d\ # std/d/*.d\
# analysis/*.d\ # analysis/*.d\
# -O3 -frelease -fno-bounds-check\ # -O3 -frelease -fno-bounds-check\
# -odscanner\ # -odscanner\
@ -38,8 +39,8 @@ dmd\
# astprinter.d\ # astprinter.d\
# formatter.d\ # formatter.d\
# outliner.d\ # outliner.d\
# stdx/*.d\ # std/*.d\
# stdx/d/*.d\ # std/d/*.d\
# analysis/*.d\ # analysis/*.d\
# -O3 -release\ # -O3 -release\
# -oq -of=dscanner\ # -oq -of=dscanner\

35
ctags.d
View File

@ -5,9 +5,9 @@
module ctags; module ctags;
import stdx.d.parser; import std.d.parser;
import stdx.d.lexer; import std.d.lexer;
import stdx.d.ast; import std.d.ast;
import std.algorithm; import std.algorithm;
import std.range; import std.range;
import std.stdio; import std.stdio;
@ -71,7 +71,9 @@ class CTagsPrinter : ASTVisitor
override void visit(const TemplateDeclaration dec) override void visit(const TemplateDeclaration dec)
{ {
tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.text, fileName, dec.name.line, context); auto name = dec.eponymousTemplateDeclaration is null ? dec.name
: dec.eponymousTemplateDeclaration.name;
tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(name.text, fileName, name.line, context);
auto c = context; auto c = context;
context = "\ttemplate:" ~ dec.name.text; context = "\ttemplate:" ~ dec.name.text;
dec.accept(this); dec.accept(this);
@ -88,6 +90,26 @@ class CTagsPrinter : ASTVisitor
context = c; context = c;
} }
override void visit(const Constructor dec)
{
tagLines ~= "this\t%s\t%d;\"\tf\tarity:%d%s\n".format(fileName,
dec.line, dec.parameters.parameters.length, context);
auto c = context;
context = "\tfunction: this";
dec.accept(this);
context = c;
}
override void visit(const Destructor dec)
{
tagLines ~= "~this\t%s\t%d;\"\tf%s\n".format(fileName, dec.line,
context);
auto c = context;
context = "\tfunction: this";
dec.accept(this);
context = c;
}
override void visit(const EnumDeclaration dec) override void visit(const EnumDeclaration dec)
{ {
if (dec.name == tok!"") if (dec.name == tok!"")
@ -134,6 +156,11 @@ class CTagsPrinter : ASTVisitor
dec.accept(this); dec.accept(this);
} }
override void visit(const Invariant dec)
{
tagLines ~= "invariant\t%s\t%d;\"\tv%s\n".format(fileName, dec.line, context);
}
alias visit = ASTVisitor.visit; alias visit = ASTVisitor.visit;
string fileName; string fileName;

2
dub.json
View File

@ -9,7 +9,7 @@
{ {
"name": "library", "name": "library",
"targetType": "library", "targetType": "library",
"sourcePaths": ["stdx"], "sourcePaths": ["std"],
}, },
{ {
"name": "dscanner", "name": "dscanner",

12
formatter.d
View File

@ -1,7 +1,7 @@
module formatter; module formatter;
import stdx.d.ast; import std.d.ast;
import stdx.d.lexer; import std.d.lexer;
/** /**
* The only brace styles worth using. * The only brace styles worth using.
@ -489,6 +489,7 @@ class Formatter(Sink)
foreach (attribute; parameter.parameterAttributes) foreach (attribute; parameter.parameterAttributes)
{ {
sink.put(str(attribute)); sink.put(str(attribute));
sink.put(" ");
} }
if (parameter.type !is null) if (parameter.type !is null)
format(parameter.type); format(parameter.type);
@ -682,16 +683,11 @@ class Formatter(Sink)
void format(const Type type) void format(const Type type)
{ {
bool first = true;
foreach (constructor; type.typeConstructors) foreach (constructor; type.typeConstructors)
{ {
if (first)
sink.put(" ");
first = false;
sink.put(str(constructor)); sink.put(str(constructor));
}
if (type.typeConstructors.length > 0)
sink.put(" "); sink.put(" ");
}
format(type.type2); format(type.type2);
foreach (suffix; type.typeSuffixes) foreach (suffix; type.typeSuffixes)
{ {

4
highlighter.d
View File

@ -8,7 +8,7 @@ module highlighter;
import std.stdio; import std.stdio;
import std.array; import std.array;
import stdx.d.lexer; import std.d.lexer;
// http://ethanschoonover.com/solarized // http://ethanschoonover.com/solarized
void highlight(R)(ref R tokens, string fileName) void highlight(R)(ref R tokens, string fileName)
@ -49,6 +49,8 @@ html { background-color: #fdf6e3; color: #002b36; }
writeSpan("num", t.text); writeSpan("num", t.text);
else if (isOperator(t.type)) else if (isOperator(t.type))
writeSpan("op", str(t.type)); writeSpan("op", str(t.type));
else if (t.type == tok!"specialTokenSequence" || t.type == tok!"scriptLine")
writeSpan("cons", t.text.replace("<", "&lt;"));
else else
{ {
version(Windows) version(Windows)

17
imports.d
View File

@ -5,16 +5,21 @@
module imports; module imports;
import stdx.d.ast; import std.d.ast;
import std.stdio; import std.stdio;
import std.container;
class ImportPrinter : ASTVisitor class ImportPrinter : ASTVisitor
{ {
this()
{
imports = new RedBlackTree!string;
}
override void visit(const SingleImport singleImport) override void visit(const SingleImport singleImport)
{ {
ignore = false; ignore = false;
singleImport.accept(this); singleImport.accept(this);
writeln();
ignore = true; ignore = true;
} }
@ -22,15 +27,19 @@ class ImportPrinter : ASTVisitor
{ {
if (ignore) return; if (ignore) return;
bool first = true; bool first = true;
string s;
foreach (ident; identifierChain.identifiers) foreach (ident; identifierChain.identifiers)
{ {
if (!first) if (!first)
write("."); s ~= ".";
write(ident.text); s ~= ident.text;
first = false; first = false;
} }
imports.insert(s);
} }
RedBlackTree!string imports;
alias visit = ASTVisitor.visit; alias visit = ASTVisitor.visit;
bool ignore = true; bool ignore = true;

14
main.d
View File

@ -13,9 +13,9 @@ import std.getopt;
import std.path; import std.path;
import std.stdio; import std.stdio;
import std.range; import std.range;
import stdx.lexer; import std.lexer;
import stdx.d.lexer; import std.d.lexer;
import stdx.d.parser; import std.d.parser;
import highlighter; import highlighter;
import stats; import stats;
@ -101,6 +101,7 @@ int main(string[] args)
config.whitespaceBehavior = WhitespaceBehavior.include; config.whitespaceBehavior = WhitespaceBehavior.include;
config.stringBehavior = StringBehavior.source; config.stringBehavior = StringBehavior.source;
config.commentBehavior = CommentBehavior.include; config.commentBehavior = CommentBehavior.include;
config.specialTokenBehavior = SpecialTokenBehavior.include;
auto tokens = byToken(bytes, config, cache); auto tokens = byToken(bytes, config, cache);
if (highlight) if (highlight)
{ {
@ -169,11 +170,14 @@ int main(string[] args)
else if (imports || ast || outline) else if (imports || ast || outline)
{ {
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1])); auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]); auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1],
null, &doNothing);
if (imports) if (imports)
{ {
auto visitor = new ImportPrinter; auto visitor = new ImportPrinter;
visitor.visit(mod); visitor.visit(mod);
foreach (imp; visitor.imports[])
writeln(imp);
} }
else if (ast) else if (ast)
{ {
@ -291,3 +295,5 @@ options:
directories and its sub-directories.`, directories and its sub-directories.`,
programName); programName);
} }
void doNothing(string, size_t, size_t, string, bool) {}

4
outliner.d
View File

@ -3,8 +3,8 @@
// (See accompanying file LICENSE_1_0.txt or copy at // (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt) // http://www.boost.org/LICENSE_1_0.txt)
import stdx.d.lexer; import std.d.lexer;
import stdx.d.ast; import std.d.ast;
import std.stdio; import std.stdio;
import std.string; import std.string;
import std.array; import std.array;

2
stats.d
View File

@ -7,7 +7,7 @@ module stats;
import std.stdio; import std.stdio;
import std.algorithm; import std.algorithm;
import stdx.d.lexer; import std.d.lexer;
pure nothrow bool isLineOfCode(IdType t) pure nothrow bool isLineOfCode(IdType t)
{ {

387
stdx/allocator.d → std/allocator.d
View File

@ -246,7 +246,7 @@ uses an array of allocators, one per bucket, to satisfy requests.))
) )
*/ */
module stdx.allocator; module std.allocator;
// Example in the synopsis above // Example in the synopsis above
unittest unittest
@ -422,6 +422,99 @@ unittest
static assert(stateSize!C3 == 2 * size_t.sizeof + char.sizeof); static assert(stateSize!C3 == 2 * size_t.sizeof + char.sizeof);
} }
/**
* Shortcut that encapsulates a cast and a call to emplace()
* Params:
* a = the allocator to use
* args = the arguments to $(D T)'s constructor
* Returns: a pointer to an instance of $(D T).
*/
T* allocate(T, Allocator, Args...)(auto ref Allocator a, auto ref Args args)
@trusted if (is (T == struct))
{
import std.conv : emplace;
void[] mem = a.allocate(T.sizeof);
return emplace(cast(T*) mem.ptr, args);
}
///
unittest
{
auto allocator = Mallocator.it;
struct TestStruct { int x = 5; }
TestStruct* p = allocate!TestStruct(allocator);
assert (p !is null);
assert (p.x == 5);
Mallocator.it.deallocate((cast(void*) p)[0 .. TestStruct.sizeof]);
}
/**
* Shortcut that encapsulates a cast and a call to emplace()
* Params:
* a = the allocator to use
* args = the arguments to $(D T)'s constructor
* Returns: a reference to an instance of $(D T).
*/
T allocate(T, Allocator, Args...)(auto ref Allocator a, auto ref Args args)
@trusted if (is (T == class))
{
import std.conv : emplace;
void[] mem = a.allocate(__traits(classInstanceSize, T));
return emplace!T(mem, args);
}
///
unittest
{
auto allocator = Mallocator.it;
class TestClass { int x; this(int x) { this.x = x; } }
TestClass tc = allocate!TestClass(allocator, 10);
assert (tc !is null);
assert (tc.x == 10);
Mallocator.it.deallocate((cast(void*) tc)[0 .. __traits(classInstanceSize, TestClass)]);
}
/**
* Encapsulates some casts and pointer slicing to deallocate $(D structInstance).
* This function does NOT call T's destructor.
*/
void deallocate(T, Allocator)(auto ref Allocator a, T* structInstance)
pure nothrow @trusted if (is (T == struct))
{
a.deallocate((cast(void*) structInstance)[0 .. T.sizeof]);
}
///
unittest
{
auto allocator = Mallocator.it;
bool d = false;
struct TestStruct { float f; }
TestStruct* ts = allocate!TestStruct(allocator);
deallocate(allocator, ts);
}
/**
* Encapsulates some casts and pointer slicing to deallocate $(D classInstance).
* This function does NOT call T's destructor.
*/
void deallocate(T, Allocator)(auto ref Allocator a, T classInstance)
pure nothrow @trusted if (is (T == class))
{
a.deallocate((cast(void*) classInstance)[0 .. __traits(classInstanceSize, T)]);
}
///
unittest
{
import std.math;
auto allocator = Mallocator.it;
class TestClass { double z; }
TestClass tc = allocate!TestClass(allocator);
assert (isnan(tc.z));
deallocate(allocator, tc);
}
/** /**
$(D chooseAtRuntime) is a compile-time constant of type $(D size_t) that several $(D chooseAtRuntime) is a compile-time constant of type $(D size_t) that several
parameterized structures in this module recognize to mean deferral to runtime of parameterized structures in this module recognize to mean deferral to runtime of
@ -452,7 +545,7 @@ enum uint platformAlignment = std.algorithm.max(double.alignof, real.alignof);
The default good size allocation is deduced as $(D n) rounded up to the The default good size allocation is deduced as $(D n) rounded up to the
allocator's alignment. allocator's alignment.
*/ */
size_t goodAllocSize(A)(auto ref A a, size_t n) size_t goodAllocSize(A)(auto ref A a, size_t n) pure nothrow
{ {
return n.roundUpToMultipleOf(a.alignment); return n.roundUpToMultipleOf(a.alignment);
} }
@ -528,10 +621,16 @@ struct NullAllocator
No-op. No-op.
*/ */
void deallocateAll() shared { } void deallocateAll() shared { }
static shared(NullAllocator) it() pure nothrow @property @trusted
{
return cast(typeof(return)) _it;
}
/** /**
Returns the $(D shared) global instance of the $(D NullAllocator). Returns the $(D shared) global instance of the $(D NullAllocator).
*/ */
static shared NullAllocator it; private static shared const NullAllocator _it;
} }
unittest unittest
@ -557,16 +656,18 @@ struct GCAllocator
enum uint alignment = platformAlignment; enum uint alignment = platformAlignment;
/** /**
Standard allocator methods per the semantics defined above. The $(D deallocate) and $(D reallocate) methods are $(D @system) because they may move memory around, leaving dangling pointers in user code. Standard allocator methods per the semantics defined above. The
$(D deallocate) and $(D reallocate) methods are $(D @system) because they
may move memory around, leaving dangling pointers in user code.
*/ */
@trusted void[] allocate(size_t bytes) shared @trusted void[] allocate(size_t bytes) shared nothrow pure
{ {
auto p = GC.malloc(bytes); auto p = GC.malloc(bytes);
return p ? p[0 .. bytes] : null; return p ? p[0 .. bytes] : null;
} }
/// Ditto /// Ditto
@trusted bool expand(ref void[] b, size_t delta) shared @trusted bool expand(ref void[] b, size_t delta) shared nothrow pure
{ {
auto newSize = GC.extend(b.ptr, b.length + delta, auto newSize = GC.extend(b.ptr, b.length + delta,
b.length + delta); b.length + delta);
@ -581,7 +682,7 @@ struct GCAllocator
} }
/// Ditto /// Ditto
@system bool reallocate(ref void[] b, size_t newSize) shared @system bool reallocate(ref void[] b, size_t newSize) shared nothrow pure
{ {
import core.exception : OutOfMemoryError; import core.exception : OutOfMemoryError;
try try
@ -598,15 +699,20 @@ struct GCAllocator
} }
/// Ditto /// Ditto
@system void deallocate(void[] b) shared @system void deallocate(void[] b) shared nothrow pure
{ {
GC.free(b.ptr); GC.free(b.ptr);
} }
static shared(GCAllocator) it() pure nothrow @property @trusted
{
return cast(typeof(return)) _it;
}
/** /**
Returns the global instance of this allocator type. The garbage collected allocator is thread-safe, therefore all of its methods and $(D it) itself are $(D shared). Returns the global instance of this allocator type. The garbage collected allocator is thread-safe, therefore all of its methods and $(D it) itself are $(D shared).
*/ */
static shared GCAllocator it; private static shared const GCAllocator _it;
// Leave it undocummented for now. // Leave it undocummented for now.
@trusted void collect() shared @trusted void collect() shared
@ -629,12 +735,19 @@ unittest
assert(GCAllocator.it.expand(b, 1)); assert(GCAllocator.it.expand(b, 1));
} }
private extern (C)
{
void* malloc(size_t) pure nothrow @trusted;
void free(void*) pure nothrow @trusted;
void* realloc(void*, size_t) pure nothrow @trusted;
}
/** /**
The C heap allocator. The C heap allocator.
*/ */
struct Mallocator struct Mallocator
{ {
private import core.stdc.stdlib; // private import core.stdc.stdlib;
/** /**
The alignment is a static constant equal to $(D platformAlignment), which ensures proper alignment for any D data type. The alignment is a static constant equal to $(D platformAlignment), which ensures proper alignment for any D data type.
@ -642,22 +755,26 @@ struct Mallocator
enum uint alignment = platformAlignment; enum uint alignment = platformAlignment;
/** /**
Standard allocator methods per the semantics defined above. The $(D deallocate) and $(D reallocate) methods are $(D @system) because they may move memory around, leaving dangling pointers in user code. Somewhat paradoxically, $(D malloc) is $(D @safe) but that's only useful to safe programs that can afford to leak memory allocated. Standard allocator methods per the semantics defined above. The
$(D deallocate) and $(D reallocate) methods are $(D @system) because they
may move memory around, leaving dangling pointers in user code. Somewhat
paradoxically, $(D malloc) is $(D @safe) but that's only useful to safe
programs that can afford to leak memory allocated.
*/ */
@trusted void[] allocate(size_t bytes) shared void[] allocate(size_t bytes) shared pure nothrow @trusted
{ {
auto p = malloc(bytes); auto p = malloc(bytes);
return p ? p[0 .. bytes] : null; return p ? p[0 .. bytes] : null;
} }
/// Ditto /// Ditto
@system void deallocate(void[] b) shared void deallocate(void[] b) shared pure nothrow @system
{ {
free(b.ptr); free(b.ptr);
} }
/// Ditto /// Ditto
@system bool reallocate(ref void[] b, size_t s) shared bool reallocate(ref void[] b, size_t s) shared pure nothrow @system
{ {
if (!s) if (!s)
{ {
@ -673,10 +790,15 @@ struct Mallocator
return true; return true;
} }
static shared(Mallocator) it() pure nothrow @property @trusted
{
return cast(typeof(return)) _it;
}
/** /**
Returns the global instance of this allocator type. The C heap allocator is thread-safe, therefore all of its methods and $(D it) itself are $(D shared). Returns the global instance of this allocator type. The C heap allocator is thread-safe, therefore all of its methods and $(D it) itself are $(D shared).
*/ */
static shared Mallocator it; private static shared const Mallocator _it;
} }
/// ///
@ -854,7 +976,7 @@ unittest
/** /**
Returns s rounded up to a multiple of base. Returns s rounded up to a multiple of base.
*/ */
private size_t roundUpToMultipleOf(size_t s, uint base) private size_t roundUpToMultipleOf(size_t s, uint base) pure nothrow @safe
{ {
assert(base); assert(base);
auto rem = s % base; auto rem = s % base;
@ -872,7 +994,7 @@ unittest
/** /**
Returns s rounded up to a multiple of base. Returns s rounded up to a multiple of base.
*/ */
private void[] roundStartToMultipleOf(void[] s, uint base) private void[] roundStartToMultipleOf(void[] s, uint base) pure nothrow @trusted
{ {
assert(base); assert(base);
auto p = cast(void*) roundUpToMultipleOf( auto p = cast(void*) roundUpToMultipleOf(
@ -892,7 +1014,7 @@ unittest
/** /**
Returns $(D s) rounded up to the nearest power of 2. Returns $(D s) rounded up to the nearest power of 2.
*/ */
private size_t roundUpToPowerOf2(size_t s) private size_t roundUpToPowerOf2(size_t s) pure nothrow @safe
{ {
assert(s <= (size_t.max >> 1) + 1); assert(s <= (size_t.max >> 1) + 1);
--s; --s;
@ -943,7 +1065,7 @@ struct AffixAllocator(Allocator, Prefix, Suffix = void)
static assert( static assert(
!stateSize!Prefix || Allocator.alignment >= Prefix.alignof, !stateSize!Prefix || Allocator.alignment >= Prefix.alignof,
"AffixAllocator does not work with allocators offering a smaller" "AffixAllocator does not work with allocators offering a smaller"
" alignment than the prefix alignment."); ~ " alignment than the prefix alignment.");
static assert(alignment % Suffix.alignof == 0, static assert(alignment % Suffix.alignof == 0,
"This restriction could be relaxed in the future."); "This restriction could be relaxed in the future.");
@ -960,11 +1082,11 @@ struct AffixAllocator(Allocator, Prefix, Suffix = void)
parent allocator. parent allocator.
*/ */
static if (stateSize!Allocator) Allocator parent; static if (stateSize!Allocator) Allocator parent;
else alias Allocator.it parent; else alias parent = Allocator.it;
template Impl() template Impl()
{ {
size_t goodAllocSize(size_t s) size_t goodAllocSize(size_t s) pure nothrow const
{ {
return parent.goodAllocSize(actualAllocationSize(s)); return parent.goodAllocSize(actualAllocationSize(s));
} }
@ -1064,27 +1186,30 @@ struct AffixAllocator(Allocator, Prefix, Suffix = void)
which may use the global default). Also, the methods will be $(D which may use the global default). Also, the methods will be $(D
shared) if the parent allocator defines them as such. shared) if the parent allocator defines them as such.
*/ */
size_t goodAllocSize(size_t); size_t goodAllocSize(size_t) pure nothrow const;
/// Ditto /// Ditto
void[] allocate(size_t); void[] allocate(size_t) pure nothrow;
/// Ditto /// Ditto
bool owns(void[]); bool owns(void[]) pure nothrow;
/// Ditto /// Ditto
bool expand(ref void[] b, size_t delta); bool expand(ref void[] b, size_t delta) pure nothrow;
/// Ditto /// Ditto
bool reallocate(ref void[] b, size_t s); bool reallocate(ref void[] b, size_t s)pure nothrow;
/// Ditto /// Ditto
void deallocate(void[] b); void deallocate(void[] b) pure nothrow;
/// Ditto /// Ditto
void deallocateAll(); void deallocateAll() pure nothrow;
/** /**
The $(D it) singleton is defined if and only if the parent allocator has no state and defines its own $(D it) object. The $(D it) singleton is defined if and only if the parent allocator has
no state and defines its own $(D it) object.
*/ */
static AffixAllocator it; static AffixAllocator it;
/** /**
Affix access functions offering mutable references to the affixes of a block previously allocated with this allocator. $(D b) may not be null. They are defined if and only if the corresponding affix is not $(D void). Affix access functions offering mutable references to the affixes of a
block previously allocated with this allocator. $(D b) may not be null.
They are defined if and only if the corresponding affix is not $(D void).
Precondition: $(D b !is null) Precondition: $(D b !is null)
*/ */
@ -1118,7 +1243,7 @@ unittest
unittest unittest
{ {
alias AffixAllocator!(Mallocator, size_t) A; alias A = AffixAllocator!(Mallocator, size_t);
auto b = A.it.allocate(10); auto b = A.it.allocate(10);
A.it.prefix(b) = 10; A.it.prefix(b) = 10;
assert(A.it.prefix(b) == 10); assert(A.it.prefix(b) == 10);
@ -1129,9 +1254,10 @@ unittest
} }
/** /**
Returns the number of most significant ones before a zero can be found in $(D x). If $(D x) contains no zeros (i.e. is equal to $(D ulong.max)), returns 64. Returns the number of most significant ones before a zero can be found in $(D x).
If $(D x) contains no zeros (i.e. is equal to $(D ulong.max)), returns 64.
*/ */
private uint leadingOnes(ulong x) private uint leadingOnes(ulong x) pure nothrow @safe
{ {
uint result = 0; uint result = 0;
while (cast(long) x < 0) while (cast(long) x < 0)
@ -1156,7 +1282,7 @@ unittest
/** /**
Finds a run of contiguous ones in $(D x) of length at least $(D n). Finds a run of contiguous ones in $(D x) of length at least $(D n).
*/ */
private uint findContigOnes(ulong x, uint n) private uint findContigOnes(ulong x, uint n) pure nothrow @safe
{ {
while (n > 1) while (n > 1)
{ {
@ -1184,7 +1310,7 @@ unittest
/** /**
Returns the number of trailing zeros of $(D x). Returns the number of trailing zeros of $(D x).
*/ */
private uint trailingZeros(ulong x) private uint trailingZeros(ulong x) pure nothrow @safe
{ {
uint result; uint result;
while (result < 64 && !(x & (1UL << result))) while (result < 64 && !(x & (1UL << result)))
@ -1206,7 +1332,7 @@ unittest
/* /*
Unconditionally sets the bits from lsb through msb in w to zero. Unconditionally sets the bits from lsb through msb in w to zero.
*/ */
private void setBits(ref ulong w, uint lsb, uint msb) private void setBits(ref ulong w, uint lsb, uint msb) pure nothrow @safe
{ {
assert(lsb <= msb && msb < 64); assert(lsb <= msb && msb < 64);
const mask = (ulong.max << lsb) & (ulong.max >> (63 - msb)); const mask = (ulong.max << lsb) & (ulong.max >> (63 - msb));
@ -1225,7 +1351,7 @@ unittest
/* Are bits from lsb through msb in w zero? If so, make then 1 /* Are bits from lsb through msb in w zero? If so, make then 1
and return the resulting w. Otherwise, just return 0. and return the resulting w. Otherwise, just return 0.
*/ */
private bool setBitsIfZero(ref ulong w, uint lsb, uint msb) private bool setBitsIfZero(ref ulong w, uint lsb, uint msb) pure nothrow @safe
{ {
assert(lsb <= msb && msb < 64); assert(lsb <= msb && msb < 64);
const mask = (ulong.max << lsb) & (ulong.max >> (63 - msb)); const mask = (ulong.max << lsb) & (ulong.max >> (63 - msb));
@ -1234,14 +1360,24 @@ private bool setBitsIfZero(ref ulong w, uint lsb, uint msb)
return true; return true;
} }
unittest
{
// TODO
}
// Assigns bits in w from lsb through msb to zero. // Assigns bits in w from lsb through msb to zero.
private void resetBits(ref ulong w, uint lsb, uint msb) private void resetBits(ref ulong w, uint lsb, uint msb) pure nothrow @safe
{ {
assert(lsb <= msb && msb < 64); assert(lsb <= msb && msb < 64);
const mask = (ulong.max << lsb) & (ulong.max >> (63 - msb)); const mask = (ulong.max << lsb) & (ulong.max >> (63 - msb));
w &= ~mask; w &= ~mask;
} }
unittest
{
// TODO
}
/** /**
$(D HeapBlock) implements a simple heap consisting of one contiguous area $(D HeapBlock) implements a simple heap consisting of one contiguous area
@ -1329,7 +1465,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
_blocks = blocks; _blocks = blocks;
} }
private void initialize() private void initialize() pure nothrow @trusted
{ {
assert(_blocks); assert(_blocks);
const controlBytes = ((_blocks + 63) / 64) * 8; const controlBytes = ((_blocks + 63) / 64) * 8;
@ -1344,11 +1480,11 @@ struct HeapBlock(Allocator, size_t theBlockSize,
_control = m[0 .. controlBytes / 8]; _control = m[0 .. controlBytes / 8];
_control[] = 0; _control[] = 0;
_payload = m[controlBytesRounded / 8 .. $]; _payload = m[controlBytesRounded / 8 .. $];
assert(_payload.length == _blocks * blockSize, assert(_payload.length == _blocks * blockSize/+,
text(_payload.length, " != ", _blocks * blockSize)); text(_payload.length, " != ", _blocks * blockSize)+/);
} }
private void initialize(void[] store) private void initialize(void[] store) pure nothrow @trusted
{ {
assert(store.length); assert(store.length);
// Round store to be ulong-aligned // Round store to be ulong-aligned
@ -1363,9 +1499,9 @@ struct HeapBlock(Allocator, size_t theBlockSize,
auto blocks = cast(size_t) (approxBlocks + nextDown(1.0)); auto blocks = cast(size_t) (approxBlocks + nextDown(1.0));
assert(blocks > 0); assert(blocks > 0);
assert(blockSize); assert(blockSize);
assert(blocks * blockSize + ((blocks + 63) / 64) * 8 >= store.length, assert(blocks * blockSize + ((blocks + 63) / 64) * 8 >= store.length/+,
text(approxBlocks, " ", blocks, " ", blockSize, " ", text(approxBlocks, " ", blocks, " ", blockSize, " ",
store.length)); store.length)+/);
while (blocks * blockSize + ((blocks + 63) / 64) * 8 > store.length) while (blocks * blockSize + ((blocks + 63) / 64) * 8 > store.length)
{ {
--blocks; --blocks;
@ -1385,13 +1521,13 @@ struct HeapBlock(Allocator, size_t theBlockSize,
} }
private void initialize(ulong[] control, void[] payload, size_t blockSize) private void initialize(ulong[] control, void[] payload, size_t blockSize)
pure nothrow @trusted
{ {
enforce(payload.length % blockSize == 0, assert(payload.length % blockSize == 0);
text(payload.length, " % ", blockSize, " != 0"));
assert(payload.length / blockSize <= uint.max); assert(payload.length / blockSize <= uint.max);
_blocks = cast(uint) (payload.length / blockSize); _blocks = cast(uint) (payload.length / blockSize);
const controlWords = (_blocks + 63) / 64; const controlWords = (_blocks + 63) / 64;
enforce(controlWords == control.length); assert(controlWords == control.length);
_control = control; _control = control;
assert(control.equal(repeat(0, control.length))); assert(control.equal(repeat(0, control.length)));
_payload = payload; _payload = payload;
@ -1434,7 +1570,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
BUGS: Neither $(D deallocateAll) nor the destructor free the original memory BUGS: Neither $(D deallocateAll) nor the destructor free the original memory
block. Either user code or the parent allocator should carry that. block. Either user code or the parent allocator should carry that.
*/ */
@trusted void[] allocate(const size_t s) void[] allocate(const size_t s) pure nothrow @trusted
{ {
if (!_control) if (!_control)
{ {
@ -1486,7 +1622,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
} }
/// Ditto /// Ditto
bool owns(void[] b) const bool owns(void[] b) const pure nothrow @trusted
{ {
return b.ptr >= _payload.ptr return b.ptr >= _payload.ptr
&& b.ptr + b.length <= _payload.ptr + _payload.length && b.ptr + b.length <= _payload.ptr + _payload.length
@ -1500,7 +1636,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
0). Otherwise, returns a tuple with the next position to search. 0). Otherwise, returns a tuple with the next position to search.
*/ */
private Tuple!(size_t, uint) allocateAt(size_t wordIdx, uint msbIdx, private Tuple!(size_t, uint) allocateAt(size_t wordIdx, uint msbIdx,
size_t blocks, ref void[] result) size_t blocks, ref void[] result) pure nothrow @trusted
{ {
assert(blocks > 0); assert(blocks > 0);
assert(wordIdx < _control.length); assert(wordIdx < _control.length);
@ -1557,9 +1693,9 @@ struct HeapBlock(Allocator, size_t theBlockSize,
return available; return available;
} }
private void[] smallAlloc(uint blocks) private void[] smallAlloc(uint blocks) pure nothrow @trusted
{ {
assert(blocks >= 2 && blocks <= 64, text(blocks)); assert(blocks >= 2 && blocks <= 64/+, text(blocks)+/);
foreach (i; _startIdx .. _control.length) foreach (i; _startIdx .. _control.length)
{ {
// Test within the current 64-bit word // Test within the current 64-bit word
@ -1589,7 +1725,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
return null; return null;
} }
private void[] hugeAlloc(size_t blocks) private void[] hugeAlloc(size_t blocks) pure nothrow @trusted
{ {
assert(blocks > 64); assert(blocks > 64);
void[] result; void[] result;
@ -1629,7 +1765,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
} }
/// Ditto /// Ditto
@trusted bool expand(ref void[] b, size_t delta) bool expand(ref void[] b, size_t delta) pure nothrow @trusted
{ {
//debug writefln("expand(%s, %s, %s)", b, minDelta, desiredDelta); //debug writefln("expand(%s, %s, %s)", b, minDelta, desiredDelta);
if (b is null) if (b is null)
@ -1664,14 +1800,14 @@ struct HeapBlock(Allocator, size_t theBlockSize,
return false; return false;
} }
// Expansion successful // Expansion successful
assert(p.ptr == b.ptr + blocksOld * blockSize, assert(p.ptr == b.ptr + blocksOld * blockSize/+,
text(p.ptr, " != ", b.ptr + blocksOld * blockSize)); text(p.ptr, " != ", b.ptr + blocksOld * blockSize)+/);
b = b.ptr[0 .. b.length + delta]; b = b.ptr[0 .. b.length + delta];
return true; return true;
} }
/// Ditto /// Ditto
@system bool reallocate(ref void[] b, size_t newSize) bool reallocate(ref void[] b, size_t newSize) pure nothrow @system
{ {
if (newSize == 0) if (newSize == 0)
{ {
@ -1902,10 +2038,10 @@ struct FallbackAllocator(Primary, Fallback)
If both $(D Primary) and $(D Fallback) are stateless, $(D FallbackAllocator) If both $(D Primary) and $(D Fallback) are stateless, $(D FallbackAllocator)
defines a static instance $(D it). defines a static instance $(D it).
*/ */
static if (!stateSize!Primary && !stateSize!Fallback) /+static if (!stateSize!Primary && !stateSize!Fallback)
{ {
static FallbackAllocator it; static FallbackAllocator it;
} }+/
/** /**
The alignment offered is the minimum of the two allocators' alignment. The alignment offered is the minimum of the two allocators' alignment.
@ -1916,7 +2052,7 @@ struct FallbackAllocator(Primary, Fallback)
Allocates memory trying the primary allocator first. If it returns $(D Allocates memory trying the primary allocator first. If it returns $(D
null), the fallback allocator is tried. null), the fallback allocator is tried.
*/ */
void[] allocate(size_t s) void[] allocate(size_t s) pure nothrow @safe
{ {
auto result = primary.allocate(s); auto result = primary.allocate(s);
return result ? result : fallback.allocate(s); return result ? result : fallback.allocate(s);
@ -1959,12 +2095,12 @@ struct FallbackAllocator(Primary, Fallback)
allocation from $(D fallback) to $(D primary). allocation from $(D fallback) to $(D primary).
*/ */
bool reallocate(ref void[] b, size_t newSize) bool reallocate(ref void[] b, size_t newSize) pure nothrow @trusted
{ {
bool crossAllocatorMove(From, To)(ref From from, ref To to) bool crossAllocatorMove(From, To)(auto ref From from, auto ref To to)
{ {
auto b1 = to.allocate(newSize); auto b1 = to.allocate(newSize);
if (!b1) return false; if (b1 is null) return false;
if (b.length < newSize) b1[0 .. b.length] = b[]; if (b.length < newSize) b1[0 .. b.length] = b[];
else b1[] = b[0 .. newSize]; else b1[] = b[0 .. newSize];
static if (hasMember!(From, "deallocate")) static if (hasMember!(From, "deallocate"))
@ -2004,7 +2140,7 @@ struct FallbackAllocator(Primary, Fallback)
*/ */
static if (hasMember!(Primary, "deallocate") static if (hasMember!(Primary, "deallocate")
|| hasMember!(Fallback, "deallocate")) || hasMember!(Fallback, "deallocate"))
void deallocate(void[] b) void deallocate(void[] b) pure nothrow @trusted
{ {
if (primary.owns(b)) if (primary.owns(b))
{ {
@ -2014,7 +2150,7 @@ struct FallbackAllocator(Primary, Fallback)
else else
{ {
static if (hasMember!(Fallback, "deallocate")) static if (hasMember!(Fallback, "deallocate"))
return fallback.deallocate(b); fallback.deallocate(b);
} }
} }
} }
@ -2022,7 +2158,7 @@ struct FallbackAllocator(Primary, Fallback)
/// ///
unittest unittest
{ {
FallbackAllocator!(InSituRegion!16384, GCAllocator) a; FallbackAllocator!(InSituRegion!16_384, GCAllocator) a;
// This allocation uses the stack // This allocation uses the stack
auto b1 = a.allocate(1024); auto b1 = a.allocate(1024);
assert(b1.length == 1024, text(b1.length)); assert(b1.length == 1024, text(b1.length));
@ -2079,7 +2215,7 @@ struct Freelist(ParentAllocator,
else else
{ {
size_t _min = chooseAtRuntime; size_t _min = chooseAtRuntime;
@property size_t min() const @property size_t min() const nothrow pure @safe
{ {
assert(_min != chooseAtRuntime); assert(_min != chooseAtRuntime);
return _min; return _min;
@ -2102,7 +2238,7 @@ struct Freelist(ParentAllocator,
} }
} }
private bool tooSmall(size_t n) const private bool tooSmall(size_t n) const nothrow pure @safe
{ {
static if (minSize == 0) return false; static if (minSize == 0) return false;
else return n < min; else return n < min;
@ -2123,13 +2259,13 @@ struct Freelist(ParentAllocator,
} }
} }
private bool tooLarge(size_t n) const private bool tooLarge(size_t n) const nothrow pure @safe
{ {
static if (maxSize == unbounded) return false; static if (maxSize == unbounded) return false;
else return n > max; else return n > max;
} }
private bool inRange(size_t n) const private bool inRange(size_t n) const nothrow pure @safe
{ {
static if (minSize == maxSize && minSize != chooseAtRuntime) static if (minSize == maxSize && minSize != chooseAtRuntime)
return n == maxSize; return n == maxSize;
@ -2206,7 +2342,7 @@ struct Freelist(ParentAllocator,
Returns $(D max) for sizes in the interval $(D [min, max]), and $(D Returns $(D max) for sizes in the interval $(D [min, max]), and $(D
parent.goodAllocSize(bytes)) otherwise. parent.goodAllocSize(bytes)) otherwise.
*/ */
size_t goodAllocSize(size_t bytes) size_t goodAllocSize(size_t bytes) const nothrow pure @safe
{ {
if (inRange(bytes)) return maxSize == unbounded ? bytes : max; if (inRange(bytes)) return maxSize == unbounded ? bytes : max;
return parent.goodAllocSize(bytes); return parent.goodAllocSize(bytes);
@ -2215,12 +2351,7 @@ struct Freelist(ParentAllocator,
/** /**
Allocates memory either off of the free list or from the parent allocator. Allocates memory either off of the free list or from the parent allocator.
*/ */
void[] allocate(size_t bytes) void[] allocate(size_t bytes) pure nothrow @trusted
in
{
assert (_root !is null);
}
body
{ {
assert(bytes < size_t.max / 2); assert(bytes < size_t.max / 2);
if (!inRange(bytes)) return parent.allocate(bytes); if (!inRange(bytes)) return parent.allocate(bytes);
@ -2234,7 +2365,7 @@ struct Freelist(ParentAllocator,
return result; return result;
} }
private void[] allocateFresh(const size_t bytes) private void[] allocateFresh(const size_t bytes) pure nothrow @trusted
{ {
assert(!_root); assert(!_root);
assert(bytes == max || max == unbounded); assert(bytes == max || max == unbounded);
@ -2251,9 +2382,9 @@ struct Freelist(ParentAllocator,
} }
else else
{ {
assert((parent.alignment + bytes) % Node.alignof == 0, assert((parent.alignment + bytes) % Node.alignof == 0/+,
text("(", parent.alignment, " + ", bytes, ") % ", text("(", parent.alignment, " + ", bytes, ") % ",
Node.alignof)); Node.alignof)+/);
} }
auto data = parent.allocate(nodesAtATime * bytes); auto data = parent.allocate(nodesAtATime * bytes);
@ -2282,7 +2413,7 @@ struct Freelist(ParentAllocator,
Freelist) handle deallocations of objects of the appropriate size, Freelist) handle deallocations of objects of the appropriate size,
even for allocators that don't support $(D owns) (such as $(D Mallocator)). even for allocators that don't support $(D owns) (such as $(D Mallocator)).
*/ */
bool owns(void[] b) bool owns(void[] b) const pure nothrow @safe
{ {
if (inRange(b.length)) return true; if (inRange(b.length)) return true;
static if (hasMember!(ParentAllocator, "owns")) static if (hasMember!(ParentAllocator, "owns"))
@ -2560,7 +2691,7 @@ struct SharedFreelist(ParentAllocator,
do do
{ {
oldRoot = _root; // atomic load oldRoot = _root; // atomic load
next = oldRoot.next; // atomic load next = oldRoot is null ? null : oldRoot.next; // atomic load
} }
while (!cas(&_root, oldRoot, next)); while (!cas(&_root, oldRoot, next));
// great, snatched the root // great, snatched the root
@ -2657,7 +2788,8 @@ struct SharedFreelist(ParentAllocator,
} }
} }
unittest // This deadlocks
version (none) unittest
{ {
import core.thread, std.concurrency; import core.thread, std.concurrency;
@ -2723,14 +2855,14 @@ private struct BasicRegion(uint minAlign = platformAlignment)
/** /**
Constructs a region backed by a user-provided store. Constructs a region backed by a user-provided store.
*/ */
this(void[] store) this(void[] store) pure nothrow @trusted
{ {
static if (minAlign > 1) static if (minAlign > 1)
{ {
auto newStore = cast(void*) roundUpToMultipleOf( auto newStore = cast(void*) roundUpToMultipleOf(
cast(ulong) store.ptr, cast(size_t) store.ptr,
alignment); alignment);
enforce(newStore <= store.ptr + store.length); assert(newStore <= store.ptr + store.length);
_current = newStore; _current = newStore;
} }
else else
@ -2752,7 +2884,7 @@ private struct BasicRegion(uint minAlign = platformAlignment)
enum uint alignment = minAlign; enum uint alignment = minAlign;
/// Ditto /// Ditto
void[] allocate(size_t bytes) void[] allocate(size_t bytes) pure nothrow @trusted
{ {
static if (minAlign > 1) static if (minAlign > 1)
const rounded = bytes.roundUpToMultipleOf(alignment); const rounded = bytes.roundUpToMultipleOf(alignment);
@ -2772,7 +2904,7 @@ private struct BasicRegion(uint minAlign = platformAlignment)
// Just bump the pointer to the next good allocation // Just bump the pointer to the next good allocation
auto save = _current; auto save = _current;
_current = cast(void*) roundUpToMultipleOf( _current = cast(void*) roundUpToMultipleOf(
cast(ulong) _current, a); cast(size_t) _current, a);
if (auto b = allocate(bytes)) return b; if (auto b = allocate(bytes)) return b;
// Failed, rollback // Failed, rollback
_current = save; _current = save;
@ -2829,7 +2961,7 @@ struct Region(uint minAlign = platformAlignment)
Constructs a $(D Region) object backed by $(D buffer), which must be aligned Constructs a $(D Region) object backed by $(D buffer), which must be aligned
to $(D minAlign). to $(D minAlign).
*/ */
this(void[] buffer) this(void[] buffer) pure nothrow @safe
{ {
base = BasicRegion!minAlign(buffer); base = BasicRegion!minAlign(buffer);
assert(buffer.ptr !is &this); assert(buffer.ptr !is &this);
@ -2932,7 +3064,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
{ {
assert(!_crt); assert(!_crt);
_crt = cast(void*) roundUpToMultipleOf( _crt = cast(void*) roundUpToMultipleOf(
cast(ulong) _store.ptr, alignment); cast(size_t) _store.ptr, alignment);
_end = _store.ptr + _store.length; _end = _store.ptr + _store.length;
} }
@ -2941,7 +3073,12 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
accommodate the request. For efficiency reasons, if $(D bytes == 0) the accommodate the request. For efficiency reasons, if $(D bytes == 0) the
function returns an empty non-null slice. function returns an empty non-null slice.
*/ */
void[] allocate(size_t bytes) void[] allocate(size_t bytes) pure nothrow @trusted
out (result)
{
assert (result is null || owns(result));
}
body
{ {
// Oddity: we don't return null for null allocation. Instead, we return // Oddity: we don't return null for null allocation. Instead, we return
// an empty slice with a non-null ptr. // an empty slice with a non-null ptr.
@ -2972,7 +3109,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
// Just bump the pointer to the next good allocation // Just bump the pointer to the next good allocation
auto save = _crt; auto save = _crt;
_crt = cast(void*) roundUpToMultipleOf( _crt = cast(void*) roundUpToMultipleOf(
cast(ulong) _crt, a); cast(size_t) _crt, a);
if (auto b = allocate(bytes)) return b; if (auto b = allocate(bytes)) return b;
// Failed, rollback // Failed, rollback
_crt = save; _crt = save;
@ -2984,7 +3121,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
allocation. For efficiency reasons, if $(D b is null) the function returns allocation. For efficiency reasons, if $(D b is null) the function returns
$(D false). $(D false).
*/ */
bool owns(void[] b) const bool owns(const void[] b) const nothrow pure @trusted
{ {
// No nullptr // No nullptr
return b.ptr >= _store.ptr return b.ptr >= _store.ptr
@ -2994,7 +3131,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
/** /**
Deallocates all memory allocated with this allocator. Deallocates all memory allocated with this allocator.
*/ */
void deallocateAll() void deallocateAll() pure nothrow @safe
{ {
_crt = _store.ptr; _crt = _store.ptr;
} }
@ -3715,7 +3852,14 @@ struct CascadingAllocator(alias make)
enum uint alignment = Allocator.alignment; enum uint alignment = Allocator.alignment;
/// Ditto /// Ditto
void[] allocate(size_t s) void[] allocate(size_t s) pure nothrow @trusted
out (res)
{
import std.string;
assert (res.length == 0 || res.length == s,
"res.length = %d, s = %d".format(res.length, s));
}
body
{ {
auto result = allocateNoGrow(s); auto result = allocateNoGrow(s);
if (result) return result; if (result) return result;
@ -3805,6 +3949,7 @@ struct CascadingAllocator(alias make)
if (!_root) return false; if (!_root) return false;
for (auto n = _root; ; n = n.next) for (auto n = _root; ; n = n.next)
{ {
static if (hasMember!(Allocator, "owns"))
if (n.a.owns(b) && n.a.reallocate(b, s)) return true; if (n.a.owns(b) && n.a.reallocate(b, s)) return true;
if (!n.nextIsInitialized) break; if (!n.nextIsInitialized) break;
} }
@ -3870,7 +4015,7 @@ struct CascadingAllocator(alias make)
unittest unittest
{ {
// Create an allocator based upon 4MB regions, fetched from the GC heap. // Create an allocator based upon 4MB regions, fetched from the GC heap.
CascadingAllocator!({ return Region!()(new void[1024 * 4096]); }) a; CascadingAllocator!(function() nothrow { return Region!()(new void[1024 * 4096]); }) a;
auto b1 = a.allocate(1024 * 8192); auto b1 = a.allocate(1024 * 8192);
assert(b1 is null); // can't allocate more than 4MB at a time assert(b1 is null); // can't allocate more than 4MB at a time
b1 = a.allocate(1024 * 10); b1 = a.allocate(1024 * 10);
@ -3997,21 +4142,21 @@ struct Segregator(size_t threshold, SmallAllocator, LargeAllocator)
template Impl() template Impl()
{ {
void[] allocate(size_t s) void[] allocate(size_t s) nothrow pure @trusted
{ {
return s <= threshold ? _small.allocate(s) : _large.allocate(s); return s <= threshold ? _small.allocate(s) : _large.allocate(s);
} }
static if (hasMember!(SmallAllocator, "deallocate") static if (hasMember!(SmallAllocator, "deallocate")
&& hasMember!(LargeAllocator, "deallocate")) && hasMember!(LargeAllocator, "deallocate"))
void deallocate(void[] data) void deallocate(void[] data) nothrow pure @trusted
{ {
data.length <= threshold data.length <= threshold
? _small.deallocate(data) ? _small.deallocate(data)
: _large.deallocate(data); : _large.deallocate(data);
} }
size_t goodAllocSize(size_t s) size_t goodAllocSize(size_t s) const nothrow pure @safe
{ {
return s <= threshold return s <= threshold
? _small.goodAllocSize(s) ? _small.goodAllocSize(s)
@ -4020,7 +4165,7 @@ struct Segregator(size_t threshold, SmallAllocator, LargeAllocator)
static if (hasMember!(SmallAllocator, "owns") static if (hasMember!(SmallAllocator, "owns")
&& hasMember!(LargeAllocator, "owns")) && hasMember!(LargeAllocator, "owns"))
bool owns(void[] b) bool owns(void[] b) const nothrow pure @safe
{ {
return b.length <= threshold ? _small.owns(b) : _large.owns(b); return b.length <= threshold ? _small.owns(b) : _large.owns(b);
} }
@ -4087,7 +4232,11 @@ struct Segregator(size_t threshold, SmallAllocator, LargeAllocator)
static if (sharedMethods) static if (sharedMethods)
{ {
static shared Segregator it; static shared(typeof(this)) it() pure nothrow @property @trusted
{
return cast(typeof(return)) _it;
}
private static const shared Segregator _it;
shared { mixin Impl!(); } shared { mixin Impl!(); }
} }
else else
@ -4312,7 +4461,7 @@ class CAllocator
{ {
/// Returns the alignment offered. By default this method returns $(D /// Returns the alignment offered. By default this method returns $(D
/// platformAlignment). /// platformAlignment).
@property uint alignment() uint alignment() pure nothrow const @property
{ {
return platformAlignment; return platformAlignment;
} }
@ -4323,7 +4472,7 @@ class CAllocator
throw an exception if it does allow setting the alignment but an invalid throw an exception if it does allow setting the alignment but an invalid
value is passed. value is passed.
*/ */
@property bool alignment(uint) @property bool alignment(uint) pure nothrow @property
{ {
return false; return false;
} }
@ -4333,7 +4482,7 @@ class CAllocator
fragmentation. By default returns $(D s) rounded up to the nearest multiple fragmentation. By default returns $(D s) rounded up to the nearest multiple
of $(D alignment). of $(D alignment).
*/ */
size_t goodAllocSize(size_t s) size_t goodAllocSize(size_t s) pure nothrow const @property
{ {
return s.roundUpToMultipleOf(alignment); return s.roundUpToMultipleOf(alignment);
} }
@ -4341,7 +4490,7 @@ class CAllocator
/** /**
Allocates memory. Allocates memory.
*/ */
abstract void[] allocate(size_t); abstract void[] allocate(size_t) pure nothrow @safe;
/** /**
Returns $(D true) if the allocator supports $(D owns). By default returns Returns $(D true) if the allocator supports $(D owns). By default returns
@ -4362,17 +4511,17 @@ class CAllocator
} }
/// Expands a memory block in place. /// Expands a memory block in place.
abstract bool expand(ref void[], size_t); abstract bool expand(ref void[], size_t) pure nothrow;
/// Reallocates a memory block. /// Reallocates a memory block.
abstract bool reallocate(ref void[] b, size_t); abstract bool reallocate(ref void[] b, size_t) pure nothrow;
/// Deallocates a memory block. Returns $(D false) if deallocation is not /// Deallocates a memory block. Returns $(D false) if deallocation is not
/// supported. /// supported.
abstract bool deallocate(void[]); abstract bool deallocate(void[]) pure nothrow;
/// Deallocates all memory. Returns $(D false) if not supported. /// Deallocates all memory. Returns $(D false) if not supported.
abstract bool deallocateAll(); abstract bool deallocateAll() pure nothrow;
/// Returns $(D true) if allocator supports $(D allocateAll). By default /// Returns $(D true) if allocator supports $(D allocateAll). By default
/// returns $(D false). /// returns $(D false).
@ -4405,7 +4554,7 @@ class CAllocatorImpl(Allocator) : CAllocator
else alias impl = Allocator.it; else alias impl = Allocator.it;
/// Returns $(D impl.alignment). /// Returns $(D impl.alignment).
override @property uint alignment() override uint alignment() pure nothrow const @property
{ {
return impl.alignment; return impl.alignment;
} }
@ -4414,7 +4563,7 @@ class CAllocatorImpl(Allocator) : CAllocator
If $(D Allocator) supports alignment setting, performs it and returns $(D If $(D Allocator) supports alignment setting, performs it and returns $(D
true). Otherwise, returns $(D false). true). Otherwise, returns $(D false).
*/ */
override @property bool alignment(uint a) override bool alignment(uint a) pure nothrow const @property
{ {
static if (is(typeof(impl.alignment = a))) static if (is(typeof(impl.alignment = a)))
{ {
@ -4430,7 +4579,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/** /**
Returns $(D impl.goodAllocSize(s)). Returns $(D impl.goodAllocSize(s)).
*/ */
override size_t goodAllocSize(size_t s) override size_t goodAllocSize(size_t s) pure nothrow const @property
{ {
return impl.goodAllocSize(s); return impl.goodAllocSize(s);
} }
@ -4438,7 +4587,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/** /**
Returns $(D impl.allocate(s)). Returns $(D impl.allocate(s)).
*/ */
override void[] allocate(size_t s) override void[] allocate(size_t s) pure nothrow @safe
{ {
return impl.allocate(s); return impl.allocate(s);
} }
@ -4446,7 +4595,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/** /**
Returns $(D true) if $(D Allocator) supports $(D owns). Returns $(D true) if $(D Allocator) supports $(D owns).
*/ */
override bool supportsOwns() override bool supportsOwns() pure nothrow const @safe
{ {
return hasMember!(Allocator, "owns"); return hasMember!(Allocator, "owns");
} }
@ -4462,7 +4611,7 @@ class CAllocatorImpl(Allocator) : CAllocator
} }
/// Returns $(D impl.expand(b, s)) if defined, $(D false) otherwise. /// Returns $(D impl.expand(b, s)) if defined, $(D false) otherwise.
override bool expand(ref void[] b, size_t s) override bool expand(ref void[] b, size_t s) pure nothrow
{ {
static if (hasMember!(Allocator, "expand")) static if (hasMember!(Allocator, "expand"))
return impl.expand(b, s); return impl.expand(b, s);
@ -4478,7 +4627,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/// Calls $(D impl.deallocate(b)) and returns $(D true) if defined, /// Calls $(D impl.deallocate(b)) and returns $(D true) if defined,
/// otherwise returns $(D false). /// otherwise returns $(D false).
override bool deallocate(void[] b) override bool deallocate(void[] b) pure nothrow
{ {
static if (hasMember!(Allocator, "deallocate")) static if (hasMember!(Allocator, "deallocate"))
{ {
@ -4493,7 +4642,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/// Calls $(D impl.deallocateAll()) and returns $(D true) if defined, /// Calls $(D impl.deallocateAll()) and returns $(D true) if defined,
/// otherwise returns $(D false). /// otherwise returns $(D false).
override bool deallocateAll() override bool deallocateAll() pure nothrow
{ {
static if (hasMember!(Allocator, "deallocateAll")) static if (hasMember!(Allocator, "deallocateAll"))
{ {
@ -4508,7 +4657,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/// Returns $(D true) if allocator supports $(D allocateAll). By default /// Returns $(D true) if allocator supports $(D allocateAll). By default
/// returns $(D false). /// returns $(D false).
override bool supportsAllocateAll() override bool supportsAllocateAll() pure nothrow const
{ {
return hasMember!(Allocator, "allocateAll"); return hasMember!(Allocator, "allocateAll");
} }
@ -4518,7 +4667,7 @@ class CAllocatorImpl(Allocator) : CAllocator
returns $(D impl.allocateAll()). returns $(D impl.allocateAll()).
*/ */
static if (hasMember!(Allocator, "allocateAll")) static if (hasMember!(Allocator, "allocateAll"))
override void[] allocateAll() override void[] allocateAll() pure nothrow
{ {
return impl.allocateAll(); return impl.allocateAll();
} }

267
stdx/d/ast.d → std/d/ast.d
View File

File diff suppressed because it is too large Load Diff

2
stdx/d/entities.d → std/d/entities.d
View File

@ -9,7 +9,7 @@
* Source: $(PHOBOSSRC std/d/_entities.d) * Source: $(PHOBOSSRC std/d/_entities.d)
*/ */
module stdx.d.entities; module std.d.entities;
/** /**
* Generated from $(LINK http://www.w3.org/TR/html5/entities.json) * Generated from $(LINK http://www.w3.org/TR/html5/entities.json)

52
stdx/d/lexer.d → std/d/lexer.d
View File

@ -1,12 +1,12 @@
module stdx.d.lexer; module std.d.lexer;
import std.typecons; import std.typecons;
import std.typetuple; import std.typetuple;
import std.array; import std.array;
import std.algorithm; import std.algorithm;
import std.range; import std.range;
import stdx.lexer; import std.lexer;
public import stdx.lexer : StringCache; public import std.lexer : StringCache;
private enum operators = [ private enum operators = [
",", ".", "..", "...", "/", "/=", "!", "!<", "!<=", "!<>", "!<>=", "!=", ",", ".", "..", "...", "/", "/=", "!", "!<", "!<=", "!<>", "!<>=", "!=",
@ -41,7 +41,7 @@ private enum dynamicTokens = [
"whitespace", "doubleLiteral", "floatLiteral", "idoubleLiteral", "whitespace", "doubleLiteral", "floatLiteral", "idoubleLiteral",
"ifloatLiteral", "intLiteral", "longLiteral", "realLiteral", "ifloatLiteral", "intLiteral", "longLiteral", "realLiteral",
"irealLiteral", "uintLiteral", "ulongLiteral", "characterLiteral", "irealLiteral", "uintLiteral", "ulongLiteral", "characterLiteral",
"dstringLiteral", "stringLiteral", "wstringLiteral", "scriptLine" "dstringLiteral", "stringLiteral", "wstringLiteral"
]; ];
private enum pseudoTokenHandlers = [ private enum pseudoTokenHandlers = [
@ -91,7 +91,7 @@ private enum extraFields = q{
return 0; return 0;
} }
}; };
public alias Token = stdx.lexer.TokenStructure!(IdType, extraFields); public alias Token = std.lexer.TokenStructure!(IdType, extraFields);
/** /**
* Configure string lexing behavior * Configure string lexing behavior
@ -119,6 +119,18 @@ public enum WhitespaceBehavior : ubyte
/// Whitespace is treated as a token /// Whitespace is treated as a token
include include
} }
/**
* Configure special token handling behavior
*/
public enum SpecialTokenBehavior : ubyte
{
/// Special tokens are skipped
skip,
/// Special tokens are treated as a token
include
}
/** /**
* Configure comment handling behavior * Configure comment handling behavior
*/ */
@ -136,6 +148,7 @@ public struct LexerConfig
StringBehavior stringBehavior; StringBehavior stringBehavior;
WhitespaceBehavior whitespaceBehavior; WhitespaceBehavior whitespaceBehavior;
CommentBehavior commentBehavior; CommentBehavior commentBehavior;
SpecialTokenBehavior specialTokenBehavior;
} }
public bool isBasicType(IdType type) nothrow pure @safe public bool isBasicType(IdType type) nothrow pure @safe
@ -412,8 +425,8 @@ public struct DLexer
private static bool isDocComment(string comment) pure nothrow @safe private static bool isDocComment(string comment) pure nothrow @safe
{ {
return comment.length >= 3 && (comment[2] == '/' return comment.length >= 3 && (comment[0 .. 3] == "///"
|| comment[2] == '*' || comment[2] == '+'); || comment[0 .. 3] == "/++" || comment[0 .. 3] == "/**");
} }
public void popFront() pure public void popFront() pure
@ -434,6 +447,7 @@ public struct DLexer
} }
do _popFront(); while (front == tok!"comment"); do _popFront(); while (front == tok!"comment");
if (front == tok!"whitespace") goto case tok!"whitespace"; if (front == tok!"whitespace") goto case tok!"whitespace";
if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence";
} }
break; break;
case tok!"whitespace": case tok!"whitespace":
@ -441,6 +455,15 @@ public struct DLexer
{ {
do _popFront(); while (front == tok!"whitespace"); do _popFront(); while (front == tok!"whitespace");
if (front == tok!"comment") goto case tok!"comment"; if (front == tok!"comment") goto case tok!"comment";
if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence";
}
break;
case tok!"specialTokenSequence":
if (config.specialTokenBehavior == SpecialTokenBehavior.skip)
{
do _popFront(); while (front == tok!"specialTokenSequence");
if (front == tok!"comment") goto case tok!"comment";
if (front == tok!"whitespace") goto case tok!"whitespace";
} }
break; break;
default: default:
@ -557,7 +580,7 @@ public struct DLexer
Token lexNumber() pure nothrow Token lexNumber() pure nothrow
{ {
mixin (tokenStart); mixin (tokenStart);
if (range.canPeek(1) && range.front == '0') if (range.front == '0' && range.canPeek(1))
{ {
auto ahead = range.peek(1)[1]; auto ahead = range.peek(1)[1];
switch (ahead) switch (ahead)
@ -742,13 +765,18 @@ public struct DLexer
// "double identifier". // "double identifier".
if (range.canPeek(1)) if (range.canPeek(1))
{ {
switch (range.peekAt(1)) auto ch = range.peekAt(1);
if (ch <= 0x2f
|| (ch >= '0' && ch <= '9')
|| (ch >= ':' && ch <= '@')
|| (ch >= '[' && ch <= '^')
|| (ch >= '{' && ch <= '~')
|| ch == '`' || ch == '_')
{ {
case '0': .. case '9':
goto doubleLiteral; goto doubleLiteral;
default:
break decimalLoop;
} }
else
break decimalLoop;
} }
else else
{ {

220
stdx/d/parser.d → std/d/parser.d
View File

@ -2,17 +2,17 @@
/** /**
* MACROS: * MACROS:
* GRAMMAR = <pre>$0</pre> * GRAMMAR = <pre class="grammar">$0</pre>
* RULEDEF = $(B $(DDOC_ANCHOR $0) $0) * RULEDEF = $(B $(DDOC_ANCHOR $0) $0)
* RULE = $(LINK2 #$0, $0) * RULE = $(LINK2 #$0, $0)
* LITERAL = $(D_STRING $(I $0)) * LITERAL = $(D_STRING $(I $0))
*/ */
module stdx.d.parser; module std.d.parser;
import stdx.d.lexer; import std.d.lexer;
import stdx.d.ast; import std.d.ast;
import stdx.allocator; import std.allocator;
import std.conv; import std.conv;
import std.algorithm; import std.algorithm;
import std.array; import std.array;
@ -23,113 +23,7 @@ import std.string : format;
// Caution: generates 180 megabytes of logging for std.datetime // Caution: generates 180 megabytes of logging for std.datetime
//version = std_parser_verbose; //version = std_parser_verbose;
/** alias ParseAllocator = CAllocatorImpl!(Mallocator);
* The parse allocator is designed so that very large number of node instances
* allocated by the parser can be deallocated all at once. This saves time by
* preventing the GC from having to scan the nodes.
*/
class ParseAllocator : CAllocator
{
public:
this(string name = "none")
{
this.name = name;
}
string name;
override void[] allocate(size_t size)
in
{
assert (size > 0);
assert (size < blockSize);
}
out (result)
{
assert (result.length == size);
}
body
{
enum size_t mask = ~ (cast(size_t) 7);
enum s = ((Node.sizeof - 1) & mask) + 8;
Node* current = root;
while (true)
{
while (current !is null)
{
immutable size_t blockLength = current.block.length;
immutable size_t oldUsed = current.used;
immutable size_t newUsed = oldUsed + size;
if (newUsed > blockLength)
current = current.next;
else
{
current.used = ((newUsed - 1) & mask) + 8;
// assert (current.used >= oldUsed + size);
// assert (current.block.ptr + blockSize > current.block.ptr + newUsed);
// assert (newUsed > oldUsed);
// writefln("Allocating 0x%012x - 0x%012x",
// cast(size_t) current.block.ptr + oldUsed,
// cast(size_t) current.block.ptr + newUsed);
current.block[oldUsed .. newUsed] = 0;
return current.block[oldUsed .. newUsed];
}
}
import core.memory;
// stderr.writeln("Allocating new block while processing ", name);
ubyte* newBlock = cast(ubyte*) GC.malloc(blockSize, GC.BlkAttr.NO_SCAN);
// assert (newBlock !is null);
// stderr.writefln("Memory spans from 0x%012x to 0x%012x",
// cast(size_t) newBlock, cast(size_t) newBlock + blockSize);
root = new Node (root, s, newBlock[0 .. blockSize]);
// assert (root.block.length == blockSize);
current = root;
}
assert (false);
}
/**
* Deallocates all memory held by this allocator. All node instances created
* by a parser using this allocator instance are invalid after calling this
* function.
*/
override bool deallocateAll()
{
deallocateRecursive(root);
root = null;
return true;
}
override bool expand(ref void[], size_t) { return false; }
override bool reallocate(ref void[], size_t) { return false; }
override bool deallocate(void[]) { return false; }
private:
void deallocateRecursive(Node* node)
{
import core.memory;
// import std.c.stdlib;
// node.block[] = 0;
// free(node.block.ptr);
GC.free(node.block.ptr);
if (node.next !is null)
deallocateRecursive(node.next);
node.next = null;
}
Node* root;
enum blockSize = 1024 * 1024 * 4;
struct Node
{
Node* next;
size_t used;
ubyte[] block;
}
}
/** /**
* Params: * Params:
@ -431,7 +325,7 @@ alias core.sys.posix.stdio.fileno fileno;
* Parses an ArrayLiteral * Parses an ArrayLiteral
* *
* $(GRAMMAR $(RULEDEF arrayLiteral): * $(GRAMMAR $(RULEDEF arrayLiteral):
* $(LITERAL '[') ($(RULE assignExpression) ($(LITERAL ',') $(RULE assignExpression))*)? $(LITERAL ']') * $(LITERAL '[') $(RULE argumentList)? $(LITERAL ']')
* ;) * ;)
*/ */
ArrayLiteral parseArrayLiteral() ArrayLiteral parseArrayLiteral()
@ -780,6 +674,8 @@ alias core.sys.posix.stdio.fileno fileno;
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto node = allocate!AssignExpression; auto node = allocate!AssignExpression;
node.line = current().line;
node.column = current().column;
node.ternaryExpression = parseTernaryExpression(); node.ternaryExpression = parseTernaryExpression();
if (currentIsOneOf(tok!"=", tok!">>>=", if (currentIsOneOf(tok!"=", tok!">>>=",
tok!">>=", tok!"<<=", tok!">>=", tok!"<<=",
@ -1574,6 +1470,8 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
auto t = expect(tok!"this"); auto t = expect(tok!"this");
if (t is null) return null; if (t is null) return null;
node.location = t.index; node.location = t.index;
node.line = t.line;
node.column = t.column;
auto p = peekPastParens(); auto p = peekPastParens();
bool isTemplate = false; bool isTemplate = false;
if (p !is null && p.type == tok!"(") if (p !is null && p.type == tok!"(")
@ -2068,7 +1966,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
* Parses a Destructor * Parses a Destructor
* *
* $(GRAMMAR $(RULEDEF destructor): * $(GRAMMAR $(RULEDEF destructor):
* $(LITERAL '~') $(LITERAL 'this') $(LITERAL '$(LPAREN)') $(LITERAL '$(RPAREN)') ($(RULE functionBody) | $(LITERAL ';')) * $(LITERAL '~') $(LITERAL 'this') $(LITERAL '$(LPAREN)') $(LITERAL '$(RPAREN)') $(RULE memberFunctionAttribute)* ($(RULE functionBody) | $(LITERAL ';'))
* ;) * ;)
*/ */
Destructor parseDestructor() Destructor parseDestructor()
@ -2078,13 +1976,22 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
node.comment = comment; node.comment = comment;
comment = null; comment = null;
if (expect(tok!"~") is null) return null; if (expect(tok!"~") is null) return null;
node.index = current.index;
node.line = current.line;
node.column = current.column;
if (expect(tok!"this") is null) return null; if (expect(tok!"this") is null) return null;
if (expect(tok!"(") is null) return null; if (expect(tok!"(") is null) return null;
if (expect(tok!")") is null) return null; if (expect(tok!")") is null) return null;
if (currentIs(tok!";")) if (currentIs(tok!";"))
advance(); advance();
else else
{
MemberFunctionAttribute[] memberFunctionAttributes;
while(moreTokens() && currentIsMemberFunctionAttribute())
memberFunctionAttributes ~= parseMemberFunctionAttribute();
node.memberFunctionAttributes = ownArray(memberFunctionAttributes);
node.functionBody = parseFunctionBody(); node.functionBody = parseFunctionBody();
}
return node; return node;
} }
@ -2377,7 +2284,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
{ {
if (!canBeRange) if (!canBeRange)
{ {
error(`Cannot have more than one foreach varible for a foreach range statement`); error(`Cannot have more than one foreach variable for a foreach range statement`);
return null; return null;
} }
advance(); advance();
@ -2878,6 +2785,8 @@ body {} // six
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto node = allocate!IfStatement; auto node = allocate!IfStatement;
node.line = current().line;
node.column = current().column;
if (expect(tok!"if") is null) return null; if (expect(tok!"if") is null) return null;
node.startIndex = current().index; node.startIndex = current().index;
if (expect(tok!"(") is null) return null; if (expect(tok!"(") is null) return null;
@ -3278,6 +3187,8 @@ interface "Four"
Invariant parseInvariant() Invariant parseInvariant()
{ {
auto node = allocate!Invariant; auto node = allocate!Invariant;
node.index = current.index;
node.line = current.line;
if (expect(tok!"invariant") is null) return null; if (expect(tok!"invariant") is null) return null;
if (currentIs(tok!"(")) if (currentIs(tok!"("))
{ {
@ -3491,7 +3402,7 @@ invariant() foo();
* Parses a LinkageAttribute * Parses a LinkageAttribute
* *
* $(GRAMMAR $(RULEDEF linkageAttribute): * $(GRAMMAR $(RULEDEF linkageAttribute):
* $(LITERAL 'extern') $(LITERAL '$(LPAREN)') $(LITERAL Identifier) $(LITERAL '++')? $(LITERAL '$(RPAREN)') * $(LITERAL 'extern') $(LITERAL '$(LPAREN)') $(LITERAL Identifier) ($(LITERAL '++') ($(LITERAL ',') $(RULE identifierChain))?)? $(LITERAL '$(RPAREN)')
* ;) * ;)
*/ */
LinkageAttribute parseLinkageAttribute() LinkageAttribute parseLinkageAttribute()
@ -3507,6 +3418,11 @@ invariant() foo();
{ {
advance(); advance();
node.hasPlusPlus = true; node.hasPlusPlus = true;
version(DIP61) if (currentIs(tok!","))
{
advance();
node.identifierChain = parseIdentifierChain();
}
} }
expect(tok!")"); expect(tok!")");
return node; return node;
@ -3645,7 +3561,7 @@ invariant() foo();
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
Module m = allocate!Module; Module m = allocate!Module;
if (currentIs(tok!"scriptLine")) if (currentIs(tok!"scriptLine"))
advance(); m.scriptLine = advance();
if (currentIs(tok!"module")) if (currentIs(tok!"module"))
m.moduleDeclaration = parseModuleDeclaration(); m.moduleDeclaration = parseModuleDeclaration();
while (moreTokens()) while (moreTokens())
@ -3879,6 +3795,7 @@ invariant() foo();
* $(RULE assignExpression) * $(RULE assignExpression)
* | $(RULE arrayInitializer) * | $(RULE arrayInitializer)
* | $(RULE structInitializer) * | $(RULE structInitializer)
* | $(RULE functionBody)
* ;) * ;)
*/ */
NonVoidInitializer parseNonVoidInitializer() NonVoidInitializer parseNonVoidInitializer()
@ -3890,6 +3807,8 @@ invariant() foo();
auto b = peekPastBraces(); auto b = peekPastBraces();
if (b !is null && (b.type == tok!"(")) if (b !is null && (b.type == tok!"("))
node.assignExpression = parseAssignExpression(); node.assignExpression = parseAssignExpression();
else if (hasMagicDelimiter!(tok!"{", tok!";")())
node.functionBody = parseFunctionBody();
else else
node.structInitializer = parseStructInitializer(); node.structInitializer = parseStructInitializer();
} }
@ -3907,8 +3826,15 @@ invariant() foo();
else else
node.assignExpression = parseAssignExpression(); node.assignExpression = parseAssignExpression();
} }
else if (currentIsOneOf(tok!"in", tok!"out", tok!"body"))
node.functionBody = parseFunctionBody();
else else
node.assignExpression = parseAssignExpression(); node.assignExpression = parseAssignExpression();
if (node.assignExpression is null && node.arrayInitializer is null
&& node.structInitializer is null && node.functionBody is null)
{
return null;
}
return node; return node;
} }
@ -4508,10 +4434,14 @@ q{(int a, ...)
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto node = allocate!ReturnStatement; auto node = allocate!ReturnStatement;
if (expect(tok!"return") is null) return null; auto start = expect(tok!"return");
if (start is null) return null;
node.startLocation = start.index;
if (!currentIs(tok!";")) if (!currentIs(tok!";"))
node.expression = parseExpression(); node.expression = parseExpression();
if (expect(tok!";") is null) return null; auto semicolon = expect(tok!";");
if (semicolon is null) return null;
node.endLocation = semicolon.index;
return node; return node;
} }
@ -4619,8 +4549,18 @@ q{(int a, ...)
if (!currentIs(tok!"]")) if (!currentIs(tok!"]"))
{ {
node.lower = parseAssignExpression(); node.lower = parseAssignExpression();
if (node.lower is null)
{
error("assignExpression expected");
return null;
}
expect(tok!".."); expect(tok!"..");
node.upper = parseAssignExpression(); node.upper = parseAssignExpression();
if (node.upper is null)
{
error("assignExpression expected");
return null;
}
} }
if (expect(tok!"]") is null) return null; if (expect(tok!"]") is null) return null;
return node; return node;
@ -4874,8 +4814,13 @@ q{(int a, ...)
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto node = allocate!StructInitializer; auto node = allocate!StructInitializer;
expect(tok!"{"); expect(tok!"{");
if (currentIs(tok!"}"))
advance();
else
{
node.structMemberInitializers = parseStructMemberInitializers(); node.structMemberInitializers = parseStructMemberInitializers();
expect(tok!"}"); expect(tok!"}");
}
return node; return node;
} }
@ -6069,6 +6014,7 @@ q{doStuff(5)}c;
* *
* $(GRAMMAR $(RULEDEF variableDeclaration): * $(GRAMMAR $(RULEDEF variableDeclaration):
* $(RULE _type) $(RULE declarator) ($(LITERAL ',') $(RULE declarator))* $(LITERAL ';') * $(RULE _type) $(RULE declarator) ($(LITERAL ',') $(RULE declarator))* $(LITERAL ';')
* | $(RULE _type) $(RULE declarator) $(LITERAL '=') $(RULE functionBody)
* | $(RULE autoDeclaration) * | $(RULE autoDeclaration)
* ;) * ;)
*/ */
@ -6096,9 +6042,19 @@ q{doStuff(5)}c;
break; break;
} }
node.declarators = ownArray(declarators); node.declarators = ownArray(declarators);
// if (node.declarators.length == 1
// && node.declarators[0].initializer !is null
// && node.declarators[0].initializer.nonVoidInitializer !is null
// && node.declarators[0].initializer.nonVoidInitializer.functionBody !is null)
// {
// return node;
// }
// else
{
expect(tok!";"); expect(tok!";");
return node; return node;
} }
}
/** /**
* Parses a Vector * Parses a Vector
@ -6248,7 +6204,7 @@ q{doStuff(5)}c;
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
if (startsWith(tok!"[", tok!"]")) if (startsWith(tok!"[", tok!"]"))
return true; return true;
return hasMagicDelimiter!(tok!"..")(); return hasMagicDelimiter!(tok!"[", tok!"..")();
} }
void setTokens(const(Token)[] tokens) void setTokens(const(Token)[] tokens)
@ -6271,6 +6227,7 @@ protected:
if (allocator is null) if (allocator is null)
return from; return from;
T[] to = cast(T[]) allocator.allocate(T.sizeof * from.length); T[] to = cast(T[]) allocator.allocate(T.sizeof * from.length);
assert (to.length == from.length, format("from.length = %d, to.length = %d", from.length, to.length));
to[] = from[]; to[] = from[];
return to; return to;
} }
@ -6282,6 +6239,7 @@ protected:
return new T(args); return new T(args);
enum numBytes = __traits(classInstanceSize, T); enum numBytes = __traits(classInstanceSize, T);
void[] mem = allocator.allocate(numBytes); void[] mem = allocator.allocate(numBytes);
assert (mem.length == numBytes, format("%d", mem.length));
T t = emplace!T(mem, args); T t = emplace!T(mem, args);
assert (cast(void*) t == mem.ptr, "%x, %x".format(cast(void*) t, mem.ptr)); assert (cast(void*) t == mem.ptr, "%x, %x".format(cast(void*) t, mem.ptr));
return t; return t;
@ -6306,22 +6264,25 @@ protected:
bool isAssociativeArrayLiteral() bool isAssociativeArrayLiteral()
{ {
return hasMagicDelimiter!(tok!":")(); auto b = setBookmark();
scope(exit) goToBookmark(b);
advance();
return !currentIs(tok!"]") && parseExpression() !is null && currentIs(tok!":");
} }
bool hasMagicDelimiter(alias T)() bool hasMagicDelimiter(alias L, alias T)()
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto i = index; auto i = index;
scope(exit) index = i; scope(exit) index = i;
assert(currentIs(tok!"[")); assert(currentIs(L));
advance(); advance();
while (moreTokens()) switch (current.type) while (moreTokens()) switch (current.type)
{ {
case tok!"{": skipBraces(); break; case tok!"{": skipBraces(); break;
case tok!"(": skipParens(); break; case tok!"(": skipParens(); break;
case tok!"[": skipBrackets(); break; case tok!"[": skipBrackets(); break;
case tok!"]": return false; case tok!"]": case tok!"}": return false;
case T: return true; case T: return true;
default: advance(); break; default: advance(); break;
} }
@ -6392,8 +6353,9 @@ protected:
case tok!"abstract": case tok!"abstract":
case tok!"pure": case tok!"pure":
case tok!"nothrow": case tok!"nothrow":
mixin(BASIC_TYPE_CASES);
return true; return true;
mixin(BASIC_TYPE_CASES);
return !peekIs(tok!".");
case tok!"case": case tok!"case":
case tok!"default": case tok!"default":
case tok!"return": case tok!"return":
@ -6479,6 +6441,14 @@ protected:
return true; return true;
} }
return true; return true;
case tok!"pragma":
auto b = setBookmark();
scope(exit) goToBookmark(b);
advance();
auto past = peekPastParens();
if (past is null || *past == tok!";")
return false;
return true;
case tok!"deprecated": case tok!"deprecated":
case tok!"private": case tok!"private":
case tok!"package": case tok!"package":

39
stdx/lexer.d → std/lexer.d
View File

@ -33,7 +33,7 @@
* ) * )
* Examples: * Examples:
* $(UL * $(UL
* $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/stdx/d/lexer.d, here).) * $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/std/d/lexer.d, here).)
* $(LI A _lexer for Lua is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/lualexer.d, here).) * $(LI A _lexer for Lua is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/lualexer.d, here).)
* $(LI A _lexer for JSON is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/jsonlexer.d, here).) * $(LI A _lexer for JSON is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/jsonlexer.d, here).)
* ) * )
@ -112,7 +112,7 @@
* Source: $(PHOBOSSRC std/_lexer.d) * Source: $(PHOBOSSRC std/_lexer.d)
*/ */
module stdx.lexer; module std.lexer;
/** /**
* Template for determining the type used for a token type. Selects the smallest * Template for determining the type used for a token type. Selects the smallest
@ -249,6 +249,11 @@ struct TokenStructure(IdType, string extraFields = "")
{ {
public: public:
bool opEquals(ref const typeof(this) other) const pure nothrow @safe
{
return this.type == other.type && this.text == other.text;
}
/** /**
* Returs: true if the token has the given type, false otherwise. * Returs: true if the token has the given type, false otherwise.
*/ */
@ -420,6 +425,7 @@ mixin template Lexer(Token, alias defaultTokenFunction,
private static string generateCaseStatements() private static string generateCaseStatements()
{ {
import std.algorithm;
import std.conv; import std.conv;
import std.string; import std.string;
import std.range; import std.range;
@ -443,6 +449,7 @@ mixin template Lexer(Token, alias defaultTokenFunction,
private static string printCase(string[] tokens, string[] pseudoTokens) private static string printCase(string[] tokens, string[] pseudoTokens)
{ {
import std.algorithm;
string[] t = tokens; string[] t = tokens;
string[] sortedTokens = stupidToArray(sort!"a.length > b.length"(t)); string[] sortedTokens = stupidToArray(sort!"a.length > b.length"(t));
import std.conv; import std.conv;
@ -727,7 +734,8 @@ struct LexerRange
} }
/** /**
* Implements the algorithm _popFrontN more efficiently. * Implements the algorithm _popFrontN more efficiently. This function does
* not detect or handle newlines.
*/ */
void popFrontN(size_t n) pure nothrow @safe void popFrontN(size_t n) pure nothrow @safe
{ {
@ -765,9 +773,6 @@ struct LexerRange
size_t line; size_t line;
} }
/**
* FREAKIN' MAAAGIC
*/
shared struct StringCache shared struct StringCache
{ {
import core.sync.mutex; import core.sync.mutex;
@ -784,6 +789,20 @@ public:
allocating = false; allocating = false;
} }
~this()
{
import core.memory;
shared(Block)* current = rootBlock;
while (current !is null)
{
shared(Block)* prev = current;
current = current.next;
free(cast(void*) prev.bytes.ptr);
}
rootBlock = null;
buckets = [];
}
/** /**
* Caches a string. * Caches a string.
* Params: str = the string to intern * Params: str = the string to intern
@ -854,7 +873,6 @@ private:
if (bytes is null || bytes.length == 0) if (bytes is null || bytes.length == 0)
return ""; return "";
import core.atomic; import core.atomic;
import core.memory;
shared ubyte[] mem; shared ubyte[] mem;
shared(Node*)* oldBucketRoot = &buckets[hash % buckets.length]; shared(Node*)* oldBucketRoot = &buckets[hash % buckets.length];
while (true) while (true)
@ -928,7 +946,7 @@ private:
import core.atomic; import core.atomic;
import core.memory; import core.memory;
if (numBytes > (blockSize / 4)) if (numBytes > (blockSize / 4))
return cast(shared) (cast(ubyte*) GC.malloc(numBytes, GC.BlkAttr.NO_SCAN))[0 .. numBytes]; return cast(shared) (cast(ubyte*) malloc(numBytes))[0 .. numBytes];
shared(Block)* r = rootBlock; shared(Block)* r = rootBlock;
while (true) while (true)
{ {
@ -949,7 +967,7 @@ private:
if (cas(&allocating, false, true)) if (cas(&allocating, false, true))
{ {
shared(Block)* b = new shared Block( shared(Block)* b = new shared Block(
cast(shared) (cast(ubyte*) GC.malloc(blockSize, GC.BlkAttr.NO_SCAN))[0 .. blockSize], cast(shared) (cast(ubyte*) malloc(blockSize))[0 .. blockSize],
numBytes, numBytes,
r); r);
atomicStore(rootBlock, b); atomicStore(rootBlock, b);
@ -1048,3 +1066,6 @@ private:
shared(Node)*[] buckets; shared(Node)*[] buckets;
shared(Block)* rootBlock; shared(Block)* rootBlock;
} }
private extern(C) void* malloc(size_t) nothrow pure;
private extern(C) void free(void*) nothrow pure;