forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/Hackerpilot/Dscanner

This commit is contained in:
Hackerpilot 2014-05-08 02:45:07 -07:00
parent 22bacd83ce d97a2c5f34
commit 017b90e31f
31 changed files with 1036 additions and 423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@ -19,7 +19,7 @@ The following examples assume that we are analyzing a simple file called hellowo
### Token Count
The "--tokenCount" or "-t" option prints the number of tokens in the given file
$ dscanner --tokencount helloworld.d
$ dscanner --tokenCount helloworld.d
20
### Import Listing
@ -52,13 +52,15 @@ given source files.
* Left side of a *foreach* or *foreach\_reverse* range expression is larger than the right.
* Left side of a slice expression is larger than the right
* Variable, struct, class, union, module, package, and interface names that do not comply with Phobos style guidelines
* Struct constructors that have a single parameter that has a default argument.
* Assign expressions where the left side of the '=' operator is the same as the right
* 'if' statements where the 'else' block is the same as the 'if' block.
#### Wishlish
* Assigning to foreach variables that are not "ref".
* Unused variables.
* Unused imports.
* Unused parameters (check is skipped if function is marked "override")
* Struct constructors that have a single parameter that has a default argument.
* Variables that are never modified and not declared immutable.
* Public declarations not documented
* Declaring opEquals without toHash
@ -196,6 +198,6 @@ but not yet implemented.
# Useful code
The source code for DScanner has a complete lexer, parser, and abstact syntax
tree library for D code under the stdx/d/ directory. It is intended that these
tree library for D code under the std/d/ directory. It is intended that these
modules eventually end up in Phobos, so feel free to use them for your own D
tools.

2
analysis/base.d
View File

@ -2,7 +2,7 @@ module analysis.base;
import std.container;
import std.string;
import stdx.d.ast;
import std.d.ast;
import std.array;
struct Message

86
analysis/constructors.d Normal file
View File

@ -0,0 +1,86 @@
module analysis.constructors;
import std.d.ast;
import std.d.lexer;
import analysis.base;
class ConstructorCheck : BaseAnalyzer
{
alias visit = BaseAnalyzer.visit;
this(string fileName)
{
super(fileName);
}
override void visit(const ClassDeclaration classDeclaration)
{
bool oldHasDefault = hasDefaultArgConstructor;
bool oldHasNoArg = hasNoArgConstructor;
hasNoArgConstructor = false;
hasDefaultArgConstructor = false;
State prev = state;
state = State.inClass;
classDeclaration.accept(this);
if (hasNoArgConstructor && hasDefaultArgConstructor)
{
addErrorMessage(classDeclaration.name.line,
classDeclaration.name.column, "This class has a zero-argument"
~ " constructor as well as a constructor with one default"
~ " argument. This can be confusing");
}
hasDefaultArgConstructor = oldHasDefault;
hasNoArgConstructor = oldHasNoArg;
state = prev;
}
override void visit(const StructDeclaration structDeclaration)
{
State prev = state;
state = State.inStruct;
structDeclaration.accept(this);
state = prev;
}
override void visit(const Constructor constructor)
{
final switch (state)
{
case State.inStruct:
if (constructor.parameters.parameters.length == 1
&& constructor.parameters.parameters[0].default_ !is null)
{
addErrorMessage(constructor.line, constructor.column,
"This struct constructor can never be called with its "
~ "default argument.");
}
break;
case State.inClass:
if (constructor.parameters.parameters.length == 1
&& constructor.parameters.parameters[0].default_ !is null)
{
hasDefaultArgConstructor = true;
}
else if (constructor.parameters.parameters.length == 0)
hasNoArgConstructor = true;
break;
case State.ignoring:
break;
}
}
private:
enum State: ubyte
{
ignoring,
inClass,
inStruct
}
State state;
bool hasNoArgConstructor;
bool hasDefaultArgConstructor;
}

4
analysis/del.d
View File

@ -5,8 +5,8 @@
module analysis.del;
import stdx.d.ast;
import stdx.d.lexer;
import std.d.ast;
import std.d.lexer;
import analysis.base;
/**

4
analysis/enumarrayliteral.d
View File

@ -5,8 +5,8 @@
module analysis.enumarrayliteral;
import stdx.d.ast;
import stdx.d.lexer;
import std.d.ast;
import std.d.lexer;
import analysis.base;
void doNothing(string, size_t, size_t, string, bool) {}

5
analysis/fish.d
View File

@ -5,8 +5,8 @@
module analysis.fish;
import stdx.d.ast;
import stdx.d.lexer;
import std.d.ast;
import std.d.lexer;
import analysis.base;
/**
@ -24,6 +24,7 @@ class FloatOperatorCheck : BaseAnalyzer
override void visit(const RelExpression r)
{
if (r.operator == tok!"<>"
|| r.operator == tok!"<>="
|| r.operator == tok!"!<>"
|| r.operator == tok!"!>"
|| r.operator == tok!"!<"

43
analysis/ifelsesame.d Normal file
View File

@ -0,0 +1,43 @@
// Copyright Brian Schott (Sir Alaran) 2014.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
module analysis.ifelsesame;
import std.d.ast;
import std.d.lexer;
import analysis.base;
/**
* Checks for if statements whose "then" block is the same as the "else" block
*/
class IfElseSameCheck : BaseAnalyzer
{
alias visit = BaseAnalyzer.visit;
this(string fileName)
{
super(fileName);
}
override void visit(const IfStatement ifStatement)
{
if (ifStatement.thenStatement == ifStatement.elseStatement)
addErrorMessage(ifStatement.line, ifStatement.column,
"\"Else\" branch is identical to \"Then\" branch.");
ifStatement.accept(this);
}
override void visit(const AssignExpression assignExpression)
{
const AssignExpression e = cast(const AssignExpression) assignExpression.assignExpression;
if (e !is null && assignExpression.operator == tok!"="
&& e.ternaryExpression == assignExpression.ternaryExpression)
{
addErrorMessage(assignExpression.line, assignExpression.column,
"Left side of assignment operatior is identical to the right side");
}
assignExpression.accept(this);
}
}

1
analysis/linespan.d
View File

@ -5,7 +5,6 @@
module analysis.linespan;
/**
* Used for determining which lines to include as context in the generated HTML
* report.

4
analysis/numbers.d
View File

@ -6,8 +6,8 @@
module analysis.numbers;
import std.regex;
import stdx.d.ast;
import stdx.d.lexer;
import std.d.ast;
import std.d.lexer;
import analysis.base;
/**

13
analysis/objectconst.d
View File

@ -6,12 +6,13 @@
module analysis.objectconst;
import std.regex;
import stdx.d.ast;
import stdx.d.lexer;
import std.d.ast;
import std.d.lexer;
import analysis.base;
/**
* Checks for use of the deprecated "delete" keyword
* Checks that opEquals, opCmp, toHash, and toString are either const,
* immutable, or inout.
*/
class ObjectConstCheck : BaseAnalyzer
{
@ -35,7 +36,7 @@ class ObjectConstCheck : BaseAnalyzer
&& !hasConst(d.functionDeclaration.memberFunctionAttributes)))
{
addErrorMessage(d.functionDeclaration.name.line,
d.functionDeclaration.name.column, "opCmp, ToHash, opEquals,"
d.functionDeclaration.name.column, "opCmp, toHash, opEquals,"
~ " and toString should be declared const");
}
d.accept(this);
@ -51,7 +52,9 @@ class ObjectConstCheck : BaseAnalyzer
private static bool hasConst(const MemberFunctionAttribute[] attributes)
{
import std.algorithm;
return attributes.any!(a => a.tokenType == tok!"const");
return attributes.any!(a => a.tokenType == tok!"const"
|| a.tokenType == tok!"immutable"
|| a.tokenType == tok!"inout");
}
private static bool isInteresting(string name)

4
analysis/pokemon.d
View File

@ -5,8 +5,8 @@
module analysis.pokemon;
import stdx.d.ast;
import stdx.d.lexer;
import std.d.ast;
import std.d.lexer;
import analysis.base;
/**

25
analysis/range.d
View File

@ -5,8 +5,8 @@
module analysis.range;
import stdx.d.ast;
import stdx.d.lexer;
import std.d.ast;
import std.d.lexer;
import analysis.base;
/**
@ -64,24 +64,37 @@ class BackwardsRangeCheck : BaseAnalyzer
override void visit(const PrimaryExpression primary)
{
import std.conv;
import std.string;
if (state == State.ignore || !isNumberLiteral(primary.primary.type))
return;
if (state == State.left)
{
line = primary.primary.line;
this.column = primary.primary.column;
left = to!long(primary.primary.text.removechars("_uUlL"));
left = parseNumber(primary.primary.text);
hasLeft = true;
}
else
{
right = to!long(primary.primary.text.removechars("_uUlL"));
right = parseNumber(primary.primary.text);
hasRight = true;
}
}
long parseNumber(string te)
{
import std.conv;
import std.string;
string t = te.removechars("_uUlL");
if (t.length > 2)
{
if (t[1] == 'x' || t[1] == 'X')
return to!long(t[2..$], 16);
if (t[1] == 'b' || t[1] == 'B')
return to!long(t[2..$], 2);
}
return to!long(t);
}
override void visit(const SliceExpression sliceExpression)
{
if (sliceExpression.lower !is null && sliceExpression.upper !is null)

11
analysis/run.d
View File

@ -6,10 +6,9 @@ import std.conv;
import std.algorithm;
import std.range;
import std.array;
import stdx.d.lexer;
import stdx.d.parser;
import stdx.d.ast;
import std.d.lexer;
import std.d.parser;
import std.d.ast;
import analysis.base;
import analysis.style;
@ -20,6 +19,8 @@ import analysis.fish;
import analysis.numbers;
import analysis.objectconst;
import analysis.range;
import analysis.constructors;
import analysis.ifelsesame;
void messageFunction(string fileName, size_t line, size_t column, string message,
bool isError)
@ -70,6 +71,8 @@ void analyze(File output, string[] fileNames, bool staticAnalyze = true)
checks ~= new NumberStyleCheck(fileName);
checks ~= new ObjectConstCheck(fileName);
checks ~= new BackwardsRangeCheck(fileName);
checks ~= new IfElseSameCheck(fileName);
checks ~= new ConstructorCheck(fileName);
foreach (check; checks)
{

4
analysis/style.d
View File

@ -5,8 +5,8 @@
module analysis.style;
import stdx.d.ast;
import stdx.d.lexer;
import std.d.ast;
import std.d.lexer;
import std.regex;
import std.array;
import std.conv;

23
astprinter.d
View File

@ -3,8 +3,8 @@
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
import stdx.d.lexer;
import stdx.d.ast;
import std.d.lexer;
import std.d.ast;
import std.stdio;
import std.string;
import std.array;
@ -133,7 +133,7 @@ class XMLPrinter : ASTVisitor
output.writeln("<assignExpression>");
else
output.writeln("<assignExpression operator=\"",
str(assignExpression.operator), "\">");
xmlEscape(str(assignExpression.operator)), "\">");
assignExpression.accept(this);
output.writeln("</assignExpression>");
}
@ -479,7 +479,7 @@ class XMLPrinter : ASTVisitor
override void visit(const ForStatement forStatement)
{
output.writeln("<forStatement>");
if (forStatement.declarationOrStatement !is null)
if (forStatement.initialization !is null)
{
output.writeln("<initialization>");
visit(forStatement.initialization);
@ -497,7 +497,8 @@ class XMLPrinter : ASTVisitor
visit(forStatement.increment);
output.writeln("</increment>");
}
visit(forStatement.declarationOrStatement);
if (forStatement.declarationOrStatement !is null)
visit(forStatement.declarationOrStatement);
output.writeln("</forStatement>");
}
@ -592,7 +593,8 @@ class XMLPrinter : ASTVisitor
{
output.writeln("<gotoStatement>");
output.writeln("<case>");
visit(gotoStatement.expression);
if (gotoStatement.expression)
visit(gotoStatement.expression);
output.writeln("</case>");
output.writeln("</gotoStatement>");
}
@ -665,10 +667,10 @@ class XMLPrinter : ASTVisitor
override void visit(const ImportBind importBind)
{
if (importBind.right.type == tok!"")
output.writeln("<importBind symbol=\"", importBind.left.text, "\">");
output.writeln("<importBind symbol=\"", importBind.left.text, "\"/>");
else
output.writeln("<importBind symbol=\"", importBind.right.text,
"\" rename=\"", importBind.left.text, "\">");
"\" rename=\"", importBind.left.text, "\"/>");
}
override void visit(const ImportBindings importBindings)
@ -779,7 +781,7 @@ class XMLPrinter : ASTVisitor
output.writeln("</key>");
output.writeln("<value>");
visit(keyValuePair.value);
output.writeln("<value>");
output.writeln("</value>");
output.writeln("</keyValuePair>");
}
@ -1312,8 +1314,9 @@ class XMLPrinter : ASTVisitor
case tok!"stringLiteral": tagName = "stringLiteral"; break;
case tok!"dstringLiteral": tagName = "dstringLiteral"; break;
case tok!"wstringLiteral": tagName = "wstringLiteral"; break;
case tok!"scriptLine": tagName = "scriptLine"; break;
case tok!"$": output.writeln("<dollar/>"); return;
default: output.writeln("<", str(token.type), "/>"); return;
default: tagName = "token"; break;
}
output.writeln("<", tagName, ">", xmlEscape(token.text), "</", tagName, ">");
}

2
build.bat
View File

@ -1 +1 @@
dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/allocator.d stdx/lexer.d stdx/d/ast.d stdx/d/parser.d stdx/d/lexer.d analysis/base.d analysis/del.d analysis/enumarrayliteral.d analysis/fish.d analysis/numbers.d analysis/objectconst.d analysis/package.d analysis/pokemon.d analysis/range.d analysis/run.d analysis/style.d -ofdscanner.exe -O -release -noboundscheck -inline
dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d std/allocator.d std/lexer.d std/d/ast.d std/d/parser.d std/d/lexer.d analysis/base.d analysis/del.d analysis/enumarrayliteral.d analysis/constructors.d analysis/ifelsesame.d analysis/fish.d analysis/numbers.d analysis/objectconst.d analysis/package.d analysis/pokemon.d analysis/range.d analysis/run.d analysis/style.d -ofdscanner.exe -version=DIP61 -O -release -noboundscheck -inline

17
build.sh
View File

@ -7,12 +7,13 @@ dmd\
astprinter.d\
formatter.d\
outliner.d\
stdx/*.d\
stdx/d/*.d\
std/*.d\
std/d/*.d\
analysis/*.d\
-version=DIP61\
-ofdscanner\
-m64 -g\
-O -release -noboundscheck -inline
-g\
-O -release
#gdc\
# main.d\
@ -23,8 +24,8 @@ dmd\
# astprinter.d\
# formatter.d\
# outliner.d\
# stdx/*.d\
# stdx/d/*.d\
# std/*.d\
# std/d/*.d\
# analysis/*.d\
# -O3 -frelease -fno-bounds-check\
# -odscanner\
@ -38,8 +39,8 @@ dmd\
# astprinter.d\
# formatter.d\
# outliner.d\
# stdx/*.d\
# stdx/d/*.d\
# std/*.d\
# std/d/*.d\
# analysis/*.d\
# -O3 -release\
# -oq -of=dscanner\

35
ctags.d
View File

@ -5,9 +5,9 @@
module ctags;
import stdx.d.parser;
import stdx.d.lexer;
import stdx.d.ast;
import std.d.parser;
import std.d.lexer;
import std.d.ast;
import std.algorithm;
import std.range;
import std.stdio;
@ -71,7 +71,9 @@ class CTagsPrinter : ASTVisitor
override void visit(const TemplateDeclaration dec)
{
tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.text, fileName, dec.name.line, context);
auto name = dec.eponymousTemplateDeclaration is null ? dec.name
: dec.eponymousTemplateDeclaration.name;
tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(name.text, fileName, name.line, context);
auto c = context;
context = "\ttemplate:" ~ dec.name.text;
dec.accept(this);
@ -88,6 +90,26 @@ class CTagsPrinter : ASTVisitor
context = c;
}
override void visit(const Constructor dec)
{
tagLines ~= "this\t%s\t%d;\"\tf\tarity:%d%s\n".format(fileName,
dec.line, dec.parameters.parameters.length, context);
auto c = context;
context = "\tfunction: this";
dec.accept(this);
context = c;
}
override void visit(const Destructor dec)
{
tagLines ~= "~this\t%s\t%d;\"\tf%s\n".format(fileName, dec.line,
context);
auto c = context;
context = "\tfunction: this";
dec.accept(this);
context = c;
}
override void visit(const EnumDeclaration dec)
{
if (dec.name == tok!"")
@ -134,6 +156,11 @@ class CTagsPrinter : ASTVisitor
dec.accept(this);
}
override void visit(const Invariant dec)
{
tagLines ~= "invariant\t%s\t%d;\"\tv%s\n".format(fileName, dec.line, context);
}
alias visit = ASTVisitor.visit;
string fileName;

2
dub.json
View File

@ -9,7 +9,7 @@
{
"name": "library",
"targetType": "library",
"sourcePaths": ["stdx"],
"sourcePaths": ["std"],
},
{
"name": "dscanner",

12
formatter.d
View File

@ -1,7 +1,7 @@
module formatter;
import stdx.d.ast;
import stdx.d.lexer;
import std.d.ast;
import std.d.lexer;
/**
* The only brace styles worth using.
@ -489,6 +489,7 @@ class Formatter(Sink)
foreach (attribute; parameter.parameterAttributes)
{
sink.put(str(attribute));
sink.put(" ");
}
if (parameter.type !is null)
format(parameter.type);
@ -682,16 +683,11 @@ class Formatter(Sink)
void format(const Type type)
{
bool first = true;
foreach (constructor; type.typeConstructors)
{
if (first)
sink.put(" ");
first = false;
sink.put(str(constructor));
sink.put(" ");
}
if (type.typeConstructors.length > 0)
sink.put(" ");
format(type.type2);
foreach (suffix; type.typeSuffixes)
{

4
highlighter.d
View File

@ -8,7 +8,7 @@ module highlighter;
import std.stdio;
import std.array;
import stdx.d.lexer;
import std.d.lexer;
// http://ethanschoonover.com/solarized
void highlight(R)(ref R tokens, string fileName)
@ -49,6 +49,8 @@ html { background-color: #fdf6e3; color: #002b36; }
writeSpan("num", t.text);
else if (isOperator(t.type))
writeSpan("op", str(t.type));
else if (t.type == tok!"specialTokenSequence" || t.type == tok!"scriptLine")
writeSpan("cons", t.text.replace("<", "&lt;"));
else
{
version(Windows)

17
imports.d
View File

@ -5,16 +5,21 @@
module imports;
import stdx.d.ast;
import std.d.ast;
import std.stdio;
import std.container;
class ImportPrinter : ASTVisitor
{
this()
{
imports = new RedBlackTree!string;
}
override void visit(const SingleImport singleImport)
{
ignore = false;
singleImport.accept(this);
writeln();
ignore = true;
}
@ -22,15 +27,19 @@ class ImportPrinter : ASTVisitor
{
if (ignore) return;
bool first = true;
string s;
foreach (ident; identifierChain.identifiers)
{
if (!first)
write(".");
write(ident.text);
s ~= ".";
s ~= ident.text;
first = false;
}
imports.insert(s);
}
RedBlackTree!string imports;
alias visit = ASTVisitor.visit;
bool ignore = true;

14
main.d
View File

@ -13,9 +13,9 @@ import std.getopt;
import std.path;
import std.stdio;
import std.range;
import stdx.lexer;
import stdx.d.lexer;
import stdx.d.parser;
import std.lexer;
import std.d.lexer;
import std.d.parser;
import highlighter;
import stats;
@ -101,6 +101,7 @@ int main(string[] args)
config.whitespaceBehavior = WhitespaceBehavior.include;
config.stringBehavior = StringBehavior.source;
config.commentBehavior = CommentBehavior.include;
config.specialTokenBehavior = SpecialTokenBehavior.include;
auto tokens = byToken(bytes, config, cache);
if (highlight)
{
@ -169,11 +170,14 @@ int main(string[] args)
else if (imports || ast || outline)
{
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1],
null, &doNothing);
if (imports)
{
auto visitor = new ImportPrinter;
visitor.visit(mod);
foreach (imp; visitor.imports[])
writeln(imp);
}
else if (ast)
{
@ -291,3 +295,5 @@ options:
directories and its sub-directories.`,
programName);
}
void doNothing(string, size_t, size_t, string, bool) {}

4
outliner.d
View File

@ -3,8 +3,8 @@
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
import stdx.d.lexer;
import stdx.d.ast;
import std.d.lexer;
import std.d.ast;
import std.stdio;
import std.string;
import std.array;

2
stats.d
View File

@ -7,7 +7,7 @@ module stats;
import std.stdio;
import std.algorithm;
import stdx.d.lexer;
import std.d.lexer;
pure nothrow bool isLineOfCode(IdType t)
{

389
stdx/allocator.d → std/allocator.d
View File

@ -246,7 +246,7 @@ uses an array of allocators, one per bucket, to satisfy requests.))
)
*/
module stdx.allocator;
module std.allocator;
// Example in the synopsis above
unittest
@ -422,6 +422,99 @@ unittest
static assert(stateSize!C3 == 2 * size_t.sizeof + char.sizeof);
}
/**
* Shortcut that encapsulates a cast and a call to emplace()
* Params:
* a = the allocator to use
* args = the arguments to $(D T)'s constructor
* Returns: a pointer to an instance of $(D T).
*/
T* allocate(T, Allocator, Args...)(auto ref Allocator a, auto ref Args args)
@trusted if (is (T == struct))
{
import std.conv : emplace;
void[] mem = a.allocate(T.sizeof);
return emplace(cast(T*) mem.ptr, args);
}
///
unittest
{
auto allocator = Mallocator.it;
struct TestStruct { int x = 5; }
TestStruct* p = allocate!TestStruct(allocator);
assert (p !is null);
assert (p.x == 5);
Mallocator.it.deallocate((cast(void*) p)[0 .. TestStruct.sizeof]);
}
/**
* Shortcut that encapsulates a cast and a call to emplace()
* Params:
* a = the allocator to use
* args = the arguments to $(D T)'s constructor
* Returns: a reference to an instance of $(D T).
*/
T allocate(T, Allocator, Args...)(auto ref Allocator a, auto ref Args args)
@trusted if (is (T == class))
{
import std.conv : emplace;
void[] mem = a.allocate(__traits(classInstanceSize, T));
return emplace!T(mem, args);
}
///
unittest
{
auto allocator = Mallocator.it;
class TestClass { int x; this(int x) { this.x = x; } }
TestClass tc = allocate!TestClass(allocator, 10);
assert (tc !is null);
assert (tc.x == 10);
Mallocator.it.deallocate((cast(void*) tc)[0 .. __traits(classInstanceSize, TestClass)]);
}
/**
* Encapsulates some casts and pointer slicing to deallocate $(D structInstance).
* This function does NOT call T's destructor.
*/
void deallocate(T, Allocator)(auto ref Allocator a, T* structInstance)
pure nothrow @trusted if (is (T == struct))
{
a.deallocate((cast(void*) structInstance)[0 .. T.sizeof]);
}
///
unittest
{
auto allocator = Mallocator.it;
bool d = false;
struct TestStruct { float f; }
TestStruct* ts = allocate!TestStruct(allocator);
deallocate(allocator, ts);
}
/**
* Encapsulates some casts and pointer slicing to deallocate $(D classInstance).
* This function does NOT call T's destructor.
*/
void deallocate(T, Allocator)(auto ref Allocator a, T classInstance)
pure nothrow @trusted if (is (T == class))
{
a.deallocate((cast(void*) classInstance)[0 .. __traits(classInstanceSize, T)]);
}
///
unittest
{
import std.math;
auto allocator = Mallocator.it;
class TestClass { double z; }
TestClass tc = allocate!TestClass(allocator);
assert (isnan(tc.z));
deallocate(allocator, tc);
}
/**
$(D chooseAtRuntime) is a compile-time constant of type $(D size_t) that several
parameterized structures in this module recognize to mean deferral to runtime of
@ -452,7 +545,7 @@ enum uint platformAlignment = std.algorithm.max(double.alignof, real.alignof);
The default good size allocation is deduced as $(D n) rounded up to the
allocator's alignment.
*/
size_t goodAllocSize(A)(auto ref A a, size_t n)
size_t goodAllocSize(A)(auto ref A a, size_t n) pure nothrow
{
return n.roundUpToMultipleOf(a.alignment);
}
@ -528,10 +621,16 @@ struct NullAllocator
No-op.
*/
void deallocateAll() shared { }
static shared(NullAllocator) it() pure nothrow @property @trusted
{
return cast(typeof(return)) _it;
}
/**
Returns the $(D shared) global instance of the $(D NullAllocator).
*/
static shared NullAllocator it;
private static shared const NullAllocator _it;
}
unittest
@ -557,16 +656,18 @@ struct GCAllocator
enum uint alignment = platformAlignment;
/**
Standard allocator methods per the semantics defined above. The $(D deallocate) and $(D reallocate) methods are $(D @system) because they may move memory around, leaving dangling pointers in user code.
Standard allocator methods per the semantics defined above. The
$(D deallocate) and $(D reallocate) methods are $(D @system) because they
may move memory around, leaving dangling pointers in user code.
*/
@trusted void[] allocate(size_t bytes) shared
@trusted void[] allocate(size_t bytes) shared nothrow pure
{
auto p = GC.malloc(bytes);
return p ? p[0 .. bytes] : null;
}
/// Ditto
@trusted bool expand(ref void[] b, size_t delta) shared
@trusted bool expand(ref void[] b, size_t delta) shared nothrow pure
{
auto newSize = GC.extend(b.ptr, b.length + delta,
b.length + delta);
@ -581,7 +682,7 @@ struct GCAllocator
}
/// Ditto
@system bool reallocate(ref void[] b, size_t newSize) shared
@system bool reallocate(ref void[] b, size_t newSize) shared nothrow pure
{
import core.exception : OutOfMemoryError;
try
@ -598,15 +699,20 @@ struct GCAllocator
}
/// Ditto
@system void deallocate(void[] b) shared
@system void deallocate(void[] b) shared nothrow pure
{
GC.free(b.ptr);
}
static shared(GCAllocator) it() pure nothrow @property @trusted
{
return cast(typeof(return)) _it;
}
/**
Returns the global instance of this allocator type. The garbage collected allocator is thread-safe, therefore all of its methods and $(D it) itself are $(D shared).
*/
static shared GCAllocator it;
private static shared const GCAllocator _it;
// Leave it undocummented for now.
@trusted void collect() shared
@ -629,12 +735,19 @@ unittest
assert(GCAllocator.it.expand(b, 1));
}
private extern (C)
{
void* malloc(size_t) pure nothrow @trusted;
void free(void*) pure nothrow @trusted;
void* realloc(void*, size_t) pure nothrow @trusted;
}
/**
The C heap allocator.
*/
struct Mallocator
{
private import core.stdc.stdlib;
// private import core.stdc.stdlib;
/**
The alignment is a static constant equal to $(D platformAlignment), which ensures proper alignment for any D data type.
@ -642,22 +755,26 @@ struct Mallocator
enum uint alignment = platformAlignment;
/**
Standard allocator methods per the semantics defined above. The $(D deallocate) and $(D reallocate) methods are $(D @system) because they may move memory around, leaving dangling pointers in user code. Somewhat paradoxically, $(D malloc) is $(D @safe) but that's only useful to safe programs that can afford to leak memory allocated.
Standard allocator methods per the semantics defined above. The
$(D deallocate) and $(D reallocate) methods are $(D @system) because they
may move memory around, leaving dangling pointers in user code. Somewhat
paradoxically, $(D malloc) is $(D @safe) but that's only useful to safe
programs that can afford to leak memory allocated.
*/
@trusted void[] allocate(size_t bytes) shared
void[] allocate(size_t bytes) shared pure nothrow @trusted
{
auto p = malloc(bytes);
return p ? p[0 .. bytes] : null;
}
/// Ditto
@system void deallocate(void[] b) shared
void deallocate(void[] b) shared pure nothrow @system
{
free(b.ptr);
}
/// Ditto
@system bool reallocate(ref void[] b, size_t s) shared
bool reallocate(ref void[] b, size_t s) shared pure nothrow @system
{
if (!s)
{
@ -673,10 +790,15 @@ struct Mallocator
return true;
}
static shared(Mallocator) it() pure nothrow @property @trusted
{
return cast(typeof(return)) _it;
}
/**
Returns the global instance of this allocator type. The C heap allocator is thread-safe, therefore all of its methods and $(D it) itself are $(D shared).
*/
static shared Mallocator it;
private static shared const Mallocator _it;
}
///
@ -854,7 +976,7 @@ unittest
/**
Returns s rounded up to a multiple of base.
*/
private size_t roundUpToMultipleOf(size_t s, uint base)
private size_t roundUpToMultipleOf(size_t s, uint base) pure nothrow @safe
{
assert(base);
auto rem = s % base;
@ -872,7 +994,7 @@ unittest
/**
Returns s rounded up to a multiple of base.
*/
private void[] roundStartToMultipleOf(void[] s, uint base)
private void[] roundStartToMultipleOf(void[] s, uint base) pure nothrow @trusted
{
assert(base);
auto p = cast(void*) roundUpToMultipleOf(
@ -892,7 +1014,7 @@ unittest
/**
Returns $(D s) rounded up to the nearest power of 2.
*/
private size_t roundUpToPowerOf2(size_t s)
private size_t roundUpToPowerOf2(size_t s) pure nothrow @safe
{
assert(s <= (size_t.max >> 1) + 1);
--s;
@ -943,7 +1065,7 @@ struct AffixAllocator(Allocator, Prefix, Suffix = void)
static assert(
!stateSize!Prefix || Allocator.alignment >= Prefix.alignof,
"AffixAllocator does not work with allocators offering a smaller"
" alignment than the prefix alignment.");
~ " alignment than the prefix alignment.");
static assert(alignment % Suffix.alignof == 0,
"This restriction could be relaxed in the future.");
@ -960,11 +1082,11 @@ struct AffixAllocator(Allocator, Prefix, Suffix = void)
parent allocator.
*/
static if (stateSize!Allocator) Allocator parent;
else alias Allocator.it parent;
else alias parent = Allocator.it;
template Impl()
{
size_t goodAllocSize(size_t s)
size_t goodAllocSize(size_t s) pure nothrow const
{
return parent.goodAllocSize(actualAllocationSize(s));
}
@ -1064,27 +1186,30 @@ struct AffixAllocator(Allocator, Prefix, Suffix = void)
which may use the global default). Also, the methods will be $(D
shared) if the parent allocator defines them as such.
*/
size_t goodAllocSize(size_t);
size_t goodAllocSize(size_t) pure nothrow const;
/// Ditto
void[] allocate(size_t);
void[] allocate(size_t) pure nothrow;
/// Ditto
bool owns(void[]);
bool owns(void[]) pure nothrow;
/// Ditto
bool expand(ref void[] b, size_t delta);
bool expand(ref void[] b, size_t delta) pure nothrow;
/// Ditto
bool reallocate(ref void[] b, size_t s);
bool reallocate(ref void[] b, size_t s)pure nothrow;
/// Ditto
void deallocate(void[] b);
void deallocate(void[] b) pure nothrow;
/// Ditto
void deallocateAll();
void deallocateAll() pure nothrow;
/**
The $(D it) singleton is defined if and only if the parent allocator has no state and defines its own $(D it) object.
The $(D it) singleton is defined if and only if the parent allocator has
no state and defines its own $(D it) object.
*/
static AffixAllocator it;
/**
Affix access functions offering mutable references to the affixes of a block previously allocated with this allocator. $(D b) may not be null. They are defined if and only if the corresponding affix is not $(D void).
Affix access functions offering mutable references to the affixes of a
block previously allocated with this allocator. $(D b) may not be null.
They are defined if and only if the corresponding affix is not $(D void).
Precondition: $(D b !is null)
*/
@ -1118,7 +1243,7 @@ unittest
unittest
{
alias AffixAllocator!(Mallocator, size_t) A;
alias A = AffixAllocator!(Mallocator, size_t);
auto b = A.it.allocate(10);
A.it.prefix(b) = 10;
assert(A.it.prefix(b) == 10);
@ -1129,9 +1254,10 @@ unittest
}
/**
Returns the number of most significant ones before a zero can be found in $(D x). If $(D x) contains no zeros (i.e. is equal to $(D ulong.max)), returns 64.
Returns the number of most significant ones before a zero can be found in $(D x).
If $(D x) contains no zeros (i.e. is equal to $(D ulong.max)), returns 64.
*/
private uint leadingOnes(ulong x)
private uint leadingOnes(ulong x) pure nothrow @safe
{
uint result = 0;
while (cast(long) x < 0)
@ -1156,7 +1282,7 @@ unittest
/**
Finds a run of contiguous ones in $(D x) of length at least $(D n).
*/
private uint findContigOnes(ulong x, uint n)
private uint findContigOnes(ulong x, uint n) pure nothrow @safe
{
while (n > 1)
{
@ -1184,7 +1310,7 @@ unittest
/**
Returns the number of trailing zeros of $(D x).
*/
private uint trailingZeros(ulong x)
private uint trailingZeros(ulong x) pure nothrow @safe
{
uint result;
while (result < 64 && !(x & (1UL << result)))
@ -1206,7 +1332,7 @@ unittest
/*
Unconditionally sets the bits from lsb through msb in w to zero.
*/
private void setBits(ref ulong w, uint lsb, uint msb)
private void setBits(ref ulong w, uint lsb, uint msb) pure nothrow @safe
{
assert(lsb <= msb && msb < 64);
const mask = (ulong.max << lsb) & (ulong.max >> (63 - msb));
@ -1225,7 +1351,7 @@ unittest
/* Are bits from lsb through msb in w zero? If so, make then 1
and return the resulting w. Otherwise, just return 0.
*/
private bool setBitsIfZero(ref ulong w, uint lsb, uint msb)
private bool setBitsIfZero(ref ulong w, uint lsb, uint msb) pure nothrow @safe
{
assert(lsb <= msb && msb < 64);
const mask = (ulong.max << lsb) & (ulong.max >> (63 - msb));
@ -1234,14 +1360,24 @@ private bool setBitsIfZero(ref ulong w, uint lsb, uint msb)
return true;
}
unittest
{
// TODO
}
// Assigns bits in w from lsb through msb to zero.
private void resetBits(ref ulong w, uint lsb, uint msb)
private void resetBits(ref ulong w, uint lsb, uint msb) pure nothrow @safe
{
assert(lsb <= msb && msb < 64);
const mask = (ulong.max << lsb) & (ulong.max >> (63 - msb));
w &= ~mask;
}
unittest
{
// TODO
}
/**
$(D HeapBlock) implements a simple heap consisting of one contiguous area
@ -1329,7 +1465,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
_blocks = blocks;
}
private void initialize()
private void initialize() pure nothrow @trusted
{
assert(_blocks);
const controlBytes = ((_blocks + 63) / 64) * 8;
@ -1344,11 +1480,11 @@ struct HeapBlock(Allocator, size_t theBlockSize,
_control = m[0 .. controlBytes / 8];
_control[] = 0;
_payload = m[controlBytesRounded / 8 .. $];
assert(_payload.length == _blocks * blockSize,
text(_payload.length, " != ", _blocks * blockSize));
assert(_payload.length == _blocks * blockSize/+,
text(_payload.length, " != ", _blocks * blockSize)+/);
}
private void initialize(void[] store)
private void initialize(void[] store) pure nothrow @trusted
{
assert(store.length);
// Round store to be ulong-aligned
@ -1363,9 +1499,9 @@ struct HeapBlock(Allocator, size_t theBlockSize,
auto blocks = cast(size_t) (approxBlocks + nextDown(1.0));
assert(blocks > 0);
assert(blockSize);
assert(blocks * blockSize + ((blocks + 63) / 64) * 8 >= store.length,
assert(blocks * blockSize + ((blocks + 63) / 64) * 8 >= store.length/+,
text(approxBlocks, " ", blocks, " ", blockSize, " ",
store.length));
store.length)+/);
while (blocks * blockSize + ((blocks + 63) / 64) * 8 > store.length)
{
--blocks;
@ -1385,13 +1521,13 @@ struct HeapBlock(Allocator, size_t theBlockSize,
}
private void initialize(ulong[] control, void[] payload, size_t blockSize)
pure nothrow @trusted
{
enforce(payload.length % blockSize == 0,
text(payload.length, " % ", blockSize, " != 0"));
assert(payload.length % blockSize == 0);
assert(payload.length / blockSize <= uint.max);
_blocks = cast(uint) (payload.length / blockSize);
const controlWords = (_blocks + 63) / 64;
enforce(controlWords == control.length);
assert(controlWords == control.length);
_control = control;
assert(control.equal(repeat(0, control.length)));
_payload = payload;
@ -1434,7 +1570,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
BUGS: Neither $(D deallocateAll) nor the destructor free the original memory
block. Either user code or the parent allocator should carry that.
*/
@trusted void[] allocate(const size_t s)
void[] allocate(const size_t s) pure nothrow @trusted
{
if (!_control)
{
@ -1486,7 +1622,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
}
/// Ditto
bool owns(void[] b) const
bool owns(void[] b) const pure nothrow @trusted
{
return b.ptr >= _payload.ptr
&& b.ptr + b.length <= _payload.ptr + _payload.length
@ -1500,7 +1636,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
0). Otherwise, returns a tuple with the next position to search.
*/
private Tuple!(size_t, uint) allocateAt(size_t wordIdx, uint msbIdx,
size_t blocks, ref void[] result)
size_t blocks, ref void[] result) pure nothrow @trusted
{
assert(blocks > 0);
assert(wordIdx < _control.length);
@ -1557,9 +1693,9 @@ struct HeapBlock(Allocator, size_t theBlockSize,
return available;
}
private void[] smallAlloc(uint blocks)
private void[] smallAlloc(uint blocks) pure nothrow @trusted
{
assert(blocks >= 2 && blocks <= 64, text(blocks));
assert(blocks >= 2 && blocks <= 64/+, text(blocks)+/);
foreach (i; _startIdx .. _control.length)
{
// Test within the current 64-bit word
@ -1589,7 +1725,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
return null;
}
private void[] hugeAlloc(size_t blocks)
private void[] hugeAlloc(size_t blocks) pure nothrow @trusted
{
assert(blocks > 64);
void[] result;
@ -1629,7 +1765,7 @@ struct HeapBlock(Allocator, size_t theBlockSize,
}
/// Ditto
@trusted bool expand(ref void[] b, size_t delta)
bool expand(ref void[] b, size_t delta) pure nothrow @trusted
{
//debug writefln("expand(%s, %s, %s)", b, minDelta, desiredDelta);
if (b is null)
@ -1664,14 +1800,14 @@ struct HeapBlock(Allocator, size_t theBlockSize,
return false;
}
// Expansion successful
assert(p.ptr == b.ptr + blocksOld * blockSize,
text(p.ptr, " != ", b.ptr + blocksOld * blockSize));
assert(p.ptr == b.ptr + blocksOld * blockSize/+,
text(p.ptr, " != ", b.ptr + blocksOld * blockSize)+/);
b = b.ptr[0 .. b.length + delta];
return true;
}
/// Ditto
@system bool reallocate(ref void[] b, size_t newSize)
bool reallocate(ref void[] b, size_t newSize) pure nothrow @system
{
if (newSize == 0)
{
@ -1902,10 +2038,10 @@ struct FallbackAllocator(Primary, Fallback)
If both $(D Primary) and $(D Fallback) are stateless, $(D FallbackAllocator)
defines a static instance $(D it).
*/
static if (!stateSize!Primary && !stateSize!Fallback)
/+static if (!stateSize!Primary && !stateSize!Fallback)
{
static FallbackAllocator it;
}
}+/
/**
The alignment offered is the minimum of the two allocators' alignment.
@ -1916,7 +2052,7 @@ struct FallbackAllocator(Primary, Fallback)
Allocates memory trying the primary allocator first. If it returns $(D
null), the fallback allocator is tried.
*/
void[] allocate(size_t s)
void[] allocate(size_t s) pure nothrow @safe
{
auto result = primary.allocate(s);
return result ? result : fallback.allocate(s);
@ -1959,12 +2095,12 @@ struct FallbackAllocator(Primary, Fallback)
allocation from $(D fallback) to $(D primary).
*/
bool reallocate(ref void[] b, size_t newSize)
bool reallocate(ref void[] b, size_t newSize) pure nothrow @trusted
{
bool crossAllocatorMove(From, To)(ref From from, ref To to)
bool crossAllocatorMove(From, To)(auto ref From from, auto ref To to)
{
auto b1 = to.allocate(newSize);
if (!b1) return false;
if (b1 is null) return false;
if (b.length < newSize) b1[0 .. b.length] = b[];
else b1[] = b[0 .. newSize];
static if (hasMember!(From, "deallocate"))
@ -2004,7 +2140,7 @@ struct FallbackAllocator(Primary, Fallback)
*/
static if (hasMember!(Primary, "deallocate")
|| hasMember!(Fallback, "deallocate"))
void deallocate(void[] b)
void deallocate(void[] b) pure nothrow @trusted
{
if (primary.owns(b))
{
@ -2014,7 +2150,7 @@ struct FallbackAllocator(Primary, Fallback)
else
{
static if (hasMember!(Fallback, "deallocate"))
return fallback.deallocate(b);
fallback.deallocate(b);
}
}
}
@ -2022,7 +2158,7 @@ struct FallbackAllocator(Primary, Fallback)
///
unittest
{
FallbackAllocator!(InSituRegion!16384, GCAllocator) a;
FallbackAllocator!(InSituRegion!16_384, GCAllocator) a;
// This allocation uses the stack
auto b1 = a.allocate(1024);
assert(b1.length == 1024, text(b1.length));
@ -2079,7 +2215,7 @@ struct Freelist(ParentAllocator,
else
{
size_t _min = chooseAtRuntime;
@property size_t min() const
@property size_t min() const nothrow pure @safe
{
assert(_min != chooseAtRuntime);
return _min;
@ -2102,7 +2238,7 @@ struct Freelist(ParentAllocator,
}
}
private bool tooSmall(size_t n) const
private bool tooSmall(size_t n) const nothrow pure @safe
{
static if (minSize == 0) return false;
else return n < min;
@ -2123,13 +2259,13 @@ struct Freelist(ParentAllocator,
}
}
private bool tooLarge(size_t n) const
private bool tooLarge(size_t n) const nothrow pure @safe
{
static if (maxSize == unbounded) return false;
else return n > max;
}
private bool inRange(size_t n) const
private bool inRange(size_t n) const nothrow pure @safe
{
static if (minSize == maxSize && minSize != chooseAtRuntime)
return n == maxSize;
@ -2206,7 +2342,7 @@ struct Freelist(ParentAllocator,
Returns $(D max) for sizes in the interval $(D [min, max]), and $(D
parent.goodAllocSize(bytes)) otherwise.
*/
size_t goodAllocSize(size_t bytes)
size_t goodAllocSize(size_t bytes) const nothrow pure @safe
{
if (inRange(bytes)) return maxSize == unbounded ? bytes : max;
return parent.goodAllocSize(bytes);
@ -2215,12 +2351,7 @@ struct Freelist(ParentAllocator,
/**
Allocates memory either off of the free list or from the parent allocator.
*/
void[] allocate(size_t bytes)
in
{
assert (_root !is null);
}
body
void[] allocate(size_t bytes) pure nothrow @trusted
{
assert(bytes < size_t.max / 2);
if (!inRange(bytes)) return parent.allocate(bytes);
@ -2234,7 +2365,7 @@ struct Freelist(ParentAllocator,
return result;
}
private void[] allocateFresh(const size_t bytes)
private void[] allocateFresh(const size_t bytes) pure nothrow @trusted
{
assert(!_root);
assert(bytes == max || max == unbounded);
@ -2251,9 +2382,9 @@ struct Freelist(ParentAllocator,
}
else
{
assert((parent.alignment + bytes) % Node.alignof == 0,
assert((parent.alignment + bytes) % Node.alignof == 0/+,
text("(", parent.alignment, " + ", bytes, ") % ",
Node.alignof));
Node.alignof)+/);
}
auto data = parent.allocate(nodesAtATime * bytes);
@ -2282,7 +2413,7 @@ struct Freelist(ParentAllocator,
Freelist) handle deallocations of objects of the appropriate size,
even for allocators that don't support $(D owns) (such as $(D Mallocator)).
*/
bool owns(void[] b)
bool owns(void[] b) const pure nothrow @safe
{
if (inRange(b.length)) return true;
static if (hasMember!(ParentAllocator, "owns"))
@ -2560,7 +2691,7 @@ struct SharedFreelist(ParentAllocator,
do
{
oldRoot = _root; // atomic load
next = oldRoot.next; // atomic load
next = oldRoot is null ? null : oldRoot.next; // atomic load
}
while (!cas(&_root, oldRoot, next));
// great, snatched the root
@ -2657,7 +2788,8 @@ struct SharedFreelist(ParentAllocator,
}
}
unittest
// This deadlocks
version (none) unittest
{
import core.thread, std.concurrency;
@ -2723,14 +2855,14 @@ private struct BasicRegion(uint minAlign = platformAlignment)
/**
Constructs a region backed by a user-provided store.
*/
this(void[] store)
this(void[] store) pure nothrow @trusted
{
static if (minAlign > 1)
{
auto newStore = cast(void*) roundUpToMultipleOf(
cast(ulong) store.ptr,
cast(size_t) store.ptr,
alignment);
enforce(newStore <= store.ptr + store.length);
assert(newStore <= store.ptr + store.length);
_current = newStore;
}
else
@ -2752,7 +2884,7 @@ private struct BasicRegion(uint minAlign = platformAlignment)
enum uint alignment = minAlign;
/// Ditto
void[] allocate(size_t bytes)
void[] allocate(size_t bytes) pure nothrow @trusted
{
static if (minAlign > 1)
const rounded = bytes.roundUpToMultipleOf(alignment);
@ -2772,7 +2904,7 @@ private struct BasicRegion(uint minAlign = platformAlignment)
// Just bump the pointer to the next good allocation
auto save = _current;
_current = cast(void*) roundUpToMultipleOf(
cast(ulong) _current, a);
cast(size_t) _current, a);
if (auto b = allocate(bytes)) return b;
// Failed, rollback
_current = save;
@ -2829,7 +2961,7 @@ struct Region(uint minAlign = platformAlignment)
Constructs a $(D Region) object backed by $(D buffer), which must be aligned
to $(D minAlign).
*/
this(void[] buffer)
this(void[] buffer) pure nothrow @safe
{
base = BasicRegion!minAlign(buffer);
assert(buffer.ptr !is &this);
@ -2932,7 +3064,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
{
assert(!_crt);
_crt = cast(void*) roundUpToMultipleOf(
cast(ulong) _store.ptr, alignment);
cast(size_t) _store.ptr, alignment);
_end = _store.ptr + _store.length;
}
@ -2941,7 +3073,12 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
accommodate the request. For efficiency reasons, if $(D bytes == 0) the
function returns an empty non-null slice.
*/
void[] allocate(size_t bytes)
void[] allocate(size_t bytes) pure nothrow @trusted
out (result)
{
assert (result is null || owns(result));
}
body
{
// Oddity: we don't return null for null allocation. Instead, we return
// an empty slice with a non-null ptr.
@ -2972,7 +3109,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
// Just bump the pointer to the next good allocation
auto save = _crt;
_crt = cast(void*) roundUpToMultipleOf(
cast(ulong) _crt, a);
cast(size_t) _crt, a);
if (auto b = allocate(bytes)) return b;
// Failed, rollback
_crt = save;
@ -2984,7 +3121,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
allocation. For efficiency reasons, if $(D b is null) the function returns
$(D false).
*/
bool owns(void[] b) const
bool owns(const void[] b) const nothrow pure @trusted
{
// No nullptr
return b.ptr >= _store.ptr
@ -2994,7 +3131,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
/**
Deallocates all memory allocated with this allocator.
*/
void deallocateAll()
void deallocateAll() pure nothrow @safe
{
_crt = _store.ptr;
}
@ -3715,7 +3852,14 @@ struct CascadingAllocator(alias make)
enum uint alignment = Allocator.alignment;
/// Ditto
void[] allocate(size_t s)
void[] allocate(size_t s) pure nothrow @trusted
out (res)
{
import std.string;
assert (res.length == 0 || res.length == s,
"res.length = %d, s = %d".format(res.length, s));
}
body
{
auto result = allocateNoGrow(s);
if (result) return result;
@ -3805,7 +3949,8 @@ struct CascadingAllocator(alias make)
if (!_root) return false;
for (auto n = _root; ; n = n.next)
{
if (n.a.owns(b) && n.a.reallocate(b, s)) return true;
static if (hasMember!(Allocator, "owns"))
if (n.a.owns(b) && n.a.reallocate(b, s)) return true;
if (!n.nextIsInitialized) break;
}
// Failed, but we may find new memory in a new node.
@ -3870,7 +4015,7 @@ struct CascadingAllocator(alias make)
unittest
{
// Create an allocator based upon 4MB regions, fetched from the GC heap.
CascadingAllocator!({ return Region!()(new void[1024 * 4096]); }) a;
CascadingAllocator!(function() nothrow { return Region!()(new void[1024 * 4096]); }) a;
auto b1 = a.allocate(1024 * 8192);
assert(b1 is null); // can't allocate more than 4MB at a time
b1 = a.allocate(1024 * 10);
@ -3997,21 +4142,21 @@ struct Segregator(size_t threshold, SmallAllocator, LargeAllocator)
template Impl()
{
void[] allocate(size_t s)
void[] allocate(size_t s) nothrow pure @trusted
{
return s <= threshold ? _small.allocate(s) : _large.allocate(s);
}
static if (hasMember!(SmallAllocator, "deallocate")
&& hasMember!(LargeAllocator, "deallocate"))
void deallocate(void[] data)
void deallocate(void[] data) nothrow pure @trusted
{
data.length <= threshold
? _small.deallocate(data)
: _large.deallocate(data);
}
size_t goodAllocSize(size_t s)
size_t goodAllocSize(size_t s) const nothrow pure @safe
{
return s <= threshold
? _small.goodAllocSize(s)
@ -4020,7 +4165,7 @@ struct Segregator(size_t threshold, SmallAllocator, LargeAllocator)
static if (hasMember!(SmallAllocator, "owns")
&& hasMember!(LargeAllocator, "owns"))
bool owns(void[] b)
bool owns(void[] b) const nothrow pure @safe
{
return b.length <= threshold ? _small.owns(b) : _large.owns(b);
}
@ -4087,7 +4232,11 @@ struct Segregator(size_t threshold, SmallAllocator, LargeAllocator)
static if (sharedMethods)
{
static shared Segregator it;
static shared(typeof(this)) it() pure nothrow @property @trusted
{
return cast(typeof(return)) _it;
}
private static const shared Segregator _it;
shared { mixin Impl!(); }
}
else
@ -4312,7 +4461,7 @@ class CAllocator
{
/// Returns the alignment offered. By default this method returns $(D
/// platformAlignment).
@property uint alignment()
uint alignment() pure nothrow const @property
{
return platformAlignment;
}
@ -4323,7 +4472,7 @@ class CAllocator
throw an exception if it does allow setting the alignment but an invalid
value is passed.
*/
@property bool alignment(uint)
@property bool alignment(uint) pure nothrow @property
{
return false;
}
@ -4333,7 +4482,7 @@ class CAllocator
fragmentation. By default returns $(D s) rounded up to the nearest multiple
of $(D alignment).
*/
size_t goodAllocSize(size_t s)
size_t goodAllocSize(size_t s) pure nothrow const @property
{
return s.roundUpToMultipleOf(alignment);
}
@ -4341,7 +4490,7 @@ class CAllocator
/**
Allocates memory.
*/
abstract void[] allocate(size_t);
abstract void[] allocate(size_t) pure nothrow @safe;
/**
Returns $(D true) if the allocator supports $(D owns). By default returns
@ -4362,17 +4511,17 @@ class CAllocator
}
/// Expands a memory block in place.
abstract bool expand(ref void[], size_t);
abstract bool expand(ref void[], size_t) pure nothrow;
/// Reallocates a memory block.
abstract bool reallocate(ref void[] b, size_t);
abstract bool reallocate(ref void[] b, size_t) pure nothrow;
/// Deallocates a memory block. Returns $(D false) if deallocation is not
/// supported.
abstract bool deallocate(void[]);
abstract bool deallocate(void[]) pure nothrow;
/// Deallocates all memory. Returns $(D false) if not supported.
abstract bool deallocateAll();
abstract bool deallocateAll() pure nothrow;
/// Returns $(D true) if allocator supports $(D allocateAll). By default
/// returns $(D false).
@ -4405,7 +4554,7 @@ class CAllocatorImpl(Allocator) : CAllocator
else alias impl = Allocator.it;
/// Returns $(D impl.alignment).
override @property uint alignment()
override uint alignment() pure nothrow const @property
{
return impl.alignment;
}
@ -4414,7 +4563,7 @@ class CAllocatorImpl(Allocator) : CAllocator
If $(D Allocator) supports alignment setting, performs it and returns $(D
true). Otherwise, returns $(D false).
*/
override @property bool alignment(uint a)
override bool alignment(uint a) pure nothrow const @property
{
static if (is(typeof(impl.alignment = a)))
{
@ -4430,7 +4579,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/**
Returns $(D impl.goodAllocSize(s)).
*/
override size_t goodAllocSize(size_t s)
override size_t goodAllocSize(size_t s) pure nothrow const @property
{
return impl.goodAllocSize(s);
}
@ -4438,7 +4587,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/**
Returns $(D impl.allocate(s)).
*/
override void[] allocate(size_t s)
override void[] allocate(size_t s) pure nothrow @safe
{
return impl.allocate(s);
}
@ -4446,7 +4595,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/**
Returns $(D true) if $(D Allocator) supports $(D owns).
*/
override bool supportsOwns()
override bool supportsOwns() pure nothrow const @safe
{
return hasMember!(Allocator, "owns");
}
@ -4462,7 +4611,7 @@ class CAllocatorImpl(Allocator) : CAllocator
}
/// Returns $(D impl.expand(b, s)) if defined, $(D false) otherwise.
override bool expand(ref void[] b, size_t s)
override bool expand(ref void[] b, size_t s) pure nothrow
{
static if (hasMember!(Allocator, "expand"))
return impl.expand(b, s);
@ -4478,7 +4627,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/// Calls $(D impl.deallocate(b)) and returns $(D true) if defined,
/// otherwise returns $(D false).
override bool deallocate(void[] b)
override bool deallocate(void[] b) pure nothrow
{
static if (hasMember!(Allocator, "deallocate"))
{
@ -4493,7 +4642,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/// Calls $(D impl.deallocateAll()) and returns $(D true) if defined,
/// otherwise returns $(D false).
override bool deallocateAll()
override bool deallocateAll() pure nothrow
{
static if (hasMember!(Allocator, "deallocateAll"))
{
@ -4508,7 +4657,7 @@ class CAllocatorImpl(Allocator) : CAllocator
/// Returns $(D true) if allocator supports $(D allocateAll). By default
/// returns $(D false).
override bool supportsAllocateAll()
override bool supportsAllocateAll() pure nothrow const
{
return hasMember!(Allocator, "allocateAll");
}
@ -4518,7 +4667,7 @@ class CAllocatorImpl(Allocator) : CAllocator
returns $(D impl.allocateAll()).
*/
static if (hasMember!(Allocator, "allocateAll"))
override void[] allocateAll()
override void[] allocateAll() pure nothrow
{
return impl.allocateAll();
}

401
stdx/d/ast.d → std/d/ast.d
View File

File diff suppressed because it is too large Load Diff

2
stdx/d/entities.d → std/d/entities.d
View File

@ -9,7 +9,7 @@
* Source: $(PHOBOSSRC std/d/_entities.d)
*/
module stdx.d.entities;
module std.d.entities;
/**
* Generated from $(LINK http://www.w3.org/TR/html5/entities.json)

52
stdx/d/lexer.d → std/d/lexer.d
View File

@ -1,12 +1,12 @@
module stdx.d.lexer;
module std.d.lexer;
import std.typecons;
import std.typetuple;
import std.array;
import std.algorithm;
import std.range;
import stdx.lexer;
public import stdx.lexer : StringCache;
import std.lexer;
public import std.lexer : StringCache;
private enum operators = [
",", ".", "..", "...", "/", "/=", "!", "!<", "!<=", "!<>", "!<>=", "!=",
@ -41,7 +41,7 @@ private enum dynamicTokens = [
"whitespace", "doubleLiteral", "floatLiteral", "idoubleLiteral",
"ifloatLiteral", "intLiteral", "longLiteral", "realLiteral",
"irealLiteral", "uintLiteral", "ulongLiteral", "characterLiteral",
"dstringLiteral", "stringLiteral", "wstringLiteral", "scriptLine"
"dstringLiteral", "stringLiteral", "wstringLiteral"
];
private enum pseudoTokenHandlers = [
@ -91,7 +91,7 @@ private enum extraFields = q{
return 0;
}
};
public alias Token = stdx.lexer.TokenStructure!(IdType, extraFields);
public alias Token = std.lexer.TokenStructure!(IdType, extraFields);
/**
* Configure string lexing behavior
@ -119,6 +119,18 @@ public enum WhitespaceBehavior : ubyte
/// Whitespace is treated as a token
include
}
/**
* Configure special token handling behavior
*/
public enum SpecialTokenBehavior : ubyte
{
/// Special tokens are skipped
skip,
/// Special tokens are treated as a token
include
}
/**
* Configure comment handling behavior
*/
@ -136,6 +148,7 @@ public struct LexerConfig
StringBehavior stringBehavior;
WhitespaceBehavior whitespaceBehavior;
CommentBehavior commentBehavior;
SpecialTokenBehavior specialTokenBehavior;
}
public bool isBasicType(IdType type) nothrow pure @safe
@ -412,8 +425,8 @@ public struct DLexer
private static bool isDocComment(string comment) pure nothrow @safe
{
return comment.length >= 3 && (comment[2] == '/'
|| comment[2] == '*' || comment[2] == '+');
return comment.length >= 3 && (comment[0 .. 3] == "///"
|| comment[0 .. 3] == "/++" || comment[0 .. 3] == "/**");
}
public void popFront() pure
@ -434,6 +447,7 @@ public struct DLexer
}
do _popFront(); while (front == tok!"comment");
if (front == tok!"whitespace") goto case tok!"whitespace";
if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence";
}
break;
case tok!"whitespace":
@ -441,6 +455,15 @@ public struct DLexer
{
do _popFront(); while (front == tok!"whitespace");
if (front == tok!"comment") goto case tok!"comment";
if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence";
}
break;
case tok!"specialTokenSequence":
if (config.specialTokenBehavior == SpecialTokenBehavior.skip)
{
do _popFront(); while (front == tok!"specialTokenSequence");
if (front == tok!"comment") goto case tok!"comment";
if (front == tok!"whitespace") goto case tok!"whitespace";
}
break;
default:
@ -557,7 +580,7 @@ public struct DLexer
Token lexNumber() pure nothrow
{
mixin (tokenStart);
if (range.canPeek(1) && range.front == '0')
if (range.front == '0' && range.canPeek(1))
{
auto ahead = range.peek(1)[1];
switch (ahead)
@ -742,13 +765,18 @@ public struct DLexer
// "double identifier".
if (range.canPeek(1))
{
switch (range.peekAt(1))
auto ch = range.peekAt(1);
if (ch <= 0x2f
|| (ch >= '0' && ch <= '9')
|| (ch >= ':' && ch <= '@')
|| (ch >= '[' && ch <= '^')
|| (ch >= '{' && ch <= '~')
|| ch == '`' || ch == '_')
{
case '0': .. case '9':
goto doubleLiteral;
default:
break decimalLoop;
}
else
break decimalLoop;
}
else
{

230
stdx/d/parser.d → std/d/parser.d
View File

@ -2,17 +2,17 @@
/**
* MACROS:
* GRAMMAR = <pre>$0</pre>
* GRAMMAR = <pre class="grammar">$0</pre>
* RULEDEF = $(B $(DDOC_ANCHOR $0) $0)
* RULE = $(LINK2 #$0, $0)
* LITERAL = $(D_STRING $(I $0))
*/
module stdx.d.parser;
module std.d.parser;
import stdx.d.lexer;
import stdx.d.ast;
import stdx.allocator;
import std.d.lexer;
import std.d.ast;
import std.allocator;
import std.conv;
import std.algorithm;
import std.array;
@ -23,113 +23,7 @@ import std.string : format;
// Caution: generates 180 megabytes of logging for std.datetime
//version = std_parser_verbose;
/**
* The parse allocator is designed so that very large number of node instances
* allocated by the parser can be deallocated all at once. This saves time by
* preventing the GC from having to scan the nodes.
*/
class ParseAllocator : CAllocator
{
public:
this(string name = "none")
{
this.name = name;
}
string name;
override void[] allocate(size_t size)
in
{
assert (size > 0);
assert (size < blockSize);
}
out (result)
{
assert (result.length == size);
}
body
{
enum size_t mask = ~ (cast(size_t) 7);
enum s = ((Node.sizeof - 1) & mask) + 8;
Node* current = root;
while (true)
{
while (current !is null)
{
immutable size_t blockLength = current.block.length;
immutable size_t oldUsed = current.used;
immutable size_t newUsed = oldUsed + size;
if (newUsed > blockLength)
current = current.next;
else
{
current.used = ((newUsed - 1) & mask) + 8;
// assert (current.used >= oldUsed + size);
// assert (current.block.ptr + blockSize > current.block.ptr + newUsed);
// assert (newUsed > oldUsed);
// writefln("Allocating 0x%012x - 0x%012x",
// cast(size_t) current.block.ptr + oldUsed,
// cast(size_t) current.block.ptr + newUsed);
current.block[oldUsed .. newUsed] = 0;
return current.block[oldUsed .. newUsed];
}
}
import core.memory;
// stderr.writeln("Allocating new block while processing ", name);
ubyte* newBlock = cast(ubyte*) GC.malloc(blockSize, GC.BlkAttr.NO_SCAN);
// assert (newBlock !is null);
// stderr.writefln("Memory spans from 0x%012x to 0x%012x",
// cast(size_t) newBlock, cast(size_t) newBlock + blockSize);
root = new Node (root, s, newBlock[0 .. blockSize]);
// assert (root.block.length == blockSize);
current = root;
}
assert (false);
}
/**
* Deallocates all memory held by this allocator. All node instances created
* by a parser using this allocator instance are invalid after calling this
* function.
*/
override bool deallocateAll()
{
deallocateRecursive(root);
root = null;
return true;
}
override bool expand(ref void[], size_t) { return false; }
override bool reallocate(ref void[], size_t) { return false; }
override bool deallocate(void[]) { return false; }
private:
void deallocateRecursive(Node* node)
{
import core.memory;
// import std.c.stdlib;
// node.block[] = 0;
// free(node.block.ptr);
GC.free(node.block.ptr);
if (node.next !is null)
deallocateRecursive(node.next);
node.next = null;
}
Node* root;
enum blockSize = 1024 * 1024 * 4;
struct Node
{
Node* next;
size_t used;
ubyte[] block;
}
}
alias ParseAllocator = CAllocatorImpl!(Mallocator);
/**
* Params:
@ -219,7 +113,7 @@ class Parser
node.linkageAttribute = parseLinkageAttribute();
}
warn("Prefer the new \"'alias' identifier '=' type ';'\" syntax"
~ " to the old \"'alias' type identifier ';'\" syntax");
~ " to the old \"'alias' type identifier ';'\" syntax");
if ((node.type = parseType()) is null) return null;
auto ident = expect(tok!"identifier");
if (ident is null)
@ -431,7 +325,7 @@ alias core.sys.posix.stdio.fileno fileno;
* Parses an ArrayLiteral
*
* $(GRAMMAR $(RULEDEF arrayLiteral):
* $(LITERAL '[') ($(RULE assignExpression) ($(LITERAL ',') $(RULE assignExpression))*)? $(LITERAL ']')
* $(LITERAL '[') $(RULE argumentList)? $(LITERAL ']')
* ;)
*/
ArrayLiteral parseArrayLiteral()
@ -780,6 +674,8 @@ alias core.sys.posix.stdio.fileno fileno;
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = allocate!AssignExpression;
node.line = current().line;
node.column = current().column;
node.ternaryExpression = parseTernaryExpression();
if (currentIsOneOf(tok!"=", tok!">>>=",
tok!">>=", tok!"<<=",
@ -1574,6 +1470,8 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
auto t = expect(tok!"this");
if (t is null) return null;
node.location = t.index;
node.line = t.line;
node.column = t.column;
auto p = peekPastParens();
bool isTemplate = false;
if (p !is null && p.type == tok!"(")
@ -2068,7 +1966,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
* Parses a Destructor
*
* $(GRAMMAR $(RULEDEF destructor):
* $(LITERAL '~') $(LITERAL 'this') $(LITERAL '$(LPAREN)') $(LITERAL '$(RPAREN)') ($(RULE functionBody) | $(LITERAL ';'))
* $(LITERAL '~') $(LITERAL 'this') $(LITERAL '$(LPAREN)') $(LITERAL '$(RPAREN)') $(RULE memberFunctionAttribute)* ($(RULE functionBody) | $(LITERAL ';'))
* ;)
*/
Destructor parseDestructor()
@ -2078,13 +1976,22 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
node.comment = comment;
comment = null;
if (expect(tok!"~") is null) return null;
node.index = current.index;
node.line = current.line;
node.column = current.column;
if (expect(tok!"this") is null) return null;
if (expect(tok!"(") is null) return null;
if (expect(tok!")") is null) return null;
if (currentIs(tok!";"))
advance();
else
{
MemberFunctionAttribute[] memberFunctionAttributes;
while(moreTokens() && currentIsMemberFunctionAttribute())
memberFunctionAttributes ~= parseMemberFunctionAttribute();
node.memberFunctionAttributes = ownArray(memberFunctionAttributes);
node.functionBody = parseFunctionBody();
}
return node;
}
@ -2377,7 +2284,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
{
if (!canBeRange)
{
error(`Cannot have more than one foreach varible for a foreach range statement`);
error(`Cannot have more than one foreach variable for a foreach range statement`);
return null;
}
advance();
@ -2878,6 +2785,8 @@ body {} // six
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = allocate!IfStatement;
node.line = current().line;
node.column = current().column;
if (expect(tok!"if") is null) return null;
node.startIndex = current().index;
if (expect(tok!"(") is null) return null;
@ -3278,6 +3187,8 @@ interface "Four"
Invariant parseInvariant()
{
auto node = allocate!Invariant;
node.index = current.index;
node.line = current.line;
if (expect(tok!"invariant") is null) return null;
if (currentIs(tok!"("))
{
@ -3491,7 +3402,7 @@ invariant() foo();
* Parses a LinkageAttribute
*
* $(GRAMMAR $(RULEDEF linkageAttribute):
* $(LITERAL 'extern') $(LITERAL '$(LPAREN)') $(LITERAL Identifier) $(LITERAL '++')? $(LITERAL '$(RPAREN)')
* $(LITERAL 'extern') $(LITERAL '$(LPAREN)') $(LITERAL Identifier) ($(LITERAL '++') ($(LITERAL ',') $(RULE identifierChain))?)? $(LITERAL '$(RPAREN)')
* ;)
*/
LinkageAttribute parseLinkageAttribute()
@ -3507,6 +3418,11 @@ invariant() foo();
{
advance();
node.hasPlusPlus = true;
version(DIP61) if (currentIs(tok!","))
{
advance();
node.identifierChain = parseIdentifierChain();
}
}
expect(tok!")");
return node;
@ -3645,7 +3561,7 @@ invariant() foo();
mixin(traceEnterAndExit!(__FUNCTION__));
Module m = allocate!Module;
if (currentIs(tok!"scriptLine"))
advance();
m.scriptLine = advance();
if (currentIs(tok!"module"))
m.moduleDeclaration = parseModuleDeclaration();
while (moreTokens())
@ -3879,6 +3795,7 @@ invariant() foo();
* $(RULE assignExpression)
* | $(RULE arrayInitializer)
* | $(RULE structInitializer)
* | $(RULE functionBody)
* ;)
*/
NonVoidInitializer parseNonVoidInitializer()
@ -3890,6 +3807,8 @@ invariant() foo();
auto b = peekPastBraces();
if (b !is null && (b.type == tok!"("))
node.assignExpression = parseAssignExpression();
else if (hasMagicDelimiter!(tok!"{", tok!";")())
node.functionBody = parseFunctionBody();
else
node.structInitializer = parseStructInitializer();
}
@ -3907,8 +3826,15 @@ invariant() foo();
else
node.assignExpression = parseAssignExpression();
}
else if (currentIsOneOf(tok!"in", tok!"out", tok!"body"))
node.functionBody = parseFunctionBody();
else
node.assignExpression = parseAssignExpression();
if (node.assignExpression is null && node.arrayInitializer is null
&& node.structInitializer is null && node.functionBody is null)
{
return null;
}
return node;
}
@ -4508,10 +4434,14 @@ q{(int a, ...)
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = allocate!ReturnStatement;
if (expect(tok!"return") is null) return null;
auto start = expect(tok!"return");
if (start is null) return null;
node.startLocation = start.index;
if (!currentIs(tok!";"))
node.expression = parseExpression();
if (expect(tok!";") is null) return null;
auto semicolon = expect(tok!";");
if (semicolon is null) return null;
node.endLocation = semicolon.index;
return node;
}
@ -4619,8 +4549,18 @@ q{(int a, ...)
if (!currentIs(tok!"]"))
{
node.lower = parseAssignExpression();
if (node.lower is null)
{
error("assignExpression expected");
return null;
}
expect(tok!"..");
node.upper = parseAssignExpression();
if (node.upper is null)
{
error("assignExpression expected");
return null;
}
}
if (expect(tok!"]") is null) return null;
return node;
@ -4874,8 +4814,13 @@ q{(int a, ...)
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = allocate!StructInitializer;
expect(tok!"{");
node.structMemberInitializers = parseStructMemberInitializers();
expect(tok!"}");
if (currentIs(tok!"}"))
advance();
else
{
node.structMemberInitializers = parseStructMemberInitializers();
expect(tok!"}");
}
return node;
}
@ -6069,6 +6014,7 @@ q{doStuff(5)}c;
*
* $(GRAMMAR $(RULEDEF variableDeclaration):
* $(RULE _type) $(RULE declarator) ($(LITERAL ',') $(RULE declarator))* $(LITERAL ';')
* | $(RULE _type) $(RULE declarator) $(LITERAL '=') $(RULE functionBody)
* | $(RULE autoDeclaration)
* ;)
*/
@ -6096,8 +6042,18 @@ q{doStuff(5)}c;
break;
}
node.declarators = ownArray(declarators);
expect(tok!";");
return node;
// if (node.declarators.length == 1
// && node.declarators[0].initializer !is null
// && node.declarators[0].initializer.nonVoidInitializer !is null
// && node.declarators[0].initializer.nonVoidInitializer.functionBody !is null)
// {
// return node;
// }
// else
{
expect(tok!";");
return node;
}
}
/**
@ -6248,7 +6204,7 @@ q{doStuff(5)}c;
mixin(traceEnterAndExit!(__FUNCTION__));
if (startsWith(tok!"[", tok!"]"))
return true;
return hasMagicDelimiter!(tok!"..")();
return hasMagicDelimiter!(tok!"[", tok!"..")();
}
void setTokens(const(Token)[] tokens)
@ -6271,6 +6227,7 @@ protected:
if (allocator is null)
return from;
T[] to = cast(T[]) allocator.allocate(T.sizeof * from.length);
assert (to.length == from.length, format("from.length = %d, to.length = %d", from.length, to.length));
to[] = from[];
return to;
}
@ -6282,6 +6239,7 @@ protected:
return new T(args);
enum numBytes = __traits(classInstanceSize, T);
void[] mem = allocator.allocate(numBytes);
assert (mem.length == numBytes, format("%d", mem.length));
T t = emplace!T(mem, args);
assert (cast(void*) t == mem.ptr, "%x, %x".format(cast(void*) t, mem.ptr));
return t;
@ -6306,22 +6264,25 @@ protected:
bool isAssociativeArrayLiteral()
{
return hasMagicDelimiter!(tok!":")();
auto b = setBookmark();
scope(exit) goToBookmark(b);
advance();
return !currentIs(tok!"]") && parseExpression() !is null && currentIs(tok!":");
}
bool hasMagicDelimiter(alias T)()
bool hasMagicDelimiter(alias L, alias T)()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto i = index;
scope(exit) index = i;
assert(currentIs(tok!"["));
assert(currentIs(L));
advance();
while (moreTokens()) switch (current.type)
{
case tok!"{": skipBraces(); break;
case tok!"(": skipParens(); break;
case tok!"[": skipBrackets(); break;
case tok!"]": return false;
case tok!"]": case tok!"}": return false;
case T: return true;
default: advance(); break;
}
@ -6392,8 +6353,9 @@ protected:
case tok!"abstract":
case tok!"pure":
case tok!"nothrow":
mixin(BASIC_TYPE_CASES);
return true;
mixin(BASIC_TYPE_CASES);
return !peekIs(tok!".");
case tok!"case":
case tok!"default":
case tok!"return":
@ -6479,6 +6441,14 @@ protected:
return true;
}
return true;
case tok!"pragma":
auto b = setBookmark();
scope(exit) goToBookmark(b);
advance();
auto past = peekPastParens();
if (past is null || *past == tok!";")
return false;
return true;
case tok!"deprecated":
case tok!"private":
case tok!"package":

39
stdx/lexer.d → std/lexer.d
View File

@ -33,7 +33,7 @@
* )
* Examples:
* $(UL
* $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/stdx/d/lexer.d, here).)
* $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/std/d/lexer.d, here).)
* $(LI A _lexer for Lua is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/lualexer.d, here).)
* $(LI A _lexer for JSON is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/jsonlexer.d, here).)
* )
@ -112,7 +112,7 @@
* Source: $(PHOBOSSRC std/_lexer.d)
*/
module stdx.lexer;
module std.lexer;
/**
* Template for determining the type used for a token type. Selects the smallest
@ -249,6 +249,11 @@ struct TokenStructure(IdType, string extraFields = "")
{
public:
bool opEquals(ref const typeof(this) other) const pure nothrow @safe
{
return this.type == other.type && this.text == other.text;
}
/**
* Returs: true if the token has the given type, false otherwise.
*/
@ -420,6 +425,7 @@ mixin template Lexer(Token, alias defaultTokenFunction,
private static string generateCaseStatements()
{
import std.algorithm;
import std.conv;
import std.string;
import std.range;
@ -443,6 +449,7 @@ mixin template Lexer(Token, alias defaultTokenFunction,
private static string printCase(string[] tokens, string[] pseudoTokens)
{
import std.algorithm;
string[] t = tokens;
string[] sortedTokens = stupidToArray(sort!"a.length > b.length"(t));
import std.conv;
@ -727,7 +734,8 @@ struct LexerRange
}
/**
* Implements the algorithm _popFrontN more efficiently.
* Implements the algorithm _popFrontN more efficiently. This function does
* not detect or handle newlines.
*/
void popFrontN(size_t n) pure nothrow @safe
{
@ -765,9 +773,6 @@ struct LexerRange
size_t line;
}
/**
* FREAKIN' MAAAGIC
*/
shared struct StringCache
{
import core.sync.mutex;
@ -784,6 +789,20 @@ public:
allocating = false;
}
~this()
{
import core.memory;
shared(Block)* current = rootBlock;
while (current !is null)
{
shared(Block)* prev = current;
current = current.next;
free(cast(void*) prev.bytes.ptr);
}
rootBlock = null;
buckets = [];
}
/**
* Caches a string.
* Params: str = the string to intern
@ -854,7 +873,6 @@ private:
if (bytes is null || bytes.length == 0)
return "";
import core.atomic;
import core.memory;
shared ubyte[] mem;
shared(Node*)* oldBucketRoot = &buckets[hash % buckets.length];
while (true)
@ -928,7 +946,7 @@ private:
import core.atomic;
import core.memory;
if (numBytes > (blockSize / 4))
return cast(shared) (cast(ubyte*) GC.malloc(numBytes, GC.BlkAttr.NO_SCAN))[0 .. numBytes];
return cast(shared) (cast(ubyte*) malloc(numBytes))[0 .. numBytes];
shared(Block)* r = rootBlock;
while (true)
{
@ -949,7 +967,7 @@ private:
if (cas(&allocating, false, true))
{
shared(Block)* b = new shared Block(
cast(shared) (cast(ubyte*) GC.malloc(blockSize, GC.BlkAttr.NO_SCAN))[0 .. blockSize],
cast(shared) (cast(ubyte*) malloc(blockSize))[0 .. blockSize],
numBytes,
r);
atomicStore(rootBlock, b);
@ -1048,3 +1066,6 @@ private:
shared(Node)*[] buckets;
shared(Block)* rootBlock;
}
private extern(C) void* malloc(size_t) nothrow pure;
private extern(C) void free(void*) nothrow pure;