From f76167f231968d111c1778c3246682ce77fc942e Mon Sep 17 00:00:00 2001 From: Dmitry Olshansky Date: Thu, 7 Feb 2013 00:39:25 +0400 Subject: [PATCH] Avoid allocation happiness in StringCache Add nice s-box driven hash function, noticeble improvement (still StringCache alone) Optimize isWhite and drop extra check on first iter of lexWhiteSpace; almost 10% of speed gain Also dialdown -O switch of ldc2, as it makes no difference. --- build.sh | 2 +- std/d/lexer.d | 175 ++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 143 insertions(+), 34 deletions(-) diff --git a/build.sh b/build.sh index eb2ae8a..d6139af 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,3 @@ #dmd *.d std/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner -L-lsqlite3 #-inline #dmd *.d std/d/*.d -g -m64 -w -wi -property -ofdscanner -L-lsqlite3 #-unittest -ldc2 -O5 *.d std/d/*.d -of=dscanner -release -vectorize -m64 +ldc2 -O2 -release *.d std/d/*.d -of=dscanner -m64 diff --git a/std/d/lexer.d b/std/d/lexer.d index b8741dc..873a961 100644 --- a/std/d/lexer.d +++ b/std/d/lexer.d @@ -109,7 +109,7 @@ module std.d.lexer; import std.algorithm; -import std.ascii; +import std.ascii : isHexDigit; import std.conv; import std.d.entities; import std.datetime; @@ -120,6 +120,23 @@ import std.traits; import std.regex; import std.container; +//these should be in std.ascii, using tabulation there is so 90-ies + +bool isAsciiWhite(uint x) +{ + x -= 0x09; + enum high = 0x0D-0x09; + enum space = 0x20-0x09; + return x <= high || x == space; +} + +bool isAsciiDigit(uint x) +{ + x -= '0'; + enum lastNum = '9' - '0'; + return x <= lastNum; +} + public: /** @@ -432,13 +449,10 @@ private: current.column = column; current.value = null; - if (isWhite(currentElement())) - { - lexWhitespace(); - return; - } - - outer: switch (currentElement()) + auto e = currentElement(); + if (isAsciiWhite(e)) + return lexWhitespace(); + outer: switch (e) { // pragma(msg, generateCaseTrie( mixin(generateCaseTrie( @@ -694,12 +708,17 @@ private: } void lexWhitespace() + in { - current.type = TokenType.whitespace; - while (!isEoF() && isWhite(currentElement())) + assert (isAsciiWhite(currentElement())); + } + body + { + current.type = TokenType.whitespace; + do { keepChar(); - } + }while (!isEoF() && isAsciiWhite(currentElement())); if (config.iterStyle & IterationStyle.includeWhitespace) setTokenValue(); } @@ -791,7 +810,7 @@ private: { keepChar(); } - else if (isWhite(currentElement()) && (config.tokenStyle & TokenStyle.notEscaped)) + else if (isAsciiWhite(currentElement()) && (config.tokenStyle & TokenStyle.notEscaped)) { keepChar(); } @@ -830,7 +849,7 @@ private: void lexNumber() in { - assert(isDigit(cast(char) currentElement()) || currentElement() == '.'); + assert(isAsciiDigit(cast(char) currentElement()) || currentElement() == '.'); } body { @@ -2395,8 +2414,6 @@ struct StringCache { string get(const ubyte[] bytes) { - - import std.stdio; size_t bucket; hash_t h; string* val = find(bytes, bucket, h); @@ -2406,41 +2423,133 @@ struct StringCache } else { - auto s = (cast(char[]) bytes).idup; - index[bucket] ~= s; - return s; + auto s = putIntoCache(bytes); + index[bucket] ~= s; + return s; } } - + private: - + + import std.stdio; string* find(const ubyte[] data, out size_t bucket, out hash_t h) { h = hash(data); - bucket = h % mapSize; + bucket = h % mapSize; foreach (i; 0 .. index[bucket].length) { if (index[bucket][i] == data) + { return &index[bucket][i]; + } } return null; } static hash_t hash(const(ubyte)[] data) { - hash_t h = 5381; - int c; - size_t i; - while (i < data.length) - { - c = data[i++]; - h = ((h << 5) + h) ^ c; - } - return h; + uint hash = 0; + foreach ( b; data) + { + hash ^= sbox[b]; + hash *= 3; + } + return hash; } - - immutable mapSize = 997; - string[][mapSize] index; + + enum mapSize = 2048; + string[][mapSize] index; + // leave some slack for alloctors/GC meta-data + enum chunkSize = 16*1024 - size_t.sizeof*8; + ubyte*[] chunkS; + size_t next = chunkSize; + + string putIntoCache(const ubyte[] data) + { + import core.memory; + + if(next + data.length > chunkSize) + { + // avoid huge strings + if(data.length > chunkSize/4) + return (cast(char[])data).idup; + chunkS ~= cast(ubyte*)GC.malloc(chunkSize, + GC.BlkAttr.NO_SCAN | GC.BlkAttr.NO_INTERIOR); + next = 0; + } + auto slice = chunkS[$-1][next..next+data.length]; + slice[] = data[]; + next += data.length; + return cast(string)slice; + } + } +immutable uint[] sbox = [ + 0xF53E1837, 0x5F14C86B, 0x9EE3964C, 0xFA796D53, + 0x32223FC3, 0x4D82BC98, 0xA0C7FA62, 0x63E2C982, + 0x24994A5B, 0x1ECE7BEE, 0x292B38EF, 0xD5CD4E56, + 0x514F4303, 0x7BE12B83, 0x7192F195, 0x82DC7300, + 0x084380B4, 0x480B55D3, 0x5F430471, 0x13F75991, + 0x3F9CF22C, 0x2FE0907A, 0xFD8E1E69, 0x7B1D5DE8, + 0xD575A85C, 0xAD01C50A, 0x7EE00737, 0x3CE981E8, + 0x0E447EFA, 0x23089DD6, 0xB59F149F, 0x13600EC7, + 0xE802C8E6, 0x670921E4, 0x7207EFF0, 0xE74761B0, + 0x69035234, 0xBFA40F19, 0xF63651A0, 0x29E64C26, + 0x1F98CCA7, 0xD957007E, 0xE71DDC75, 0x3E729595, + 0x7580B7CC, 0xD7FAF60B, 0x92484323, 0xA44113EB, + 0xE4CBDE08, 0x346827C9, 0x3CF32AFA, 0x0B29BCF1, + 0x6E29F7DF, 0xB01E71CB, 0x3BFBC0D1, 0x62EDC5B8, + 0xB7DE789A, 0xA4748EC9, 0xE17A4C4F, 0x67E5BD03, + 0xF3B33D1A, 0x97D8D3E9, 0x09121BC0, 0x347B2D2C, + 0x79A1913C, 0x504172DE, 0x7F1F8483, 0x13AC3CF6, + 0x7A2094DB, 0xC778FA12, 0xADF7469F, 0x21786B7B, + 0x71A445D0, 0xA8896C1B, 0x656F62FB, 0x83A059B3, + 0x972DFE6E, 0x4122000C, 0x97D9DA19, 0x17D5947B, + 0xB1AFFD0C, 0x6EF83B97, 0xAF7F780B, 0x4613138A, + 0x7C3E73A6, 0xCF15E03D, 0x41576322, 0x672DF292, + 0xB658588D, 0x33EBEFA9, 0x938CBF06, 0x06B67381, + 0x07F192C6, 0x2BDA5855, 0x348EE0E8, 0x19DBB6E3, + 0x3222184B, 0xB69D5DBA, 0x7E760B88, 0xAF4D8154, + 0x007A51AD, 0x35112500, 0xC9CD2D7D, 0x4F4FB761, + 0x694772E3, 0x694C8351, 0x4A7E3AF5, 0x67D65CE1, + 0x9287DE92, 0x2518DB3C, 0x8CB4EC06, 0xD154D38F, + 0xE19A26BB, 0x295EE439, 0xC50A1104, 0x2153C6A7, + 0x82366656, 0x0713BC2F, 0x6462215A, 0x21D9BFCE, + 0xBA8EACE6, 0xAE2DF4C1, 0x2A8D5E80, 0x3F7E52D1, + 0x29359399, 0xFEA1D19C, 0x18879313, 0x455AFA81, + 0xFADFE838, 0x62609838, 0xD1028839, 0x0736E92F, + 0x3BCA22A3, 0x1485B08A, 0x2DA7900B, 0x852C156D, + 0xE8F24803, 0x00078472, 0x13F0D332, 0x2ACFD0CF, + 0x5F747F5C, 0x87BB1E2F, 0xA7EFCB63, 0x23F432F0, + 0xE6CE7C5C, 0x1F954EF6, 0xB609C91B, 0x3B4571BF, + 0xEED17DC0, 0xE556CDA0, 0xA7846A8D, 0xFF105F94, + 0x52B7CCDE, 0x0E33E801, 0x664455EA, 0xF2C70414, + 0x73E7B486, 0x8F830661, 0x8B59E826, 0xBB8AEDCA, + 0xF3D70AB9, 0xD739F2B9, 0x4A04C34A, 0x88D0F089, + 0xE02191A2, 0xD89D9C78, 0x192C2749, 0xFC43A78F, + 0x0AAC88CB, 0x9438D42D, 0x9E280F7A, 0x36063802, + 0x38E8D018, 0x1C42A9CB, 0x92AAFF6C, 0xA24820C5, + 0x007F077F, 0xCE5BC543, 0x69668D58, 0x10D6FF74, + 0xBE00F621, 0x21300BBE, 0x2E9E8F46, 0x5ACEA629, + 0xFA1F86C7, 0x52F206B8, 0x3EDF1A75, 0x6DA8D843, + 0xCF719928, 0x73E3891F, 0xB4B95DD6, 0xB2A42D27, + 0xEDA20BBF, 0x1A58DBDF, 0xA449AD03, 0x6DDEF22B, + 0x900531E6, 0x3D3BFF35, 0x5B24ABA2, 0x472B3E4C, + 0x387F2D75, 0x4D8DBA36, 0x71CB5641, 0xE3473F3F, + 0xF6CD4B7F, 0xBF7D1428, 0x344B64D0, 0xC5CDFCB6, + 0xFE2E0182, 0x2C37A673, 0xDE4EB7A3, 0x63FDC933, + 0x01DC4063, 0x611F3571, 0xD167BFAF, 0x4496596F, + 0x3DEE0689, 0xD8704910, 0x7052A114, 0x068C9EC5, + 0x75D0E766, 0x4D54CC20, 0xB44ECDE2, 0x4ABC653E, + 0x2C550A21, 0x1A52C0DB, 0xCFED03D0, 0x119BAFE2, + 0x876A6133, 0xBC232088, 0x435BA1B2, 0xAE99BBFA, + 0xBB4F08E4, 0xA62B5F49, 0x1DA4B695, 0x336B84DE, + 0xDC813D31, 0x00C134FB, 0x397A98E6, 0x151F0E64, + 0xD9EB3E69, 0xD3C7DF60, 0xD2F2C336, 0x2DDD067B, + 0xBD122835, 0xB0B3BD3A, 0xB0D54E46, 0x8641F1E4, + 0xA0B38F96, 0x51D39199, 0x37A6AD75, 0xDF84EE41, + 0x3C034CBA, 0xACDA62FC, 0x11923B8B, 0x45EF170A, +]; + //void main(string[] args) {}