commit 391628e3d0df8787d147d237a33705ee3079d364 Author: Adam D. Ruppe Date: Fri Jul 15 08:48:59 2011 -0400 adding the existing stuff diff --git a/README b/README new file mode 100644 index 0000000..3fb3706 --- /dev/null +++ b/README @@ -0,0 +1,55 @@ +This is a collection of modules I find generally useful. + +Modules are usually independent; you don't need this whole directory +but it doesn't hurt to grab it all either. + +Currently included are: + +Web related +================ + +cgi.d - base module for making webapps in D +dom.d - an xml/html DOM based on what Javascript provides in browsers +web.d - a fancier way to write web apps. Uses reflection to make functions + accessible via url with minimal boilerplate in your code + + +Database related +================ + +database.d - main interface to databases. Includes DataObject +mysql.d - a mysql engine for database.d (most mature of the three) +postgres.d - a postgres engne for database.d +sqlite.d - a sqlite engine for database.d + +Desktop app stuff +================ + +simpledisplay.d - gives quick and easy access to a window for drawing +simpleaudio.d - gives minimal audio output + +Other +================ + +sha.d - implementations of the SHA1 and SHA256 algorithms +png.d - provides some png read/write support +curl.d - a small wrapper around the curl library +csv.d - gives read support to csv files +http.d - a lighterweight alternative to curl.d + + + +Things I might add once I clean up the files (this can be expedited upon +request, to an extent): + +httpd.d - an embedded web server +oauth.d - client/server stuff for oauth1 +html.d - a bunch of dom translation functions. Think unobstructive javascript + on the server side +browser.d - a very small html widget +netman.d - handles net connections (required by httpd.d) +imagedraft.d - (temporary name) has algorithms for images +bmp.d - gives .bmp read/write +dws.d - a draft of my D windowing system (also includes some Qt code) +wav.d - reading and writing WAV files +midi.d - reading and writing MIDI files diff --git a/cgi.d b/cgi.d new file mode 120000 index 0000000..283c969 --- /dev/null +++ b/cgi.d @@ -0,0 +1 @@ +../../djs/proxy/cgi.d \ No newline at end of file diff --git a/csv.d b/csv.d new file mode 100644 index 0000000..9466a66 --- /dev/null +++ b/csv.d @@ -0,0 +1,58 @@ +module arsd.csv; + +import std.string; +import std.array; + +string[][] readCsv(string data) { + data = data.replace("\r", ""); + + auto idx = data.indexOf("\n"); + //data = data[idx + 1 .. $]; // skip headers + + string[] fields; + string[][] records; + + string[] current; + + int state = 0; + string field; + foreach(c; data) { + tryit: switch(state) { + default: assert(0); + case 0: // normal + if(c == '"') + state = 1; + else if(c == ',') { + // commit field + current ~= field; + field = null; + } else if(c == '\n') { + // commit record + current ~= field; + + records ~= current; + current = null; + field = null; + } else + field ~= c; + break; + case 1: // in quote + if(c == '"') + state = 2; + else + field ~= c; + break; + case 2: // is it a closing quote or an escaped one? + if(c == '"') { + field ~= c; + state = 1; + } else { + state = 0; + goto tryit; + } + } + } + + + return records; +} diff --git a/curl.d b/curl.d new file mode 100644 index 0000000..0aca0d8 --- /dev/null +++ b/curl.d @@ -0,0 +1,188 @@ +module arsd.curl; + +pragma(lib, "curl"); + +import std.string; +extern(C) { + typedef void CURL; + typedef void curl_slist; + + alias int CURLcode; + alias int CURLoption; + + enum int CURLOPT_URL = 10002; + enum int CURLOPT_WRITEFUNCTION = 20011; + enum int CURLOPT_WRITEDATA = 10001; + enum int CURLOPT_POSTFIELDS = 10015; + enum int CURLOPT_POSTFIELDSIZE = 60; + enum int CURLOPT_POST = 47; + enum int CURLOPT_HTTPHEADER = 10023; + enum int CURLOPT_USERPWD = 0x00002715; + + enum int CURLOPT_VERBOSE = 41; + +// enum int CURLOPT_COOKIE = 22; + enum int CURLOPT_COOKIEFILE = 10031; + enum int CURLOPT_COOKIEJAR = 10082; + + enum int CURLOPT_SSL_VERIFYPEER = 64; + + enum int CURLOPT_FOLLOWLOCATION = 52; + + CURL* curl_easy_init(); + void curl_easy_cleanup(CURL* handle); + CURLcode curl_easy_perform(CURL* curl); + + void curl_global_init(int flags); + + enum int CURL_GLOBAL_ALL = 0b1111; + + CURLcode curl_easy_setopt(CURL* handle, CURLoption option, ...); + curl_slist* curl_slist_append(curl_slist*, const char*); + void curl_slist_free_all(curl_slist*); + + // size is size of item, count is how many items + size_t write_data(void* buffer, size_t size, size_t count, void* user) { + string* str = cast(string*) user; + char* data = cast(char*) buffer; + + assert(size == 1); + + *str ~= data[0..count]; + + return count; + } + + char* curl_easy_strerror(CURLcode errornum ); +} +/* +struct CurlOptions { + string username; + string password; +} +*/ + +import std.md5; +import std.file; +/// this automatically caches to a local file for the given time. it ignores the expires header in favor of your time to keep. +version(linux) +string cachedCurl(string url, int maxCacheHours) { + string res; + + auto cacheFile = "/tmp/arsd-curl-cache-" ~ getDigestString(url); + + if(!std.file.exists(cacheFile) || std.file.lastModified(cacheFile) > 1000 * 60 * 60 * maxCacheHours) { + res = curl(url); + std.file.write(cacheFile, res); + } else { + res = readText(cacheFile); + } + + return res; +} + + +string curl(string url, string data = null, string contentType = "application/x-www-form-urlencoded") { + return curlAuth(url, data, null, null, contentType); +} + +string curlCookie(string cookieFile, string url, string data = null, string contentType = "application/x-www-form-urlencoded") { + return curlAuth(url, data, null, null, contentType, null, null, cookieFile); +} + +string curlAuth(string url, string data = null, string username = null, string password = null, string contentType = "application/x-www-form-urlencoded", string methodOverride = null, string[] customHeaders = null, string cookieJar = null) { + CURL* curl = curl_easy_init(); + if(curl is null) + throw new Exception("curl init"); + scope(exit) + curl_easy_cleanup(curl); + + string ret; + + int res; + + //curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); + + res = curl_easy_setopt(curl, CURLOPT_URL, std.string.toStringz(url)); + if(res != 0) throw new CurlException(res); + if(username !is null) { + res = curl_easy_setopt(curl, CURLOPT_USERPWD, std.string.toStringz(username ~ ":" ~ password)); + if(res != 0) throw new CurlException(res); + } + res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data); + if(res != 0) throw new CurlException(res); + res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ret); + if(res != 0) throw new CurlException(res); + + curl_slist* headers = null; + //if(data !is null) + // contentType = ""; + headers = curl_slist_append(headers, toStringz("Content-Type: " ~ contentType)); + + foreach(h; customHeaders) { + headers = curl_slist_append(headers, toStringz(h)); + } + scope(exit) + curl_slist_free_all(headers); + + if(data) { + res = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.ptr); + if(res != 0) throw new CurlException(res); + res = curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.length); + if(res != 0) throw new CurlException(res); + } + + res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + if(res != 0) throw new CurlException(res); + + if(cookieJar !is null) { + res = curl_easy_setopt(curl, CURLOPT_COOKIEJAR, toStringz(cookieJar)); + if(res != 0) throw new CurlException(res); + res = curl_easy_setopt(curl, CURLOPT_COOKIEFILE, toStringz(cookieJar)); + if(res != 0) throw new CurlException(res); + } + + res = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + if(res != 0) throw new CurlException(res); + //res = curl_easy_setopt(curl, 81, 0); // FIXME verify host + //if(res != 0) throw new CurlException(res); + + res = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); + if(res != 0) throw new CurlException(res); + + if(methodOverride !is null) { + switch(methodOverride) { + default: assert(0); + case "POST": + res = curl_easy_setopt(curl, CURLOPT_POST, 1); + break; + case "GET": + //curl_easy_setopt(curl, CURLOPT_POST, 0); + break; + } + } + + auto failure = curl_easy_perform(curl); + if(failure != 0) + throw new CurlException(failure, "\nURL" ~ url); + + return ret; +} + +class CurlException : Exception { + this(CURLcode code, string msg = null, string file = __FILE__, int line = __LINE__) { + string message = file ~ ":" ~ to!string(line) ~ " (" ~ to!string(code) ~ ") "; + + auto strerror = curl_easy_strerror(code); + + while(*strerror) { + message ~= *strerror; + strerror++; + } + + super(message ~ msg); + } +} + + +import std.conv; diff --git a/database.d b/database.d new file mode 100644 index 0000000..b8c8268 --- /dev/null +++ b/database.d @@ -0,0 +1,1010 @@ +module arsd.database; + +public import std.variant; +import std.string; + +import core.vararg; + +interface Database { + /// Actually implements the query for the database. The query() method + /// below might be easier to use. + ResultSet queryImpl(string sql, Variant[] args...); + + /// Escapes data for inclusion into an sql string literal + string escape(string sqlData); + + /// query to start a transaction, only here because sqlite is apparently different in syntax... + void startTransaction(); + + // FIXME: this would be better as a template, but can't because it is an interface + + /// Just executes a query. It supports placeholders for parameters + /// by using ? in the sql string. NOTE: it only accepts string, int, long, and null types. + /// Others will fail runtime asserts. + final ResultSet query(string sql, ...) { + Variant[] args; + foreach(arg; _arguments) { + string a; + if(arg == typeid(string)) { + a = va_arg!(string)(_argptr); + } else if(arg == typeid(immutable(string))) { + a = va_arg!(immutable(string))(_argptr); + } else if(arg == typeid(const(immutable(char)[]))) { + a = va_arg!(const(immutable(char)[]))(_argptr); + } else if (arg == typeid(int)) { + auto e = va_arg!(int)(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(int))) { + auto e = va_arg!(immutable(int))(_argptr); + a = to!string(e); + } else if (arg == typeid(const(int))) { + auto e = va_arg!(const(int))(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(char))) { + auto e = va_arg!(immutable(char))(_argptr); + a = to!string(e); + } else if (arg == typeid(long)) { + auto e = va_arg!(long)(_argptr); + a = to!string(e); + } else if (arg == typeid(const(long))) { + auto e = va_arg!(const(long))(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(long))) { + auto e = va_arg!(immutable(long))(_argptr); + a = to!string(e); + } else if (arg == typeid(void*)) { + auto e = va_arg!(void*)(_argptr); + assert(e is null, "can only pass null pointer"); + a = null; + } else assert(0, "invalid type " ~ arg.toString ); + + args ~= Variant(a); + } + + return queryImpl(sql, args); + } +} +import std.stdio; + +struct Row { + package string[] row; + package ResultSet resultSet; + + string opIndex(size_t idx) { + return row[idx]; + } + + string opIndex(string idx) { + return row[resultSet.getFieldIndex(idx)]; + } + + string toString() { + return to!string(row); + } + + string[string] toAA() { + string[string] a; + + string[] fn = resultSet.fieldNames(); + + foreach(i, r; row) + a[fn[i]] = r; + + return a; + } + + int opApply(int delegate(ref string, ref string) dg) { + foreach(a, b; toAA) + mixin(yield("a, b")); + + return 0; + } + + + + string[] toStringArray() { + return row; + } +} +import std.conv; + +interface ResultSet { + // name for associative array to result index + int getFieldIndex(string field); + string[] fieldNames(); + + // this is a range that can offer other ranges to access it + bool empty(); + Row front(); + void popFront(); + int length(); + + /* deprecated */ final ResultSet byAssoc() { return this; } +} + +class DatabaseException : Exception { + this(string msg) { + super(msg); + } +} + + + + +// /////////////////////////////////////////////////////// + + +/// Note: ?n params are zero based! +string escapedVariants(Database db, in string sql, Variant[] t) { + + string toSql(Variant a) { + auto v = a.peek!(void*); + if(v && (*v is null)) + return "NULL"; + else { + string str = to!string(a); + return '\'' ~ db.escape(str) ~ '\''; + } + + assert(0); + } + + + + // if nothing to escape or nothing to escape with, don't bother + if(t.length > 0 && sql.indexOf("?") != -1) { + string fixedup; + int currentIndex; + int currentStart = 0; + foreach(i, dchar c; sql) { + if(c == '?') { + fixedup ~= sql[currentStart .. i]; + + int idx = -1; + currentStart = i + 1; + if((i + 1) < sql.length) { + auto n = sql[i + 1]; + if(n >= '0' && n <= '9') { + currentStart = i + 2; + idx = n - '0'; + } + } + if(idx == -1) { + idx = currentIndex; + currentIndex++; + } + + if(idx < 0 || idx >= t.length) + throw new Exception("SQL Parameter index is out of bounds: " ~ to!string(idx) ~ " at `"~sql[0 .. i]~"`"); + + fixedup ~= toSql(t[idx]); + } + } + + fixedup ~= sql[currentStart .. $]; + + return fixedup; + /* + string fixedup; + int pos = 0; + + + void escAndAdd(string str, int q) { + fixedup ~= sql[pos..q] ~ '\'' ~ db.escape(str) ~ '\''; + + } + + foreach(a; t) { + int q = sql[pos..$].indexOf("?"); + if(q == -1) + break; + q += pos; + + auto v = a.peek!(void*); + if(v && (*v is null)) + fixedup ~= sql[pos..q] ~ "NULL"; + else { + string str = to!string(a); + escAndAdd(str, q); + } + + pos = q+1; + } + + fixedup ~= sql[pos..$]; + + sql = fixedup; + */ + } + + return sql; +} + + + + + + +enum UpdateOrInsertMode { + CheckForMe, + AlwaysUpdate, + AlwaysInsert +} + +int updateOrInsert(Database db, string table, string[string] values, string where, UpdateOrInsertMode mode = UpdateOrInsertMode.CheckForMe, string key = "id") { + bool insert = false; + + final switch(mode) { + case UpdateOrInsertMode.CheckForMe: + auto res = db.query("SELECT "~key~" FROM `"~db.escape(table)~"` WHERE " ~ where); + insert = res.empty; + + break; + case UpdateOrInsertMode.AlwaysInsert: + insert = true; + break; + case UpdateOrInsertMode.AlwaysUpdate: + insert = false; + break; + } + + + if(insert) { + string insertSql = "INSERT INTO `" ~ db.escape(table) ~ "` "; + + bool outputted = false; + string vs, cs; + foreach(column, value; values) { + if(column is null) + continue; + if(outputted) { + vs ~= ", "; + cs ~= ", "; + } else + outputted = true; + + //cs ~= "`" ~ db.escape(column) ~ "`"; + cs ~= "`" ~ column ~ "`"; // FIXME: possible insecure + vs ~= "'" ~ db.escape(value) ~ "'"; + } + + if(!outputted) + return 0; + + + insertSql ~= "(" ~ cs ~ ")"; + insertSql ~= " VALUES "; + insertSql ~= "(" ~ vs ~ ")"; + + db.query(insertSql); + + return 0; // db.lastInsertId; + } else { + string updateSql = "UPDATE `"~db.escape(table)~"` SET "; + + bool outputted = false; + foreach(column, value; values) { + if(column is null) + continue; + if(outputted) + updateSql ~= ", "; + else + outputted = true; + + updateSql ~= "`" ~ db.escape(column) ~ "` = '" ~ db.escape(value) ~ "'"; + } + + if(!outputted) + return 0; + + updateSql ~= " WHERE " ~ where; + + db.query(updateSql); + return 0; + } +} + + + + + +string fixupSqlForDataObjectUse(string sql) { + + string[] tableNames; + + string piece = sql; + int idx; + while((idx = piece.indexOf("JOIN")) != -1) { + auto start = idx + 5; + auto i = start; + while(piece[i] != ' ' && piece[i] != '\n' && piece[i] != '\t' && piece[i] != ',') + i++; + auto end = i; + + tableNames ~= strip(piece[start..end]); + + piece = piece[end..$]; + } + + idx = sql.indexOf("FROM"); + if(idx != -1) { + auto start = idx + 5; + auto i = start; + start = i; + while(i < sql.length && !(sql[i] > 'A' && sql[i] <= 'Z')) // if not uppercase, except for A (for AS) to avoid SQL keywords (hack) + i++; + + auto from = sql[start..i]; + auto pieces = from.split(","); + foreach(p; pieces) { + p = p.strip; + start = 0; + i = 0; + while(i < p.length && p[i] != ' ' && p[i] != '\n' && p[i] != '\t' && p[i] != ',') + i++; + + tableNames ~= strip(p[start..i]); + } + + string sqlToAdd; + foreach(tbl; tableNames) { + if(tbl.length) { + sqlToAdd ~= ", " ~ tbl ~ ".id" ~ " AS " ~ "id_from_" ~ tbl; + } + } + + sqlToAdd ~= " "; + + sql = sql[0..idx] ~ sqlToAdd ~ sql[idx..$]; + } + + return sql; +} + + + + + +/* + This is like a result set + + + DataObject res = [...]; + + res.name = "Something"; + + res.commit; // runs the actual update or insert + + + res = new DataObject(fields, tables + + + + + + + + when doing a select, we need to figure out all the tables and modify the query to include the ids we need + + + search for FROM and JOIN + the next token is the table name + + right before the FROM, add the ids of each table + + + given: + SELECT name, phone FROM customers LEFT JOIN phones ON customer.id = phones.cust_id + + we want: + SELECT name, phone, customers.id AS id_from_customers, phones.id AS id_from_phones FROM customers LEFT JOIN phones ON customer.id[...]; + +*/ + +string yield(string what) { return `if(auto result = dg(`~what~`)) return result;`; } + +import std.typecons; +import std.json; // for json value making +class DataObject { + // lets you just free-form set fields, assuming they all come from the given table + // note it doesn't try to handle joins for new rows. you've gotta do that yourself + this(Database db, string table) { + assert(db !is null); + this.db = db; + this.table = table; + + mode = UpdateOrInsertMode.CheckForMe; + } + + JSONValue makeJsonValue() { + JSONValue val; + val.type = JSON_TYPE.OBJECT; + foreach(k, v; fields) { + JSONValue s; + s.type = JSON_TYPE.STRING; + s.str = v; + val.object[k] = s; + } + return val; + } + + this(Database db, string[string] res, Tuple!(string, string)[string] mappings) { + this.db = db; + this.mappings = mappings; + this.fields = res; + + mode = UpdateOrInsertMode.AlwaysUpdate; + } + + string table; + // table, column [alias] + Tuple!(string, string)[string] mappings; + + // vararg hack so property assignment works right, even with null + string opDispatch(string field)(...) + if((field.length < 8 || field[0..8] != "id_from_") && field != "popFront") + { + if(_arguments.length == 0) { + if(field !in fields) + throw new Exception("no such field " ~ field); + + return fields[field]; + } else if(_arguments.length == 1) { + auto arg = _arguments[0]; + + string a; + if(arg == typeid(string)) { + a = va_arg!(string)(_argptr); + } else if(arg == typeid(immutable(string))) { + a = va_arg!(immutable(string))(_argptr); + } else if(arg == typeid(const(immutable(char)[]))) { + a = va_arg!(const(immutable(char)[]))(_argptr); + } else if (arg == typeid(int)) { + auto e = va_arg!(int)(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(int))) { + auto e = va_arg!(immutable(int))(_argptr); + a = to!string(e); + } else if (arg == typeid(const(int))) { + auto e = va_arg!(const(int))(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(char))) { + auto e = va_arg!(immutable(char))(_argptr); + a = to!string(e); + } else if (arg == typeid(long)) { + auto e = va_arg!(long)(_argptr); + a = to!string(e); + } else if (arg == typeid(const(long))) { + auto e = va_arg!(const(long))(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(long))) { + auto e = va_arg!(immutable(long))(_argptr); + a = to!string(e); + } else if (arg == typeid(void*)) { + auto e = va_arg!(void*)(_argptr); + assert(e is null, "can only pass null pointer"); + a = null; + } else assert(0, "invalid type " ~ arg.toString ); + + + auto setTo = a; + setImpl(field, setTo); + + return setTo; + + } else assert(0, "too many arguments"); + + assert(0); // should never be reached + } + + private void setImpl(string field, string value) { + if(field in fields) { + if(fields[field] != value) + changed[field] = true; + } else { + changed[field] = true; + } + + fields[field] = value; + } + + int opApply(int delegate(ref string) dg) { + foreach(a; fields) + mixin(yield("a")); + + return 0; + } + + int opApply(int delegate(ref string, ref string) dg) { + foreach(a, b; fields) + mixin(yield("a, b")); + + return 0; + } + + + string opIndex(string field) { + if(field !in fields) + throw new DatabaseException("No such field in data object: " ~ field); + return fields[field]; + } + + string opIndexAssign(string value, string field) { + setImpl(field, value); + return value; + } + + string* opBinary(string op)(string key) if(op == "in") { + return key in fields; + } + + string[string] fields; + bool[string] changed; + + void commitChanges() { + commitChanges(cast(string) null, null); + } + + void commitChanges(string key, string keyField) { + commitChanges(key is null ? null : [key], keyField is null ? null : [keyField]); + } + + void commitChanges(string[] keys, string[] keyFields = null) { + string[string][string] toUpdate; + int updateCount = 0; + foreach(field, c; changed) { + if(c) { + string tbl, col; + if(mappings is null) { + tbl = this.table; + col = field; + } else { + if(field !in mappings) + assert(0, "no such mapping for " ~ field); + auto m = mappings[field]; + tbl = m[0]; + col = m[1]; + } + + toUpdate[tbl][col] = fields[field]; + updateCount++; + } + } + + if(updateCount) { + db.startTransaction(); + scope(success) db.query("COMMIT"); + scope(failure) db.query("ROLLBACK"); + + foreach(tbl, values; toUpdate) { + string where, keyFieldToPass; + + if(keys is null) { + keys = [null]; + } + + foreach(i, key; keys) { + string keyField; + + if(key is null) { + key = "id_from_" ~ tbl; + if(key !in fields) + key = "id"; + } + + if(i >= keyFields.length || keyFields[i] is null) { + if(key == "id_from_" ~ tbl) + keyField = "id"; + else + keyField = key; + } else { + keyField = keyFields[i]; + } + + + if(where.length) + where ~= " AND "; + + where ~= keyField ~ " = '"~db.escape(key in fields ? fields[key] : null)~"'" ; + if(keyFieldToPass.length) + keyFieldToPass ~= ", "; + + keyFieldToPass ~= keyField; + } + + + + updateOrInsert(db, tbl, values, where, mode, keyFieldToPass); + } + + changed = null; + } + } + + void commitDelete() { + if(mode == UpdateOrInsertMode.AlwaysInsert) + throw new Exception("Cannot delete an item not in the database"); + + assert(table.length); // FIXME, should work with fancy items too + + // FIXME: escaping and primary key questions + db.query("DELETE FROM " ~ table ~ " WHERE id = '" ~ db.escape(fields["id"]) ~ "'"); + } + + string getAlias(string table, string column) { + string ali; + if(mappings is null) { + if(this.table is null) { + mappings[column] = tuple(table, column); + return column; + } else { + assert(table == this.table); + ali = column; + } + } else { + foreach(a, what; mappings) + if(what[0] == table && what[1] == column + && a.indexOf("id_from_") == -1) { + ali = a; + break; + } + } + + return ali; + } + + void set(string table, string column, string value) { + string ali = getAlias(table, column); + //assert(ali in fields); + setImpl(ali, value); + } + + string select(string table, string column) { + string ali = getAlias(table, column); + //assert(ali in fields); + if(ali in fields) + return fields[ali]; + return null; + } + + DataObject addNew() { + auto n = new DataObject(db, null); + + n.db = this.db; + n.table = this.table; + n.mappings = this.mappings; + + foreach(k, v; this.fields) + if(k.indexOf("id_from_") == -1) + n.fields[k] = v; + else + n.fields[k] = null; // don't copy ids + + n.mode = UpdateOrInsertMode.AlwaysInsert; + + return n; + } + + Database db; + UpdateOrInsertMode mode; +} + +/** + You can subclass DataObject if you want to + get some compile time checks or better types. + + You'll want to disable opDispatch, then forward your + properties to the super opDispatch. +*/ + +/*mixin*/ string DataObjectField(T, string table, string column, string aliasAs = null)() { + string aliasAs_; + if(aliasAs is null) + aliasAs_ = column; + else + aliasAs_ = aliasAs; + return ` + @property void `~aliasAs_~`(`~T.stringof~` setTo) { + super.set("`~table~`", "`~column~`", to!string(setTo)); + } + + @property `~T.stringof~` `~aliasAs_~` () { + return to!(`~T.stringof~`)(super.select("`~table~`", "`~column~`")); + } + `; +} + +mixin template StrictDataObject() { + // disable opdispatch + string opDispatch(string name)(...) if (0) {} +} + + +string createDataObjectFieldsFromAlias(string table, fieldsToUse)() { + string ret; + + fieldsToUse f; + foreach(member; __traits(allMembers, fieldsToUse)) { + ret ~= DataObjectField!(typeof(__traits(getMember, f, member)), table, member); + } + + return ret; +} + + +/** + This creates an editable data object out of a simple struct. + + struct MyFields { + int id; + string name; + } + + alias SimpleDataObject!("my_table", MyFields) User; + + + User a = new User(db); + + a.id = 30; + a.name = "hello"; + a.commitChanges(); // tries an update or insert on the my_table table + + + Unlike the base DataObject class, this template provides compile time + checking for types and names, based on the struct you pass in: + + a.id = "aa"; // compile error + + a.notAField; // compile error +*/ +class SimpleDataObject(string tableToUse, fieldsToUse) : DataObject { + mixin StrictDataObject!(); + + mixin(createDataObjectFieldsFromAlias!(tableToUse, fieldsToUse)()); + + this(Database db) { + super(db, tableToUse); + } +} + +/** + Given some SQL, it finds the CREATE TABLE + instruction for the given tableName. + (this is so it can find one entry from + a file with several SQL commands. But it + may break on a complex file, so try to only + feed it simple sql files.) + + From that, it pulls out the members to create a + simple struct based on it. + + It's not terribly smart, so it will probably + break on complex tables. + + Data types handled: + INTEGER, SMALLINT, MEDIUMINT -> D's int + TINYINT -> D's bool + BIGINT -> D's long + TEXT, VARCHAR -> D's string + FLOAT, DOUBLE -> D's double + + It also reads DEFAULT values to pass to D, except for NULL. + It ignores any length restrictions. + + Bugs: + Skips all constraints + Doesn't handle nullable fields, except with strings + It only handles SQL keywords if they are all caps + + This, when combined with SimpleDataObject!(), + can automatically create usable D classes from + SQL input. +*/ +struct StructFromCreateTable(string sql, string tableName) { + mixin(getCreateTable(sql, tableName)); +} + +string getCreateTable(string sql, string tableName) { + skip: + while(readWord(sql) != "CREATE") {} + + assert(readWord(sql) == "TABLE"); + + if(readWord(sql) != tableName) + goto skip; + + assert(readWord(sql) == "("); + + int state; + int parens; + + struct Field { + string name; + string type; + string defaultValue; + } + Field[] fields; + + string word = readWord(sql); + Field current; + while(word != ")" || parens) { + if(word == ")") { + parens --; + word = readWord(sql); + continue; + } + if(word == "(") { + parens ++; + word = readWord(sql); + continue; + } + switch(state) { + default: assert(0); + case 0: + if(word[0] >= 'A' && word[0] <= 'Z') { + state = 4; + break; // we want to skip this since it starts with a keyword (we hope) + } + current.name = word; + state = 1; + break; + case 1: + current.type ~= word; + state = 2; + break; + case 2: + if(word == "DEFAULT") + state = 3; + else if (word == ",") { + fields ~= current; + current = Field(); + state = 0; // next + } + break; + case 3: + current.defaultValue = word; + state = 2; // back to skipping + break; + case 4: + if(word == ",") + state = 0; + } + + word = readWord(sql); + } + + if(current.name !is null) + fields ~= current; + + + string structCode; + foreach(field; fields) { + structCode ~= "\t"; + + switch(field.type) { + case "INTEGER": + case "SMALLINT": + case "MEDIUMINT": + structCode ~= "int"; + break; + case "BOOLEAN": + case "TINYINT": + structCode ~= "bool"; + break; + case "BIGINT": + structCode ~= "long"; + break; + case "CHAR": + case "char": + case "VARCHAR": + case "varchar": + case "TEXT": + case "text": + structCode ~= "string"; + break; + case "FLOAT": + case "DOUBLE": + structCode ~= "double"; + break; + default: + assert(0, "unknown type " ~ field.type ~ " for " ~ field.name); + } + + structCode ~= " "; + structCode ~= field.name; + + if(field.defaultValue !is null) { + structCode ~= " = " ~ field.defaultValue; + } + + structCode ~= ";\n"; + } + + return structCode; +} + +string readWord(ref string src) { + reset: + while(src[0] == ' ' || src[0] == '\t' || src[0] == '\n') + src = src[1..$]; + if(src.length >= 2 && src[0] == '-' && src[1] == '-') { // a comment, skip it + while(src[0] != '\n') + src = src[1..$]; + goto reset; + } + + int start, pos; + if(src[0] == '`') { + src = src[1..$]; + while(src[pos] != '`') + pos++; + goto gotit; + } + + + while( + (src[pos] >= 'A' && src[pos] <= 'Z') + || + (src[pos] >= 'a' && src[pos] <= 'z') + || + (src[pos] >= '0' && src[pos] <= '9') + || + src[pos] == '_' + ) + pos++; + gotit: + if(pos == 0) + pos = 1; + + string tmp = src[0..pos]; + + if(src[pos] == '`') + pos++; // skip the ending quote; + + src = src[pos..$]; + + return tmp; +} + +/// Combines StructFromCreateTable and SimpleDataObject into a one-stop template. +/// alias DataObjectFromSqlCreateTable(import("file.sql"), "my_table") MyTable; +template DataObjectFromSqlCreateTable(string sql, string tableName) { + alias SimpleDataObject!(tableName, StructFromCreateTable!(sql, tableName)) DataObjectFromSqlCreateTable; +} + +/+ +class MyDataObject : DataObject { + this() { + super(new Database("localhost", "root", "pass", "social"), null); + } + + mixin StrictDataObject!(); + + mixin(DataObjectField!(int, "users", "id")); +} + +void main() { + auto a = new MyDataObject; + + a.fields["id"] = "10"; + + a.id = 34; + + a.commitChanges; +} ++/ + +/* +alias DataObjectFromSqlCreateTable!(import("db.sql"), "users") Test; + +void main() { + auto a = new Test(null); + + a.cool = "way"; + a.value = 100; +} +*/ + +void typeinfoBugWorkaround() { + assert(0, to!string(typeid(immutable(char[])[immutable(char)[]]))); +} diff --git a/dom.d b/dom.d new file mode 120000 index 0000000..505b2fa --- /dev/null +++ b/dom.d @@ -0,0 +1 @@ +/home/me/program/djs/dom.d \ No newline at end of file diff --git a/http.d b/http.d new file mode 100644 index 0000000..aa88f54 --- /dev/null +++ b/http.d @@ -0,0 +1,221 @@ +module arsd.http; + +import std.stdio; + + +/** + Gets a textual document, ignoring headers. Throws on non-text or error. +*/ +string get(string url) { + auto hr = httpRequest("GET", url); + if(hr.code != 200) + throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); + if(hr.contentType.indexOf("text/") == -1) + throw new Exception(hr.contentType ~ " is bad content for conversion to string"); + return cast(string) hr.content; + +} + +static import std.uri; + +string post(string url, string[string] args) { + string content; + + foreach(name, arg; args) { + if(content.length) + content ~= "&"; + content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); + } + + auto hr = httpRequest("POST", url, cast(ubyte[]) content, ["Content-Type: application/x-www-form-urlencoded"]); + if(hr.code != 200) + throw new Exception(format("HTTP answered %d instead of 200", hr.code)); + if(hr.contentType.indexOf("text/") == -1) + throw new Exception(hr.contentType ~ " is bad content for conversion to string"); + + return cast(string) hr.content; +} + +struct HttpResponse { + int code; + string contentType; + string[] headers; + ubyte[] content; +} + +import std.string; +static import std.algorithm; +import std.conv; + +struct UriParts { + string original; + string method; + string host; + ushort port; + string path; + + this(string uri) { + original = uri; + if(uri[0..7] != "http://") + throw new Exception("You must use an absolute, unencrypted URL."); + + int posSlash = uri[7..$].indexOf("/"); + if(posSlash != -1) + posSlash += 7; + + if(posSlash == -1) + posSlash = uri.length; + + int posColon = uri[7..$].indexOf(":"); + if(posColon != -1) + posColon += 7; + + port = 80; + + if(posColon != -1 && posColon < posSlash) { + host = uri[7..posColon]; + port = to!ushort(uri[posColon+1..posSlash]); + } else + host = uri[7..posSlash]; + + path = uri[posSlash..$]; + if(path == "") + path = "/"; + } +} + +HttpResponse httpRequest(string method, string uri, const(ubyte)[] content = null, string headers[] = null) { + auto u = UriParts(uri); + auto f = openNetwork(u.host, u.port); + + return doHttpRequestOnFile(f, method, uri, content, headers); +} + +/** + Executes a generic http request, returning the full result. The correct formatting + of the parameters are the caller's responsibility. Content-Length is added automatically, + but YOU must give Content-Type! +*/ +HttpResponse doHttpRequestOnFile(File f, string method, string uri, const(ubyte)[] content = null, string headers[] = null) + in { + assert(method == "POST" || method == "GET"); + } +body { + auto u = UriParts(uri); + + f.writefln("%s %s HTTP/1.1", method, u.path); + f.writefln("Host: %s", u.host); + f.writefln("Connection: close"); + if(content !is null) + f.writefln("Content-Length: %d", content.length); + if(headers !is null) + foreach(header; headers) + f.writefln("%s", header); + f.writefln(""); + if(content !is null) + f.rawWrite(content); + + + HttpResponse hr; + cont: + string l = f.readln(); + if(l[0..9] != "HTTP/1.1 ") + throw new Exception("Not talking to a http server"); + + hr.code = to!int(l[9..12]); // HTTP/1.1 ### OK + + if(hr.code == 100) { // continue + do { + l = readln(); + } while(l.length > 1); + + goto cont; + } + + bool chunked = false; + + foreach(line; f.byLine) { + if(line.length <= 1) + break; + hr.headers ~= line.idup; + if(line.startsWith("Content-Type: ")) + hr.contentType = line[14..$-1].idup; + if(line.startsWith("Transfer-Encoding: chunked")) + chunked = true; + } + + ubyte[] response; + foreach(ubyte[] chunk; f.byChunk(4096)) { + response ~= chunk; + } + + + if(chunked) { + // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character + // read binary data of that length. it is our content + // repeat until a zero sized chunk + // then read footers as headers. + + int state = 0; + int size; + int start = 0; + for(int a = 0; a < response.length; a++) { + switch(state) { + case 0: // reading hex + char c = response[a]; + if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')) { + // just keep reading + } else { + int power = 1; + size = 0; + for(int b = a-1; b >= start; b--) { + char cc = response[b]; + if(cc >= 'a' && cc <= 'z') + cc -= 0x20; + int val = 0; + if(cc >= '0' && cc <= '9') + val = cc - '0'; + else + val = cc - 'A'; + + size += power * val; + power *= 16; + } + state++; + continue; + } + break; + case 1: // reading until end of line + char c = response[a]; + if(c == '\n') { + if(size == 0) + state = 3; + else + state = 2; + } + break; + case 2: // reading data + hr.content ~= response[a..a+size]; + a += size; + a+= 2; // skipping a 13 10 + start = a; + state = 0; + break; + case 3: // reading footers + goto done; // FIXME + break; + } + } + } else + hr.content = response; + done: + + return hr; +} + + +/* +void main(string args[]) { + write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); +} +*/ diff --git a/mysql.d b/mysql.d new file mode 100644 index 0000000..ae50ffc --- /dev/null +++ b/mysql.d @@ -0,0 +1,733 @@ +module arsd.mysql; +pragma(lib, "mysqlclient"); + +public import arsd.database; + +import std.stdio; +import std.exception; +import std.string; +import std.conv; +import std.typecons; + +class MySqlResult : ResultSet { + private int[string] mapping; + private MYSQL_RES* result; + + private int itemsTotal; + private int itemsUsed; + + string sql; + + this(MYSQL_RES* r, string sql) { + result = r; + itemsTotal = length(); + itemsUsed = 0; + + this.sql = sql; + + // prime it + if(itemsTotal) + fetchNext(); + } + + ~this() { + if(result !is null) + mysql_free_result(result); + } + + + MYSQL_FIELD[] fields() { + int numFields = mysql_num_fields(result); + auto fields = mysql_fetch_fields(result); + + MYSQL_FIELD[] ret; + for(int i = 0; i < numFields; i++) { + ret ~= fields[i]; + } + + return ret; + } + + + override int length() { + if(result is null) + return 0; + return cast(int) mysql_num_rows(result); + } + + override bool empty() { + return itemsUsed == itemsTotal; + } + + override Row front() { + return row; + } + + override void popFront() { + itemsUsed++; + if(itemsUsed < itemsTotal) { + fetchNext(); + } + } + + override int getFieldIndex(string field) { + if(mapping is null) + makeFieldMapping(); + debug { + if(field !in mapping) + throw new Exception(field ~ " not in result"); + } + return mapping[field]; + } + + private void makeFieldMapping() { + int numFields = mysql_num_fields(result); + auto fields = mysql_fetch_fields(result); + + for(int i = 0; i < numFields; i++) { + mapping[fromCstring(fields[i].name)] = i; + } + } + + private void fetchNext() { + assert(result); + auto r = mysql_fetch_row(result); + uint numFields = mysql_num_fields(result); + uint* lengths = mysql_fetch_lengths(result); + string[] row; + // potential FIXME: not really binary safe + + columnIsNull.length = numFields; + for(int a = 0; a < numFields; a++) { + if(*(r+a) is null) { + row ~= null; + columnIsNull[a] = true; + } else { + row ~= fromCstring(*(r+a), *(lengths + a)); + columnIsNull[a] = false; + } + } + + this.row.row = row; + this.row.resultSet = this; + } + + + override string[] fieldNames() { + int numFields = mysql_num_fields(result); + auto fields = mysql_fetch_fields(result); + + string[] names; + for(int i = 0; i < numFields; i++) { + names ~= fromCstring(fields[i].name); + } + + return names; + } + + + + bool[] columnIsNull; + Row row; +} + + + + +class MySql : Database { + this(string host, string user, string pass, string db) { + mysql = enforceEx!(DatabaseException)( + mysql_init(null), + "Couldn't init mysql"); + enforceEx!(DatabaseException)( + mysql_real_connect(mysql, toCstring(host), toCstring(user), toCstring(pass), toCstring(db), 0, null, 0), + error()); + + dbname = db; + + // we want UTF8 for everything + + query("SET NAMES 'utf8'"); + //query("SET CHARACTER SET utf8"); + } + + string dbname; + + override void startTransaction() { + query("START TRANSACTION"); + } + + string error() { + return fromCstring(mysql_error(mysql)); + } + + ~this() { + mysql_close(mysql); + } + + int lastInsertId() { + return cast(int) mysql_insert_id(mysql); + } + + + + int insert(string table, MySqlResult result, string[string] columnsToModify, string[] columnsToSkip) { + assert(!result.empty); + string sql = "INSERT INTO `" ~ table ~ "` "; + + string cols = "("; + string vals = "("; + bool outputted = false; + + string[string] columns; + auto cnames = result.fieldNames; + foreach(i, col; result.front.toStringArray) { + bool skipMe = false; + foreach(skip; columnsToSkip) { + if(cnames[i] == skip) { + skipMe = true; + break; + } + } + if(skipMe) + continue; + + if(outputted) { + cols ~= ","; + vals ~= ","; + } else + outputted = true; + + cols ~= cnames[i]; + + if(result.columnIsNull[i] && cnames[i] !in columnsToModify) + vals ~= "NULL"; + else { + string v = col; + if(cnames[i] in columnsToModify) + v = columnsToModify[cnames[i]]; + + vals ~= "'" ~ escape(v) ~ "'"; + + } + } + + cols ~= ")"; + vals ~= ")"; + + sql ~= cols ~ " VALUES " ~ vals; + + query(sql); + + result.popFront; + + return lastInsertId; + } + + string escape(string str) { + ubyte[] buffer = new ubyte[str.length * 2 + 1]; + buffer.length = mysql_real_escape_string(mysql, buffer.ptr, cast(cstring) str.ptr, str.length); + + return cast(string) buffer; + } + + string escaped(T...)(string sql, T t) { + static if(t.length > 0) { + string fixedup; + int pos = 0; + + + void escAndAdd(string str, int q) { + ubyte[] buffer = new ubyte[str.length * 2 + 1]; + buffer.length = mysql_real_escape_string(mysql, buffer.ptr, cast(cstring) str.ptr, str.length); + + fixedup ~= sql[pos..q] ~ '\'' ~ cast(string) buffer ~ '\''; + + } + + foreach(a; t) { + int q = sql[pos..$].indexOf("?"); + if(q == -1) + break; + q += pos; + + static if(__traits(compiles, t is null)) { + if(t is null) + fixedup ~= sql[pos..q] ~ "NULL"; + else + escAndAdd(to!string(*a), q); + } else { + string str = to!string(a); + escAndAdd(str, q); + } + + pos = q+1; + } + + fixedup ~= sql[pos..$]; + + sql = fixedup; + + //writefln("\n\nExecuting sql: %s", sql); + } + + return sql; + } + + + + + + + + + + + + + + + + + + + + ResultByDataObject queryDataObject(T...)(string sql, T t) { + // modify sql for the best data object grabbing + sql = fixupSqlForDataObjectUse(sql); + + auto magic = query(sql, t); + return ResultByDataObject(cast(MySqlResult) magic, this); + } + + + + + + + + int affectedRows() { + return cast(int) mysql_affected_rows(mysql); + } + + override ResultSet queryImpl(string sql, Variant[] args...) { + sql = escapedVariants(this, sql, args); + + enforceEx!(DatabaseException)( + !mysql_query(mysql, toCstring(sql)), + error() ~ " :::: " ~ sql); + + return new MySqlResult(mysql_store_result(mysql), sql); + } +/+ + Result queryOld(T...)(string sql, T t) { + sql = escaped(sql, t); + + if(sql.length == 0) + throw new DatabaseException("empty query"); + /* + static int queryCount = 0; + queryCount++; + if(sql.indexOf("INSERT") != -1) + stderr.writefln("%d: %s", queryCount, sql.replace("\n", " ").replace("\t", "")); + */ + + version(dryRun) { + pragma(msg, "This is a dry run compile, no queries will be run"); + writeln(sql); + return Result(null, null); + } + + enforceEx!(DatabaseException)( + !mysql_query(mysql, toCstring(sql)), + error() ~ " :::: " ~ sql); + + return Result(mysql_store_result(mysql), sql); + } ++/ +/+ + struct ResultByAssoc { + this(Result* r) { + result = r; + fields = r.fieldNames(); + } + + ulong length() { return result.length; } + bool empty() { return result.empty; } + void popFront() { result.popFront(); } + string[string] front() { + auto r = result.front; + string[string] ret; + foreach(i, a; r) { + ret[fields[i]] = a; + } + + return ret; + } + + @disable this(this) { } + + string[] fields; + Result* result; + } + + + struct ResultByStruct(T) { + this(Result* r) { + result = r; + fields = r.fieldNames(); + } + + ulong length() { return result.length; } + bool empty() { return result.empty; } + void popFront() { result.popFront(); } + T front() { + auto r = result.front; + string[string] ret; + foreach(i, a; r) { + ret[fields[i]] = a; + } + + T s; + // FIXME: should use tupleOf + foreach(member; s.tupleof) { + if(member.stringof in ret) + member = to!(typeof(member))(ret[member]); + } + + return s; + } + + @disable this(this) { } + + string[] fields; + Result* result; + } ++/ + +/+ + + + struct Result { + private Result* heaped() { + auto r = new Result(result, sql, false); + + r.tupleof = this.tupleof; + + this.itemsTotal = 0; + this.result = null; + + return r; + } + + this(MYSQL_RES* r, string sql, bool prime = true) { + result = r; + itemsTotal = length; + itemsUsed = 0; + this.sql = sql; + // prime it here + if(prime && itemsTotal) + fetchNext(); + } + + string sql; + + ~this() { + if(result !is null) + mysql_free_result(result); + } + + /+ + string[string][] fetchAssoc() { + + } + +/ + + ResultByAssoc byAssoc() { + return ResultByAssoc(&this); + } + + ResultByStruct!(T) byStruct(T)() { + return ResultByStruct!(T)(&this); + } + + string[] fieldNames() { + int numFields = mysql_num_fields(result); + auto fields = mysql_fetch_fields(result); + + string[] names; + for(int i = 0; i < numFields; i++) { + names ~= fromCstring(fields[i].name); + } + + return names; + } + + MYSQL_FIELD[] fields() { + int numFields = mysql_num_fields(result); + auto fields = mysql_fetch_fields(result); + + MYSQL_FIELD[] ret; + for(int i = 0; i < numFields; i++) { + ret ~= fields[i]; + } + + return ret; + } + + ulong length() { + if(result is null) + return 0; + return mysql_num_rows(result); + } + + bool empty() { + return itemsUsed == itemsTotal; + } + + Row front() { + return row; + } + + void popFront() { + itemsUsed++; + if(itemsUsed < itemsTotal) { + fetchNext(); + } + } + + void fetchNext() { + auto r = mysql_fetch_row(result); + uint numFields = mysql_num_fields(result); + uint* lengths = mysql_fetch_lengths(result); + row.length = 0; + // potential FIXME: not really binary safe + + columnIsNull.length = numFields; + for(int a = 0; a < numFields; a++) { + if(*(r+a) is null) { + row ~= null; + columnIsNull[a] = true; + } else { + row ~= fromCstring(*(r+a), *(lengths + a)); + columnIsNull[a] = false; + } + } + } + + @disable this(this) {} + private MYSQL_RES* result; + + ulong itemsTotal; + ulong itemsUsed; + + alias string[] Row; + + Row row; + bool[] columnIsNull; // FIXME: should be part of the row + } ++/ + private: + MYSQL* mysql; +} + +struct ResultByDataObject { + this(MySqlResult r, MySql mysql) { + result = r; + auto fields = r.fields(); + this.mysql = mysql; + + foreach(i, f; fields) { + string tbl = fromCstring(f.org_table is null ? f.table : f.org_table); + mappings[fromCstring(f.name)] = tuple( + tbl, + fromCstring(f.org_name is null ? f.name : f.org_name)); + } + + + } + + Tuple!(string, string)[string] mappings; + + ulong length() { return result.length; } + bool empty() { return result.empty; } + void popFront() { result.popFront(); } + DataObject front() { + return new DataObject(mysql, result.front.toAA, mappings); + } + // would it be good to add a new() method? would be valid even if empty + // it'd just fill in the ID's at random and allow you to do the rest + + @disable this(this) { } + + MySqlResult result; + MySql mysql; +} + +extern(C) { + typedef void MYSQL; + typedef void MYSQL_RES; + typedef const(ubyte)* cstring; + + struct MYSQL_FIELD { + cstring name; /* Name of column */ + cstring org_name; /* Original column name, if an alias */ + cstring table; /* Table of column if column was a field */ + cstring org_table; /* Org table name, if table was an alias */ + cstring db; /* Database for table */ + cstring catalog; /* Catalog for table */ + cstring def; /* Default value (set by mysql_list_fields) */ + uint length; /* Width of column (create length) */ + uint max_length; /* Max width for selected set */ + uint name_length; + uint org_name_length; + uint table_length; + uint org_table_length; + uint db_length; + uint catalog_length; + uint def_length; + uint flags; /* Div flags */ + uint decimals; /* Number of decimals in field */ + uint charsetnr; /* Character set */ + uint type; /* Type of field. See mysql_com.h for types */ + // type is actually an enum btw + } + + typedef cstring* MYSQL_ROW; + + cstring mysql_get_client_info(); + MYSQL* mysql_init(MYSQL*); + uint mysql_errno(MYSQL*); + cstring mysql_error(MYSQL*); + + MYSQL* mysql_real_connect(MYSQL*, cstring, cstring, cstring, cstring, uint, cstring, ulong); + + int mysql_query(MYSQL*, cstring); + + void mysql_close(MYSQL*); + + ulong mysql_num_rows(MYSQL_RES*); + uint mysql_num_fields(MYSQL_RES*); + bool mysql_eof(MYSQL_RES*); + + ulong mysql_affected_rows(MYSQL*); + ulong mysql_insert_id(MYSQL*); + + MYSQL_RES* mysql_store_result(MYSQL*); + MYSQL_RES* mysql_use_result(MYSQL*); + + MYSQL_ROW mysql_fetch_row(MYSQL_RES *); + uint* mysql_fetch_lengths(MYSQL_RES*); + MYSQL_FIELD* mysql_fetch_field(MYSQL_RES*); + MYSQL_FIELD* mysql_fetch_fields(MYSQL_RES*); + + uint mysql_real_escape_string(MYSQL*, ubyte* to, cstring from, uint length); + + void mysql_free_result(MYSQL_RES*); + +} + +import std.string; +cstring toCstring(string c) { + return cast(cstring) toStringz(c); +} + +import std.array; +string fromCstring(cstring c, int len = -1) { + string ret; + if(c is null) + return null; + if(len == -1) { + while(*c) { + ret ~= cast(char) *c; + c++; + } + } else + for(int a = 0; a < len; a++) + ret ~= cast(char) *(a+c); + + return ret; +} + +/* +void main() { + auto mysql = new MySql("localhost", "uname", "password", "test"); + scope(exit) delete mysql; + + mysql.query("INSERT INTO users (id, password) VALUES (?, ?)", 10, "lol"); + + foreach(row; mysql.query("SELECT * FROM users")) { + writefln("%s %s %s %s", row["id"], row[0], row[1], row["username"]); + } +} +*/ + +/* +struct ResultByStruct(T) { + this(MySql.Result* r) { + result = r; + fields = r.fieldNames(); + } + + ulong length() { return result.length; } + bool empty() { return result.empty; } + void popFront() { result.popFront(); } + T front() { + auto r = result.front; + T ret; + foreach(i, a; r) { + ret[fields[i]] = a; + } + + return ret; + } + + @disable this(this) { } + + string[] fields; + MySql.Result* result; +} +*/ + + +/+ + mysql.linq.tablename.field[key] // select field from tablename where id = key + + mysql.link["name"].table.field[key] // select field from table where name = key + + + auto q = mysql.prepQuery("select id from table where something"); + q.sort("name"); + q.limit(start, count); + q.page(3, pagelength = ?); + + q.execute(params here); // returns the same Result range as query ++/ + +/* +void main() { + auto db = new MySql("localhost", "uname", "password", "test"); + foreach(item; db.queryDataObject("SELECT users.*, username + FROM users, password_manager_accounts + WHERE password_manager_accounts.user_id = users.id LIMIT 5")) { + writefln("item: %s, %s", item.id, item.username); + item.first = "new"; + item.last = "new2"; + item.username = "kill"; + //item.commitChanges(); + } +} +*/ + + +/* +Copyright: Adam D. Ruppe, 2009 - 2011 +License: Boost License 1.0. +Authors: Adam D. Ruppe + + Copyright Adam D. Ruppe 2009 - 2011. +Distributed under the Boost Software License, Version 1.0. + (See accompanying file LICENSE_1_0.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +*/ + diff --git a/png.d b/png.d new file mode 100644 index 0000000..de2b091 --- /dev/null +++ b/png.d @@ -0,0 +1,600 @@ +module arsd.png; + +// By Adam D. Ruppe, 2009-2010, released into the public domain +import std.stdio; +import std.conv; +import std.file; + +import std.zlib; + +public import arsd.image; + +/** + The return value should be casted to indexed or truecolor depending on what you need. + + To get an image from a png file, do this: + + auto i = cast(TrueColorImage) imageFromPng(readPng(cast(ubyte)[]) std.file.read("file.png"))); +*/ +Image imageFromPng(PNG* png) { + PNGHeader h = getHeader(png); + + return new IndexedImage(h.width, h.height); +} + +/* +struct PNGHeader { + uint width; + uint height; + ubyte depth = 8; + ubyte type = 6; // 0 - greyscale, 2 - truecolor, 3 - indexed color, 4 - grey with alpha, 6 - true with alpha + ubyte compressionMethod = 0; // should be zero + ubyte filterMethod = 0; // should be zero + ubyte interlaceMethod = 0; // bool +} +*/ + + +PNG* pngFromImage(IndexedImage i) { + PNGHeader h; + h.width = i.width; + h.height = i.height; + h.type = 3; + if(i.numColors() <= 2) + h.depth = 1; + else if(i.numColors() <= 4) + h.depth = 2; + else if(i.numColors() <= 16) + h.depth = 4; + else if(i.numColors() <= 256) + h.depth = 8; + else throw new Exception("can't save this as an indexed png"); + + auto png = blankPNG(h); + + // do palette and alpha + // FIXME: if there is only one transparent color, set it as the special chunk for that + + // FIXME: we'd get a smaller file size if the transparent pixels were arranged first + Chunk palette; + palette.type = ['P', 'L', 'T', 'E']; + palette.size = i.palette.length * 3; + palette.payload.length = palette.size; + + Chunk alpha; + if(i.hasAlpha) { + alpha.type = ['t', 'R', 'N', 'S']; + alpha.size = i.palette.length; + alpha.payload.length = alpha.size; + } + + for(int a = 0; a < i.palette.length; a++) { + palette.payload[a*3+0] = i.palette[a].r; + palette.payload[a*3+1] = i.palette[a].g; + palette.payload[a*3+2] = i.palette[a].b; + if(i.hasAlpha) + alpha.payload[a] = i.palette[a].a; + } + + palette.checksum = crc("PLTE", palette.payload); + png.chunks ~= palette; + if(i.hasAlpha) { + alpha.checksum = crc("tRNS", alpha.payload); + png.chunks ~= alpha; + } + + // do the datastream + if(h.depth == 8) { + addImageDatastreamToPng(i.data, png); + } else { + // gotta convert it + ubyte[] datastream = new ubyte[i.width * i.height * 8 / h.depth]; // FIXME? + int shift = 0; + + switch(h.depth) { + case 1: shift = 7; break; + case 2: shift = 6; break; + case 4: shift = 4; break; + case 8: shift = 0; break; + } + int dsp = 0; + int dpos = 0; + bool justAdvanced; + for(int y = 0; y < i.height; y++) { + for(int x = 0; x < i.width; x++) { + datastream[dsp] |= i.data[dpos++] << shift; + + switch(h.depth) { + case 1: shift-= 1; break; + case 2: shift-= 2; break; + case 4: shift-= 4; break; + case 8: shift-= 8; break; + } + + justAdvanced = shift < 0; + if(shift < 0) { + dsp++; + switch(h.depth) { + case 1: shift = 7; break; + case 2: shift = 6; break; + case 4: shift = 4; break; + case 8: shift = 0; break; + } + } + } + if(!justAdvanced) + dsp++; + switch(h.depth) { + case 1: shift = 7; break; + case 2: shift = 6; break; + case 4: shift = 4; break; + case 8: shift = 0; break; + } + + } + + addImageDatastreamToPng(datastream, png); + } + + return png; +} + +PNG* pngFromImage(TrueColorImage i) { + PNGHeader h; + h.width = i.width; + h.height = i.height; + // FIXME: optimize it if it is greyscale or doesn't use alpha alpha + + auto png = blankPNG(h); + addImageDatastreamToPng(i.data, png); + + return png; +} + +/* +void main(string[] args) { + auto a = readPng(cast(ubyte[]) read(args[1])); + auto f = getDatastream(a); + + foreach(i; f) { + writef("%d ", i); + } + + writefln("\n\n%d", f.length); +} +*/ + +struct Chunk { + uint size; + ubyte[4] type; + ubyte[] payload; + uint checksum; +} + +struct PNG { + uint length; + ubyte[8] header; + Chunk[] chunks; + + Chunk* getChunk(string what) { + foreach(ref c; chunks) { + if(cast(string) c.type == what) + return &c; + } + throw new Exception("no such chunk " ~ what); + } + + Chunk* getChunkNullable(string what) { + foreach(ref c; chunks) { + if(cast(string) c.type == what) + return &c; + } + return null; + } +} + +ubyte[] writePng(PNG* p) { + ubyte[] a; + if(p.length) + a.length = p.length; + else { + a.length = 8; + foreach(c; p.chunks) + a.length += c.size + 12; + } + uint pos; + + a[0..8] = p.header[0..8]; + pos = 8; + foreach(c; p.chunks) { + a[pos++] = (c.size & 0xff000000) >> 24; + a[pos++] = (c.size & 0x00ff0000) >> 16; + a[pos++] = (c.size & 0x0000ff00) >> 8; + a[pos++] = (c.size & 0x000000ff) >> 0; + + a[pos..pos+4] = c.type[0..4]; + pos += 4; + a[pos..pos+c.size] = c.payload[0..c.size]; + pos += c.size; + + a[pos++] = (c.checksum & 0xff000000) >> 24; + a[pos++] = (c.checksum & 0x00ff0000) >> 16; + a[pos++] = (c.checksum & 0x0000ff00) >> 8; + a[pos++] = (c.checksum & 0x000000ff) >> 0; + } + + return a; +} + +PNG* readPng(ubyte[] data) { + auto p = new PNG; + + p.length = data.length; + p.header[0..8] = data[0..8]; + + uint pos = 8; + + while(pos < data.length) { + Chunk n; + n.size |= data[pos++] << 24; + n.size |= data[pos++] << 16; + n.size |= data[pos++] << 8; + n.size |= data[pos++] << 0; + n.type[0..4] = data[pos..pos+4]; + pos += 4; + n.payload.length = n.size; + n.payload[0..n.size] = data[pos..pos+n.size]; + pos += n.size; + + n.checksum |= data[pos++] << 24; + n.checksum |= data[pos++] << 16; + n.checksum |= data[pos++] << 8; + n.checksum |= data[pos++] << 0; + + p.chunks ~= n; + } + + return p; +} + +PNG* blankPNG(PNGHeader h) { + auto p = new PNG; + p.header = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; + + Chunk c; + + c.size = 13; + c.type = ['I', 'H', 'D', 'R']; + + c.payload.length = 13; + int pos = 0; + + c.payload[pos++] = h.width >> 24; + c.payload[pos++] = (h.width >> 16) & 0xff; + c.payload[pos++] = (h.width >> 8) & 0xff; + c.payload[pos++] = h.width & 0xff; + + c.payload[pos++] = h.height >> 24; + c.payload[pos++] = (h.height >> 16) & 0xff; + c.payload[pos++] = (h.height >> 8) & 0xff; + c.payload[pos++] = h.height & 0xff; + + c.payload[pos++] = h.depth; + c.payload[pos++] = h.type; + c.payload[pos++] = h.compressionMethod; + c.payload[pos++] = h.filterMethod; + c.payload[pos++] = h.interlaceMethod; + + + c.checksum = crc("IHDR", c.payload); + + p.chunks ~= c; + + return p; +} + +// should NOT have any idata already. +// FIXME: doesn't handle palettes +void addImageDatastreamToPng(const(ubyte)[] data, PNG* png) { + // we need to go through the lines and add the filter byte + // then compress it into an IDAT chunk + // then add the IEND chunk + + PNGHeader h = getHeader(png); + + auto bytesPerLine = h.width * 4; + if(h.type == 3) + bytesPerLine = h.width * 8 / h.depth; + Chunk dat; + dat.type = ['I', 'D', 'A', 'T']; + int pos = 0; + + const(ubyte)[] output; + while(pos+bytesPerLine <= data.length) { + output ~= 0; + output ~= data[pos..pos+bytesPerLine]; + pos += bytesPerLine; + } + + auto com = cast(ubyte[]) compress(output); + dat.size = com.length; + dat.payload = com; + dat.checksum = crc("IDAT", dat.payload); + + png.chunks ~= dat; + + Chunk c; + + c.size = 0; + c.type = ['I', 'E', 'N', 'D']; + c.checksum = crc("IEND", c.payload); + + png.chunks ~= c; + +} + +struct PNGHeader { + uint width; + uint height; + ubyte depth = 8; + ubyte type = 6; // 0 - greyscale, 2 - truecolor, 3 - indexed color, 4 - grey with alpha, 6 - true with alpha + ubyte compressionMethod = 0; // should be zero + ubyte filterMethod = 0; // should be zero + ubyte interlaceMethod = 0; // bool +} + +// bKGD - palette entry for background or the RGB (16 bits each) for that. or 16 bits of grey + +ubyte[] getDatastream(PNG* p) { + ubyte[] compressed; + + foreach(c; p.chunks) { + if(cast(string) c.type != "IDAT") + continue; + compressed ~= c.payload; + } + + return cast(ubyte[]) uncompress(compressed); +} + +// FIXME: Assuming 8 bits per pixel +ubyte[] getUnfilteredDatastream(PNG* p) { + PNGHeader h = getHeader(p); + assert(h.filterMethod == 0); + + assert(h.type == 3); // FIXME + assert(h.depth == 8); // FIXME + + ubyte[] data = getDatastream(p); + ubyte[] ufdata = new ubyte[data.length - h.height]; + + int bytesPerLine = ufdata.length / h.height; + + int pos = 0, pos2 = 0; + for(int a = 0; a < h.height; a++) { + assert(data[pos2] == 0); + ufdata[pos..pos+bytesPerLine] = data[pos2+1..pos2+bytesPerLine+1]; + pos+= bytesPerLine; + pos2+= bytesPerLine + 1; + } + + return ufdata; +} + +ubyte[] getFlippedUnfilteredDatastream(PNG* p) { + PNGHeader h = getHeader(p); + assert(h.filterMethod == 0); + + assert(h.type == 3); // FIXME + assert(h.depth == 8 || h.depth == 4); // FIXME + + ubyte[] data = getDatastream(p); + ubyte[] ufdata = new ubyte[data.length - h.height]; + + int bytesPerLine = ufdata.length / h.height; + + + int pos = ufdata.length - bytesPerLine, pos2 = 0; + for(int a = 0; a < h.height; a++) { + assert(data[pos2] == 0); + ufdata[pos..pos+bytesPerLine] = data[pos2+1..pos2+bytesPerLine+1]; + pos-= bytesPerLine; + pos2+= bytesPerLine + 1; + } + + return ufdata; +} + +ubyte getHighNybble(ubyte a) { + return cast(ubyte)(a >> 4); // FIXME +} + +ubyte getLowNybble(ubyte a) { + return a & 0x0f; +} + +// Takes the transparency info and returns +ubyte[] getANDMask(PNG* p) { + PNGHeader h = getHeader(p); + assert(h.filterMethod == 0); + + assert(h.type == 3); // FIXME + assert(h.depth == 8 || h.depth == 4); // FIXME + + assert(h.width % 8 == 0); // might actually be %2 + + ubyte[] data = getDatastream(p); + ubyte[] ufdata = new ubyte[h.height*((((h.width+7)/8)+3)&~3)]; // gotta pad to DWORDs... + + Color[] colors = fetchPalette(p); + + int pos = 0, pos2 = (h.width/((h.depth == 8) ? 1 : 2)+1)*(h.height-1); + bool bits = false; + for(int a = 0; a < h.height; a++) { + assert(data[pos2++] == 0); + for(int b = 0; b < h.width; b++) { + if(h.depth == 4) { + ufdata[pos/8] |= ((colors[bits? getLowNybble(data[pos2]) : getHighNybble(data[pos2])].a <= 30) << (7-(pos%8))); + } else + ufdata[pos/8] |= ((colors[data[pos2]].a == 0) << (7-(pos%8))); + pos++; + if(h.depth == 4) { + if(bits) { + pos2++; + } + bits = !bits; + } else + pos2++; + } + + int pad = 0; + for(; pad < ((pos/8) % 4); pad++) { + ufdata[pos/8] = 0; + pos+=8; + } + if(h.depth == 4) + pos2 -= h.width + 2; + else + pos2-= 2*(h.width) +2; + } + + return ufdata; +} + +// Done with assumption + +PNGHeader getHeader(PNG* p) { + PNGHeader h; + ubyte[] data = p.getChunk("IHDR").payload; + + int pos = 0; + + h.width |= data[pos++] << 24; + h.width |= data[pos++] << 16; + h.width |= data[pos++] << 8; + h.width |= data[pos++] << 0; + + h.height |= data[pos++] << 24; + h.height |= data[pos++] << 16; + h.height |= data[pos++] << 8; + h.height |= data[pos++] << 0; + + h.depth = data[pos++]; + h.type = data[pos++]; + h.compressionMethod = data[pos++]; + h.filterMethod = data[pos++]; + h.interlaceMethod = data[pos++]; + + return h; +} + +struct Color { + ubyte r; + ubyte g; + ubyte b; + ubyte a; +} + +/+ +class Image { + Color[][] trueColorData; + ubyte[] indexData; + + Color[] palette; + + uint width; + uint height; + + this(uint w, uint h) {} +} + +Image fromPNG(PNG* p) { + +} + +PNG* toPNG(Image i) { + +} ++/ struct RGBQUAD { + ubyte rgbBlue; + ubyte rgbGreen; + ubyte rgbRed; + ubyte rgbReserved; + } + +RGBQUAD[] fetchPaletteWin32(PNG* p) { + RGBQUAD[] colors; + + auto palette = p.getChunk("PLTE"); + + colors.length = (palette.size) / 3; + + for(int i = 0; i < colors.length; i++) { + colors[i].rgbRed = palette.payload[i*3+0]; + colors[i].rgbGreen = palette.payload[i*3+1]; + colors[i].rgbBlue = palette.payload[i*3+2]; + colors[i].rgbReserved = 0; + } + + return colors; + +} + +Color[] fetchPalette(PNG* p) { + Color[] colors; + + auto palette = p.getChunk("PLTE"); + + Chunk* alpha = p.getChunkNullable("tRNS"); + + colors.length = palette.size / 3; + + for(int i = 0; i < colors.length; i++) { + colors[i].r = palette.payload[i*3+0]; + colors[i].g = palette.payload[i*3+1]; + colors[i].b = palette.payload[i*3+2]; + if(alpha !is null && i < alpha.size) + colors[i].a = alpha.payload[i]; + else + colors[i].a = 255; + + //writefln("%2d: %3d %3d %3d %3d", i, colors[i].r, colors[i].g, colors[i].b, colors[i].a); + } + + return colors; +} + +void replacePalette(PNG* p, Color[] colors) { + auto palette = p.getChunk("PLTE"); + auto alpha = p.getChunk("tRNS"); + + assert(colors.length == alpha.size); + + for(int i = 0; i < colors.length; i++) { + palette.payload[i*3+0] = colors[i].r; + palette.payload[i*3+1] = colors[i].g; + palette.payload[i*3+2] = colors[i].b; + alpha.payload[i] = colors[i].a; + } + + palette.checksum = crc("PLTE", palette.payload); + alpha.checksum = crc("tRNS", alpha.payload); +} + +uint update_crc(in uint crc, in ubyte[] buf){ + static const uint[256] crc_table = [0, 1996959894, 3993919788, 2567524794, 124634137, 1886057615, 3915621685, 2657392035, 249268274, 2044508324, 3772115230, 2547177864, 162941995, 2125561021, 3887607047, 2428444049, 498536548, 1789927666, 4089016648, 2227061214, 450548861, 1843258603, 4107580753, 2211677639, 325883990, 1684777152, 4251122042, 2321926636, 335633487, 1661365465, 4195302755, 2366115317, 997073096, 1281953886, 3579855332, 2724688242, 1006888145, 1258607687, 3524101629, 2768942443, 901097722, 1119000684, 3686517206, 2898065728, 853044451, 1172266101, 3705015759, 2882616665, 651767980, 1373503546, 3369554304, 3218104598, 565507253, 1454621731, 3485111705, 3099436303, 671266974, 1594198024, 3322730930, 2970347812, 795835527, 1483230225, 3244367275, 3060149565, 1994146192, 31158534, 2563907772, 4023717930, 1907459465, 112637215, 2680153253, 3904427059, 2013776290, 251722036, 2517215374, 3775830040, 2137656763, 141376813, 2439277719, 3865271297, 1802195444, 476864866, 2238001368, 4066508878, 1812370925, 453092731, 2181625025, 4111451223, 1706088902, 314042704, 2344532202, 4240017532, 1658658271, 366619977, 2362670323, 4224994405, 1303535960, 984961486, 2747007092, 3569037538, 1256170817, 1037604311, 2765210733, 3554079995, 1131014506, 879679996, 2909243462, 3663771856, 1141124467, 855842277, 2852801631, 3708648649, 1342533948, 654459306, 3188396048, 3373015174, 1466479909, 544179635, 3110523913, 3462522015, 1591671054, 702138776, 2966460450, 3352799412, 1504918807, 783551873, 3082640443, 3233442989, 3988292384, 2596254646, 62317068, 1957810842, 3939845945, 2647816111, 81470997, 1943803523, 3814918930, 2489596804, 225274430, 2053790376, 3826175755, 2466906013, 167816743, 2097651377, 4027552580, 2265490386, 503444072, 1762050814, 4150417245, 2154129355, 426522225, 1852507879, 4275313526, 2312317920, 282753626, 1742555852, 4189708143, 2394877945, 397917763, 1622183637, 3604390888, 2714866558, 953729732, 1340076626, 3518719985, 2797360999, 1068828381, 1219638859, 3624741850, 2936675148, 906185462, 1090812512, 3747672003, 2825379669, 829329135, 1181335161, 3412177804, 3160834842, 628085408, 1382605366, 3423369109, 3138078467, 570562233, 1426400815, 3317316542, 2998733608, 733239954, 1555261956, 3268935591, 3050360625, 752459403, 1541320221, 2607071920, 3965973030, 1969922972, 40735498, 2617837225, 3943577151, 1913087877, 83908371, 2512341634, 3803740692, 2075208622, 213261112, 2463272603, 3855990285, 2094854071, 198958881, 2262029012, 4057260610, 1759359992, 534414190, 2176718541, 4139329115, 1873836001, 414664567, 2282248934, 4279200368, 1711684554, 285281116, 2405801727, 4167216745, 1634467795, 376229701, 2685067896, 3608007406, 1308918612, 956543938, 2808555105, 3495958263, 1231636301, 1047427035, 2932959818, 3654703836, 1088359270, 936918000, 2847714899, 3736837829, 1202900863, 817233897, 3183342108, 3401237130, 1404277552, 615818150, 3134207493, 3453421203, 1423857449, 601450431, 3009837614, 3294710456, 1567103746, 711928724, 3020668471, 3272380065, 1510334235, 755167117]; + + uint c = crc; + + foreach(b; buf) + c = crc_table[(c ^ b) & 0xff] ^ (c >> 8); + + return c; +} + +// lol is just the chunk name +uint crc(in string lol, in ubyte[] buf){ + uint c = update_crc(0xffffffffL, cast(ubyte[]) lol); + return update_crc(c, buf) ^ 0xffffffffL; +} + diff --git a/postgres.d b/postgres.d new file mode 100644 index 0000000..23f9820 --- /dev/null +++ b/postgres.d @@ -0,0 +1,218 @@ +module arsd.postgres; +pragma(lib, "pq"); + +public import arsd.database; + +import std.string; +import std.exception; + +// remember to CREATE DATABASE name WITH ENCODING 'utf8' + +class PostgreSql : Database { + // dbname = name is probably the most common connection string + this(string connectionString) { + conn = PQconnectdb(toStringz(connectionString)); + if(conn is null) + throw new DatabaseException("Unable to allocate PG connection object"); + if(PQstatus(conn) != CONNECTION_OK) + throw new DatabaseException(error()); + query("SET NAMES 'utf8'"); // D does everything with utf8 + } + + ~this() { + PQfinish(conn); + } + + override void startTransaction() { + query("START TRANSACTION"); + } + + ResultSet queryImpl(string sql, Variant[] args...) { + sql = escapedVariants(this, sql, args); + + auto res = PQexec(conn, toStringz(sql)); + int ress = PQresultStatus(res); + if(ress != PGRES_TUPLES_OK + && ress != PGRES_COMMAND_OK) + throw new DatabaseException(error()); + + return new PostgresResult(res); + } + + string escape(string sqlData) { + char* buffer = (new char[sqlData.length * 2 + 1]).ptr; + int size = PQescapeString (buffer, sqlData.ptr, sqlData.length); + + string ret = assumeUnique(buffer[0..size]); + + return ret; + } + + + string error() { + return copyCString(PQerrorMessage(conn)); + } + + private: + PGconn* conn; +} + +class PostgresResult : ResultSet { + // name for associative array to result index + int getFieldIndex(string field) { + if(mapping is null) + makeFieldMapping(); + return mapping[field]; + } + + + string[] fieldNames() { + if(mapping is null) + makeFieldMapping(); + return columnNames; + } + + // this is a range that can offer other ranges to access it + bool empty() { + return position == numRows; + } + + Row front() { + return row; + } + + void popFront() { + position++; + if(position < numRows) + fetchNext; + } + + int length() { + return numRows; + } + + this(PGresult* res) { + this.res = res; + numFields = PQnfields(res); + numRows = PQntuples(res); + + if(numRows) + fetchNext(); + } + + ~this() { + PQclear(res); + } + + private: + PGresult* res; + int[string] mapping; + string[] columnNames; + int numFields; + + int position; + + int numRows; + + Row row; + + void fetchNext() { + Row r; + r.resultSet = this; + string[] row; + + for(int i = 0; i < numFields; i++) { + string a; + + if(PQgetisnull(res, position, i)) + a = null; + else { + a = copyCString(PQgetvalue(res, position, i), PQgetlength(res, position, i)); + + } + row ~= a; + } + + r.row = row; + this.row = r; + } + + void makeFieldMapping() { + for(int i = 0; i < numFields; i++) { + string a = copyCString(PQfname(res, i)); + + columnNames ~= a; + mapping[a] = i; + } + + } +} + +string copyCString(const char* c, int actualLength = -1) { + const(char)* a = c; + if(a is null) + return null; + + string ret; + if(actualLength == -1) + while(*a) { + ret ~= *a; + a++; + } + else { + ret = a[0..actualLength].idup; + } + + return ret; +} + +extern(C) { + struct PGconn; + struct PGresult; + + void PQfinish(PGconn*); + PGconn* PQconnectdb(const char*); + + int PQstatus(PGconn*); // FIXME check return value + + const (char*) PQerrorMessage(PGconn*); + + PGresult* PQexec(PGconn*, const char*); + void PQclear(PGresult*); + + int PQresultStatus(PGresult*); // FIXME check return value + + int PQnfields(PGresult*); // number of fields in a result + const(char*) PQfname(PGresult*, int); // name of field + + int PQntuples(PGresult*); // number of rows in result + const(char*) PQgetvalue(PGresult*, int row, int column); + + size_t PQescapeString (char *to, const char *from, size_t length); + + enum int CONNECTION_OK = 0; + enum int PGRES_COMMAND_OK = 1; + enum int PGRES_TUPLES_OK = 2; + + int PQgetlength(const PGresult *res, + int row_number, + int column_number); + int PQgetisnull(const PGresult *res, + int row_number, + int column_number); + + +} + +/* +import std.stdio; +void main() { + auto db = new PostgreSql("dbname = test"); + + db.query("INSERT INTO users (id, name) values (?, ?)", 30, "hello mang"); + + foreach(line; db.query("SELECT * FROM users")) { + writeln(line[0], line["name"]); + } +} +*/ diff --git a/sha.d b/sha.d new file mode 100644 index 0000000..1aaedbc --- /dev/null +++ b/sha.d @@ -0,0 +1,366 @@ +module arsd.sha; + +/* + By Adam D. Ruppe, 26 Nov 2009 + I release this file into the public domain +*/ +import std.stdio; + +immutable(ubyte)[/*20*/] SHA1(T)(T data) if(isInputRange!(T)) /*const(ubyte)[] data)*/ { + uint[5] h = [0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0]; + + SHARange!(T) range; + static if(is(data == SHARange)) + range = data; + else { + range.r = data; + } + /* + ubyte[] message = data.dup; + message ~= 0b1000_0000; + while(((message.length+8) * 8) % 512) + message ~= 0; + + ulong originalLength = cast(ulong) data.length * 8; + + for(int a = 7; a >= 0; a--) + message ~= (originalLength >> (a*8)) & 0xff; // to big-endian + + assert(((message.length * 8) % 512) == 0); + + uint pos = 0; + while(pos < message.length) { + */ + while(!range.empty) { + uint[80] words; + + for(int a = 0; a < 16; a++) { + for(int b = 3; b >= 0; b--) { + words[a] |= cast(uint)(range.front()) << (b*8); + range.popFront; + // words[a] |= cast(uint)(message[pos]) << (b*8); + // pos++; + } + } + + for(int a = 16; a < 80; a++) { + uint t = words[a-3]; + t ^= words[a-8]; + t ^= words[a-14]; + t ^= words[a-16]; + asm { rol t, 1; } + words[a] = t; + } + + uint a = h[0]; + uint b = h[1]; + uint c = h[2]; + uint d = h[3]; + uint e = h[4]; + + for(int i = 0; i < 80; i++) { + uint f, k; + if(i >= 0 && i < 20) { + f = (b & c) | ((~b) & d); + k = 0x5A827999; + } else + if(i >= 20 && i < 40) { + f = b ^ c ^ d; + k = 0x6ED9EBA1; + } else + if(i >= 40 && i < 60) { + f = (b & c) | (b & d) | (c & d); + k = 0x8F1BBCDC; + } else + if(i >= 60 && i < 80) { + f = b ^ c ^ d; + k = 0xCA62C1D6; + } else assert(0); + + uint temp; + asm { + mov EAX, a; + rol EAX, 5; + add EAX, f; + add EAX, e; + add EAX, k; + mov temp, EAX; + } + temp += words[i]; + e = d; + d = c; + asm { + mov EAX, b; + rol EAX, 30; + mov c, EAX; + } + b = a; + a = temp; + } + + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + } + + + ubyte[] hash; + for(int j = 0; j < 5; j++) + for(int i = 3; i >= 0; i--) { + hash ~= cast(ubyte)(h[j] >> (i*8))&0xff; + } + + return hash.idup; +} + +import std.range; + +// This does the preprocessing of input data, fetching one byte at a time of the data until it is empty, then the padding and length at the end +template SHARange(T) if(isInputRange!(T)) { + struct SHARange { + T r; + + bool empty() { + return state == 5; + } + + void popFront() { + static int lol = 0; + if(state == 0) { + r.popFront; + /* + static if(__traits(compiles, r.front.length)) + length += r.front.length; + else + length += r.front().sizeof; + */ + length++; // FIXME + + if(r.empty) { + state = 1; + position = 2; + current = 0x80; + } + } else { + if(state == 1) { + current = 0x0; + state = 2; + position++; + } else if( state == 2) { + if(!(((position + length + 8) * 8) % 512)) { + state = 3; + position = 7; + length *= 8; + } else + position++; + } else if (state == 3) { + current = (length >> (position*8)) & 0xff; + if(position == 0) + state = 4; + else + position--; + } else if (state == 4) { + current = 0xff; + state = 5; + } + } + } + + ubyte front() { + if(state == 0) { + return cast(ubyte) r.front(); + } + return current; + } + + ubyte current; + uint position; + ulong length; + int state = 0; // reading range, reading appended bit, reading padding, reading length, done + } +} + +immutable(ubyte)[] SHA256(T)(T data) if ( isInputRange!(T)) { + uint[8] h = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19]; + immutable(uint[64]) k = [0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2]; + + SHARange!(T) range; + static if(is(data == SHARange)) + range = data; + else { + range.r = data; + } +/* + ubyte[] message = cast(ubyte[]) data.dup; + message ~= 0b1000_0000; + while(((message.length+8) * 8) % 512) + message ~= 0; + + ulong originalLength = cast(ulong) data.length * 8; + + for(int a = 7; a >= 0; a--) + message ~= (originalLength >> (a*8)) & 0xff; // to big-endian + + assert(((message.length * 8) % 512) == 0); +*/ +// uint pos = 0; + while(!range.empty) { +// while(pos < message.length) { + uint[64] words; + + for(int a = 0; a < 16; a++) { + for(int b = 3; b >= 0; b--) { + words[a] |= cast(uint)(range.front()) << (b*8); + //words[a] |= cast(uint)(message[pos]) << (b*8); + range.popFront; +// pos++; + } + } + + for(int a = 16; a < 64; a++) { + uint t1 = words[a-15]; + asm { + mov EAX, t1; + mov EBX, EAX; + mov ECX, EAX; + ror EAX, 7; + ror EBX, 18; + shr ECX, 3; + xor EAX, EBX; + xor EAX, ECX; + mov t1, EAX; + } + uint t2 = words[a-2]; + asm { + mov EAX, t2; + mov EBX, EAX; + mov ECX, EAX; + ror EAX, 17; + ror EBX, 19; + shr ECX, 10; + xor EAX, EBX; + xor EAX, ECX; + mov t2, EAX; + } + + words[a] = words[a-16] + t1 + words[a-7] + t2; + } + + uint A = h[0]; + uint B = h[1]; + uint C = h[2]; + uint D = h[3]; + uint E = h[4]; + uint F = h[5]; + uint G = h[6]; + uint H = h[7]; + + for(int i = 0; i < 64; i++) { + uint s0; + asm { + mov EAX, A; + mov EBX, EAX; + mov ECX, EAX; + ror EAX, 2; + ror EBX, 13; + ror ECX, 22; + xor EAX, EBX; + xor EAX, ECX; + mov s0, EAX; + } + uint maj = (A & B) ^ (A & C) ^ (B & C); + uint t2 = s0 + maj; + uint s1; + asm { + mov EAX, E; + mov EBX, EAX; + mov ECX, EAX; + ror EAX, 6; + ror EBX, 11; + ror ECX, 25; + xor EAX, EBX; + xor EAX, ECX; + mov s1, EAX; + } + uint ch = (E & F) ^ ((~E) & G); + uint t1 = H + s1 + ch + k[i] + words[i]; + + H = G; + G = F; + F = E; + E = D + t1; + D = C; + C = B; + B = A; + A = t1 + t2; + } + + h[0] += A; + h[1] += B; + h[2] += C; + h[3] += D; + h[4] += E; + h[5] += F; + h[6] += G; + h[7] += H; + } + + ubyte[] hash; + for(int j = 0; j < 8; j++) + for(int i = 3; i >= 0; i--) { + hash ~= cast(ubyte)(h[j] >> (i*8))&0xff; + } + + return hash.idup; +} + +import std.exception; + +string hashToString(const(ubyte)[] hash) { + char[] s; + + s.length = hash.length * 2; + + char toHex(int a) { + if(a < 10) + return cast(char) (a + '0'); + else + return cast(char) (a + 'a' - 10); + } + + for(int a = 0; a < hash.length; a++) { + s[a*2] = toHex(hash[a] >> 4); + s[a*2+1] = toHex(hash[a] & 0x0f); + } + + return assumeUnique(s); +} +/* +string tee(string t) { + writefln("%s", t); + return t; +} +*/ +unittest { + assert(hashToString(SHA1("abc")) == "a9993e364706816aba3e25717850c26c9cd0d89d"); + assert(hashToString(SHA1("sdfj983yr2ih")) == "335f1f5a4af4aa2c8e93b88d69dda2c22baeb94d"); + assert(hashToString(SHA1("$%&^54ylkufg09fd7f09sa7udsiouhcx987yw98etf7yew98yfds987f632uw90ruds09fudsf09dsuhfoidschyds98fydovipsdaidsd9fsa GA UIA duisguifgsuifgusaufisgfuisafguisagasuidgsaufsauifhuisahfuisafaoisahasiosafhffdasasdisayhfdoisayf8saiuhgduifyds8fiydsufisafoisayf8sayfd98wqyr98wqy98sayd98sayd098sayd09sayd98sayd98saicxyhckxnvjbpovc pousa09cusa 09csau csa9 dusa90d usa9d0sau dsa90 as09posufpodsufodspufdspofuds 9tu sapfusaa daosjdoisajdsapoihdsaiodyhsaioyfg d98ytewq89rysa 98yc98sdxych sa89ydsa89dy sa98ydas98c ysx9v8y cxv89ysd f8ysa89f ysa89fd sg8yhds9g8 rfjcxhvslkhdaiosy09wq7r987t98e7ys98aIYOIYOIY)(*YE (*WY *A(YSA* HDUIHDUIAYT&*ATDAUID AUI DUIAT DUIAG saoidusaoid ysqoid yhsaduiayh UIZYzuI YUIYEDSA UIDYUIADYISA YTDGS UITGUID")) == "e38a1220eaf8103d6176df2e0dd0a933e2f52001"); + + assert(hashToString(SHA256("abc")) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + assert(hashToString(SHA256("$%&^54ylkufg09fd7f09sa7udsiouhcx987yw98etf7yew98yfds987f632uw90ruds09fudsf09dsuhfoidschyds98fydovipsdaidsd9fsa GA UIA duisguifgsuifgusaufisgfuisafguisagasuidgsaufsauifhuisahfuisafaoisahasiosafhffdasasdisayhfdoisayf8saiuhgduifyds8fiydsufisafoisayf8sayfd98wqyr98wqy98sayd98sayd098sayd09sayd98sayd98saicxyhckxnvjbpovc pousa09cusa 09csau csa9 dusa90d usa9d0sau dsa90 as09posufpodsufodspufdspofuds 9tu sapfusaa daosjdoisajdsapoihdsaiodyhsaioyfg d98ytewq89rysa 98yc98sdxych sa89ydsa89dy sa98ydas98c ysx9v8y cxv89ysd f8ysa89f ysa89fd sg8yhds9g8 rfjcxhvslkhdaiosy09wq7r987t98e7ys98aIYOIYOIY)(*YE (*WY *A(YSA* HDUIHDUIAYT&*ATDAUID AUI DUIAT DUIAG saoidusaoid ysqoid yhsaduiayh UIZYzuI YUIYEDSA UIDYUIADYISA YTDGS UITGUID")) == "64ff79c67ad5ddf9ba5b2d83e07a6937ef9a5b4eb39c54fe1e913e21aad0e95c"); +} +/* +void main() { + auto hash = SHA256(InputByChar(stdin)); + writefln("%s", hashToString(hash)); +} +*/ diff --git a/simpleaudio.d b/simpleaudio.d new file mode 120000 index 0000000..128220c --- /dev/null +++ b/simpleaudio.d @@ -0,0 +1 @@ +../../dimage/simpleaudio.d \ No newline at end of file diff --git a/simpledisplay.d b/simpledisplay.d new file mode 120000 index 0000000..e93495b --- /dev/null +++ b/simpledisplay.d @@ -0,0 +1 @@ +../../dimage/simpledisplay.d \ No newline at end of file diff --git a/sqlite.d b/sqlite.d new file mode 100644 index 0000000..4cc1ec2 --- /dev/null +++ b/sqlite.d @@ -0,0 +1,740 @@ +/* + Compile with version=sqlite_extended_metadata_available + if your sqlite is compiled with the + SQLITE_ENABLE_COLUMN_METADATA C-preprocessor symbol. + + If you enable that, you get the ability to use the + queryDataObject() function with sqlite. (You can still + use DataObjects, but you'll have to set up the mappings + manually without the extended metadata.) +*/ + +module arsd.sqlite; +pragma(lib, "sqlite3"); +version(linux) +pragma(lib, "dl"); // apparently sqlite3 depends on this +public import arsd.database; + +import std.exception; + +import std.string; + +import std.c.stdlib; +import core.exception; +import core.memory; +import std.file; +import std.conv; +/* + NOTE: + + This only works correctly on INSERTs if the user can grow the + database file! This means he must have permission to write to + both the file and the directory it is in. + +*/ + + +/** + The Database interface provides a consistent and safe way to access sql RDBMSs. + + Why are all the classes scope? To ensure the database connection is closed when you are done with it. + The destructor cleans everything up. + + (maybe including rolling back a transaction if one is going and it errors.... maybe, or that could bne + scope(exit)) +*/ + +Sqlite openDBAndCreateIfNotPresent(string filename, string sql, void delegate(Sqlite db) initalize = null){ + if(exists(filename)) + return new Sqlite(filename); + else { + auto db = new Sqlite(filename); + db.exec(sql); + if(initalize !is null) + initalize(db); + return db; + } +} + +/* +import std.stdio; +void main() { + Database db = new Sqlite("test.sqlite.db"); + + db.query("CREATE TABLE users (id integer, name text)"); + + db.query("INSERT INTO users values (?, ?)", 1, "hello"); + + foreach(line; db.query("SELECT * FROM users")) { + writefln("%s %s", line[0], line["name"]); + } +} +*/ + +class Sqlite : Database { + public: + this(string filename, int flags = SQLITE_OPEN_READWRITE) { + /+ + int error = sqlite3_open_v2(toStringz(filename), &db, flags, null); + if(error == SQLITE_CANTOPEN) + throw new DatabaseException("omg cant open"); + if(error != SQLITE_OK) + throw new DatabaseException("db open " ~ error()); + +/ + int error = sqlite3_open(toStringz(filename), &db); + if(error != SQLITE_OK) + throw new DatabaseException(this.error()); + } + + ~this(){ + if(sqlite3_close(db) != SQLITE_OK) + throw new DatabaseException(error()); + } + + // my extension for easier editing + version(sqlite_extended_metadata_available) { + ResultByDataObject queryDataObject(T...)(string sql, T t) { + // modify sql for the best data object grabbing + sql = fixupSqlForDataObjectUse(sql); + + auto s = Statement(this, sql); + foreach(i, arg; t) { + s.bind(i + 1, arg); + } + + auto magic = s.execute(true); // fetch extended metadata + + return ResultByDataObject(cast(SqliteResult) magic, magic.extendedMetadata, this); + } + } + + override void startTransaction() { + query("BEGIN TRANSACTION"); + } + + override ResultSet queryImpl(string sql, Variant[] args...) { + auto s = Statement(this, sql); + foreach(i, arg; args) { + s.bind(i + 1, arg); + } + + return s.execute(); + } + + override string escape(string sql) { + if(sql is null) + return null; + char* got = sqlite3_mprintf("%q", toStringz(sql)); // FIXME: might have to be %Q, need to check this, but I think the other impls do the same as %q + auto orig = got; + string esc; + while(*got) { + esc ~= (*got); + got++; + } + + sqlite3_free(orig); + + return esc; + } + + string error(){ + char* mesg = sqlite3_errmsg(db); + char[] m; + int a = std.c.string.strlen(mesg); + m.length = a; + for(int v = 0; v < a; v++) + m[v] = mesg[v]; + + return assumeUnique(m); + } + + int affectedRows(){ + return sqlite3_changes(db); + } + + int lastInsertId(){ + return cast(int) sqlite3_last_insert_rowid(db); + } + + + int exec(string sql, void delegate (char[][char[]]) onEach = null) { + char* mesg; + if(sqlite3_exec(db, toStringz(sql), &callback, &onEach, &mesg) != SQLITE_OK) { + char[] m; + int a = std.c.string.strlen(mesg); + m.length = a; + for(int v = 0; v < a; v++) + m[v] = mesg[v]; + + sqlite3_free(mesg); + throw new DatabaseException("exec " ~ m.idup); + } + + return 0; + } +/* + Statement prepare(string sql){ + sqlite3_stmt * s; + if(sqlite3_prepare_v2(db, toStringz(sql), cast(int) sql.length, &s, null) != SQLITE_OK) + throw new DatabaseException("prepare " ~ error()); + + Statement a = new Statement(s); + + return a; + } +*/ + private: + sqlite3* db; +} + + + + + + +class SqliteResult : ResultSet { + int getFieldIndex(string field) { + foreach(i, n; columnNames) + if(n == field) + return i; + throw new Exception("no such field " ~ field); + } + + string[] fieldNames() { + return columnNames; + } + + // this is a range that can offer other ranges to access it + bool empty() { + return position == rows.length; + } + + Row front() { + Row r; + + r.resultSet = this; + if(rows.length <= position) + throw new Exception("Result is empty"); + foreach(c; rows[position]) { + r.row ~= c.coerce!(string); + } + + return r; + } + + void popFront() { + position++; + } + + int length() { + return rows.length; + } + + this(Variant[][] rows, char[][] columnNames) { + this.rows = rows; + foreach(c; columnNames) + this.columnNames ~= c.idup; + } + + private: + string[] columnNames; + Variant[][] rows; + int position = 0; +} + + + + + + +struct Statement { + private this(Sqlite db, sqlite3_stmt * S) { + this.db = db; + s = S; + finalized = false; + } + + Sqlite db; + + this(Sqlite db, string sql) { + this.db = db; + if(sqlite3_prepare_v2(db.db, toStringz(sql), cast(int) sql.length, &s, null) != SQLITE_OK) + throw new DatabaseException(db.error()); + } + + version(sqlite_extended_metadata_available) + Tuple!(string, string)[string] extendedMetadata; + + ResultSet execute(bool fetchExtendedMetadata = false) { + bool first = true; + int count; + int numRows = 0; + int r = 0; + // FIXME: doesn't handle busy database + while( SQLITE_ROW == sqlite3_step(s) ){ + numRows++; + if(numRows >= rows.length) + rows.length = rows.length + 8; + + if(first){ + count = sqlite3_column_count(s); + + columnNames.length = count; + for(int a = 0; a < count; a++){ + char* str = sqlite3_column_name(s, a); + int l = std.c.string.strlen(str); + columnNames[a].length = l; + for(int b = 0; b < l; b++) + columnNames[a][b] = str[b]; + + version(sqlite_extended_metadata_available) { + if(fetchExtendedMetadata) { + string origtbl; + string origcol; + + const(char)* rofl; + + rofl = sqlite3_column_table_name(s, a); + if(rofl is null) + throw new Exception("null table name pointer"); + while(*rofl) { + origtbl ~= *rofl; + rofl++; + } + rofl = sqlite3_column_origin_name(s, a); + if(rofl is null) + throw new Exception("null colum name pointer"); + while(*rofl) { + origcol ~= *rofl; + rofl++; + } + extendedMetadata[columnNames[a].idup] = tuple(origtbl, origcol); + } + } + } + + first = false; + } + + + rows[r].length = count; + + for(int a = 0; a < count; a++){ + Variant v; + switch(sqlite3_column_type(s, a)){ + case SQLITE_INTEGER: + v = sqlite3_column_int(s, a); + break; + case SQLITE_FLOAT: + v = sqlite3_column_double(s, a); + break; + case SQLITE3_TEXT: + char* str = sqlite3_column_text(s, a); + char[] st; + + int l = std.c.string.strlen(str); + st.length = l; + for(int aa = 0; aa < l; aa++) + st[aa] = str[aa]; + + v = assumeUnique(st); + break; + case SQLITE_BLOB: + byte* str = cast(byte*) sqlite3_column_blob(s, a); + byte[] st; + + int l = sqlite3_column_bytes(s, a); + st.length = l; + for(int aa = 0; aa < l; aa++) + st[aa] = str[aa]; + + v = assumeUnique(st); + + break; + case SQLITE_NULL: + v = null; + break; + } + + rows[r][a] = v; + } + + r++; + } + + rows.length = numRows; + length = numRows; + position = 0; + executed = true; + reset(); + + return new SqliteResult(rows.dup, columnNames); + } + +/* +template extract(A, T, R...){ + void extract(A args, out T t, out R r){ + if(r.length + 1 != args.length) + throw new DatabaseException("wrong places"); + args[0].to(t); + static if(r.length) + extract(args[1..$], r); + } +} +*/ +/* + bool next(T, R...)(out T t, out R r){ + if(position == length) + return false; + + extract(rows[position], t, r); + + position++; + return true; + } +*/ + bool step(out Variant[] row){ + assert(executed); + if(position == length) + return false; + + row = rows[position]; + position++; + + return true; + } + + bool step(out Variant[char[]] row){ + assert(executed); + if(position == length) + return false; + + for(int a = 0; a < length; a++) + row[columnNames[a].idup] = rows[position][a]; + + position++; + + return true; + } + + void reset(){ + if(sqlite3_reset(s) != SQLITE_OK) + throw new DatabaseException("reset " ~ db.error()); + } + + void resetBindings(){ + sqlite3_clear_bindings(s); + } + + void resetAll(){ + reset; + resetBindings; + executed = false; + } + + int bindNameLookUp(const char[] name){ + int a = sqlite3_bind_parameter_index(s, toStringz(name)); + if(a == 0) + throw new DatabaseException("bind name lookup failed " ~ db.error()); + return a; + } + + bool next(T, R...)(out T t, out R r){ + assert(executed); + if(position == length) + return false; + + extract(rows[position], t, r); + + position++; + return true; + } + + template bindAll(T, R...){ + void bindAll(T what, R more){ + bindAllHelper(1, what, more); + } + } + + template exec(T, R...){ + void exec(T what, R more){ + bindAllHelper(1, what, more); + execute(); + } + } + + void bindAllHelper(A, T, R...)(A where, T what, R more){ + bind(where, what); + static if(more.length) + bindAllHelper(where + 1, more); + } + + //void bind(T)(string name, T value) { + //bind(bindNameLookUp(name), value); + //} + + // This should be a template, but grrrr. + void bind (const char[] name, const char[] value){ bind(bindNameLookUp(name), value); } + void bind (const char[] name, int value){ bind(bindNameLookUp(name), value); } + void bind (const char[] name, float value){ bind(bindNameLookUp(name), value); } + void bind (const char[] name, const byte[] value){ bind(bindNameLookUp(name), value); } + + void bind(int col, const char[] value){ + if(value is null) { + if(sqlite3_bind_null(s, col) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } else { + if(sqlite3_bind_text(s, col, value.ptr, value.length, cast(void*)-1) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } + } + + void bind(int col, float value){ + if(sqlite3_bind_double(s, col, value) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } + + void bind(int col, int value){ + if(sqlite3_bind_int(s, col, value) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } + + void bind(int col, const byte[] value){ + if(value is null) { + if(sqlite3_bind_null(s, col) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } else { + if(sqlite3_bind_blob(s, col, cast(void*)value.ptr, value.length, cast(void*)-1) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } + } + + void bind(int col, Variant v) { + if(v.peek!int) + bind(col, v.get!int); + if(v.peek!string) + bind(col, v.get!string); + if(v.peek!float) + bind(col, v.get!float); + if(v.peek!(byte[])) + bind(col, v.get!(byte[])); + if(v.peek!(void*) && v.get!(void*) is null) + bind(col, cast(string) null); + } + + ~this(){ + if(!finalized) + finalize(); + } + + void finalize(){ + if(finalized) + return; + if(sqlite3_finalize(s) != SQLITE_OK) + throw new DatabaseException("finalize " ~ db.error()); + finalized = true; + } + private: + Variant[][] rows; + char[][] columnNames; + int length; + int position; + bool finalized; + + sqlite3_stmt * s; + + bool executed; + + + + + + + + new(size_t sz) + { + void* p; + + p = std.c.stdlib.malloc(sz); + if (!p) + throw new OutOfMemoryError(__FILE__, __LINE__); + GC.addRange(p, sz); + return p; + } + + delete(void* p) + { + if (p) + { GC.removeRange(p); + std.c.stdlib.free(p); + } + } + + + + +} + + + +version(sqlite_extended_metadata_available) { +import std.typecons; +struct ResultByDataObject { + this(SqliteResult r, Tuple!(string, string)[string] mappings, Sqlite db) { + result = r; + this.db = db; + this.mappings = mappings; + } + + Tuple!(string, string)[string] mappings; + + ulong length() { return result.length; } + bool empty() { return result.empty; } + void popFront() { result.popFront(); } + DataObject front() { + return new DataObject(db, result.front.toAA, mappings); + } + // would it be good to add a new() method? would be valid even if empty + // it'd just fill in the ID's at random and allow you to do the rest + + @disable this(this) { } + + SqliteResult result; + Sqlite db; +} +} + + + + + + + + + + +extern(C) int callback(void* cb, int howmany, char** text, char** columns){ + if(cb is null) + return 0; + + void delegate(char[][char[]]) onEach = *cast(void delegate(char[][char[]])*)cb; + + + char[][char[]] row; + + for(int a = 0; a < howmany; a++){ + int b = std.c.string.strlen(columns[a]); + char[] buf; + buf.length = b; + for(int c = 0; c < b; c++) + buf[c] = columns[a][c]; + + int d = std.c.string.strlen(text[a]); + char[] t; + t.length = d; + for(int c = 0; c < d; c++) + t[c] = text[a][c]; + + row[buf.idup] = t; + } + + onEach(row); + + return 0; +} + + + + + + + + + + +extern(C){ + typedef void sqlite3; + typedef void sqlite3_stmt; + int sqlite3_changes(sqlite3*); + int sqlite3_close(sqlite3 *); + int sqlite3_exec( + sqlite3*, /* An open database */ + const(char) *sql, /* SQL to be evaluted */ + int function(void*,int,char**,char**), /* Callback function */ + void *, /* 1st argument to callback */ + char **errmsg /* Error msg written here */ + ); + + int sqlite3_open( + const(char) *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb /* OUT: SQLite db handle */ + ); + +/+ +int sqlite3_open_v2( + char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + char *zVfs /* Name of VFS module to use */ +); ++/ + int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle */ + const(char) *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +int sqlite3_finalize(sqlite3_stmt *pStmt); +int sqlite3_step(sqlite3_stmt*); +long sqlite3_last_insert_rowid(sqlite3*); + +const int SQLITE_OK = 0; +const int SQLITE_ROW = 100; +const int SQLITE_DONE = 101; + +const int SQLITE_INTEGER = 1; // int +const int SQLITE_FLOAT = 2; // float +const int SQLITE3_TEXT = 3; // char[] +const int SQLITE_BLOB = 4; // byte[] +const int SQLITE_NULL = 5; // void* = null + +char *sqlite3_mprintf(const char*,...); + + +int sqlite3_reset(sqlite3_stmt *pStmt); +int sqlite3_clear_bindings(sqlite3_stmt*); +int sqlite3_bind_parameter_index(sqlite3_stmt*, const(char) *zName); + +int sqlite3_bind_blob(sqlite3_stmt*, int, void*, int n, void*); +//int sqlite3_bind_blob(sqlite3_stmt*, int, void*, int n, void(*)(void*)); +int sqlite3_bind_double(sqlite3_stmt*, int, double); +int sqlite3_bind_int(sqlite3_stmt*, int, int); +int sqlite3_bind_null(sqlite3_stmt*, int); +int sqlite3_bind_text(sqlite3_stmt*, int, const(char)*, int n, void*); +//int sqlite3_bind_text(sqlite3_stmt*, int, char*, int n, void(*)(void*)); + +void *sqlite3_column_blob(sqlite3_stmt*, int iCol); +int sqlite3_column_bytes(sqlite3_stmt*, int iCol); +double sqlite3_column_double(sqlite3_stmt*, int iCol); +int sqlite3_column_int(sqlite3_stmt*, int iCol); +char *sqlite3_column_text(sqlite3_stmt*, int iCol); +int sqlite3_column_type(sqlite3_stmt*, int iCol); +char *sqlite3_column_name(sqlite3_stmt*, int N); + +int sqlite3_column_count(sqlite3_stmt *pStmt); +void sqlite3_free(void*); + char *sqlite3_errmsg(sqlite3*); + + const int SQLITE_OPEN_READONLY = 0x1; + const int SQLITE_OPEN_READWRITE = 0x2; + const int SQLITE_OPEN_CREATE = 0x4; + const int SQLITE_CANTOPEN = 14; + + +// will need these to enable support for DataObjects here +const (char *)sqlite3_column_database_name(sqlite3_stmt*,int); +const (char *)sqlite3_column_table_name(sqlite3_stmt*,int); +const (char *)sqlite3_column_origin_name(sqlite3_stmt*,int); +} + diff --git a/web.d b/web.d new file mode 100644 index 0000000..b942690 --- /dev/null +++ b/web.d @@ -0,0 +1,2348 @@ +module arsd.web; + +/* + Running from the command line: + + ./myapp function positional args.... + ./myapp --format=json function + + _GET + _POST + _PUT + _DELETE + + ./myapp --make-nested-call + + + + + + + + Procedural vs Object Oriented + + right now it is procedural: + root/function + root/module/function + + what about an object approach: + root/object + root/class/object + + static ApiProvider.getObject + + + Formatting data: + + CoolApi.myFunc().getFormat('Element', [...same as get...]); + + You should also be able to ask for json, but with a particular format available as toString + + format("json", "html") -- gets json, but each object has it's own toString. Actually, the object adds + a member called formattedSecondarily that is the other thing. + Note: the array itself cannot be changed in format, only it's members. + Note: the literal string of the formatted object is often returned. This may more than double the bandwidth of the call + + Note: BUG: it only works with built in formats right now when doing secondary + + + // formats are: text, html, json, table, and xml + // except json, they are all represented as strings in json values + + string toString -> formatting as text + Element makeHtmlElement -> making it html (same as fragment) + JSONValue makeJsonValue -> formatting to json + Table makeHtmlTable -> making a table + (not implemented) toXml -> making it into an xml document + + + Arrays can be handled too: + + static (converts to) string makeHtmlArray(typeof(this)[] arr); + + + Envelope format: + + document (default), json, none +*/ + +public import arsd.dom; +public import arsd.cgi; // you have to import this in the actual usage file or else it won't link; surely a compiler bug +import arsd.sha; + +public import std.string; +public import std.array; +public import std.stdio : writefln; +public import std.conv; +import std.random; + +public import std.range; + +public import std.traits; +import std.json; + +struct Envelope { + bool success; + string type; + string errorMessage; + string userData; + JSONValue result; // use result.str if the format was anything other than json + debug string dFullString; +} + +string linkTo(alias func, T...)(T args) { + auto reflection = __traits(parent, func).reflection; + assert(reflection !is null); + + auto name = func.stringof; + int idx = name.indexOf("("); + if(idx != -1) + name = name[0 .. idx]; + + auto funinfo = reflection.functions[name]; + + return funinfo.originalName; +} + +/// Everything should derive from this instead of the old struct namespace used before +/// Your class must provide a default constructor. +class ApiProvider { + Cgi cgi; + static immutable(ReflectionInfo)* reflection; + string _baseUrl; // filled based on where this is called from on this request + + /// Override this if you have initialization work that must be done *after* cgi and reflection is ready. + /// It should be used instead of the constructor for most work. + void _initialize() {} + + /// This one is called at least once per call. (_initialize is only called once per process) + void _initializePerCall() {} + + /// Override this if you want to do something special to the document + void _postProcess(Document document) {} + + /// This tentatively redirects the user - depends on the envelope fomat + void redirect(string location) { + if(cgi.request("envelopeFormat", "document") == "document") + cgi.setResponseLocation(location, false); + } + + Element _sitemap() { + auto container = _getGenericContainer(); + + auto list = container.addChild("ul"); + + string[string] handled; + foreach(func; reflection.functions) { + if(func.originalName in handled) + continue; + handled[func.originalName] = func.originalName; + list.addChild("li", new Link(_baseUrl ~ "/" ~ func.name, beautify(func.originalName))); + } + + return list.parentNode.removeChild(list); + } + + Document _defaultPage() { + throw new Exception("no default"); + return null; + } + + Element _getGenericContainer() + out(ret) { + assert(ret !is null); + } + body { + auto document = new Document(""); + auto container = document.getElementById("body"); + return container; + } + + /// When in website mode, you can use this to beautify the error message + Document delegate(Throwable) _errorFunction; +} + +class ApiObject { + /* abstract this(ApiProvider parent, string identifier) */ +} + + + +struct ReflectionInfo { + FunctionInfo[string] functions; + EnumInfo[string] enums; + StructInfo[string] structs; + const(ReflectionInfo)*[string] objects; + + bool needsInstantiation; + + // the overall namespace + string name; // this is also used as the object name in the JS api + + string defaultOutputFormat = "html"; + int versionOfOutputFormat = 2; // change this in your constructor if you still need the (deprecated) old behavior + // bool apiMode = false; // no longer used - if format is json, apiMode behavior is assumed. if format is html, it is not. + // FIXME: what if you want the data formatted server side, but still in a json envelope? + // should add format-payload: +} + +struct EnumInfo { + string name; + int[] values; + string[] names; +} + +struct StructInfo { + string name; + // a struct is sort of like a function constructor... + StructMemberInfo[] members; +} + +struct StructMemberInfo { + string name; + string staticType; + string defaultValue; +} + +struct FunctionInfo { + WrapperFunction dispatcher; + + JSONValue delegate(Cgi cgi, in string[][string] sargs) documentDispatcher; + // should I also offer dispatchers for other formats like Variant[]? + string name; + string originalName; + + //string uriPath; + + Parameter[] parameters; + + string returnType; + bool returnTypeIsDocument; + + Document function(in string[string] args) createForm; +} + +struct Parameter { + string name; + string value; + + string type; + string staticType; + string validator; + + // for radio and select boxes + string[] options; + string[] optionValues; +} + +string makeJavascriptApi(const ReflectionInfo* mod, string base) { + assert(mod !is null); + + string script = `var `~mod.name~` = { + "_apiBase":'`~base~`',`; + + script ~= javascriptBase; + + script ~= "\n\t"; + + bool[string] alreadyDone; + + bool outp = false; + + foreach(s; mod.enums) { + if(outp) + script ~= ",\n\t"; + else + outp = true; + + script ~= "'"~s.name~"': {\n"; + + bool outp2 = false; + foreach(i, n; s.names) { + if(outp2) + script ~= ",\n"; + else + outp2 = true; + + // auto v = s.values[i]; + auto v = "'" ~ n ~ "'"; // we actually want to use the name here because to!enum() uses member name. + + script ~= "\t\t'"~n~"':" ~ to!string(v); + } + + script ~= "\n\t}"; + } + + foreach(s; mod.structs) { + if(outp) + script ~= ",\n\t"; + else + outp = true; + + script ~= "'"~s.name~"': function("; + + bool outp2 = false; + foreach(n; s.members) { + if(outp2) + script ~= ", "; + else + outp2 = true; + + script ~= n.name; + + } + script ~= ") { return {\n"; + + outp2 = false; + + script ~= "\t\t'_arsdTypeOf':'"~s.name~"'"; + if(s.members.length) + script ~= ","; + script ~= " // metadata, ought to be read only\n"; + + // outp2 is still false because I put the comma above + foreach(n; s.members) { + if(outp2) + script ~= ",\n"; + else + outp2 = true; + + auto v = n.defaultValue; + + script ~= "\t\t'"~n.name~"': (typeof "~n.name~" == 'undefined') ? "~n.name~" : '" ~ to!string(v) ~ "'"; + } + + script ~= "\n\t}; }"; + } + + // FIXME: it should output the classes too +/* + foreach(obj; mod.objects) { + if(outp) + script ~= ",\n\t"; + else + outp = true; + + script ~= makeJavascriptApi(obj, base); + } +*/ + + foreach(func; mod.functions) { + if(func.originalName in alreadyDone) + continue; // there's url friendly and code friendly, only need one + + alreadyDone[func.originalName] = true; + + if(outp) + script ~= ",\n\t"; + else + outp = true; + + + string args; + string obj; + bool outputted = false; + foreach(i, arg; func.parameters) { + if(outputted) { + args ~= ","; + obj ~= ","; + } else + outputted = true; + + args ~= arg.name; + + // FIXME: we could probably do better checks here too like on type + obj ~= `'`~arg.name~`':(typeof `~arg.name ~ ` == "undefined" ? this._raiseError('InsufficientParametersException', '`~func.originalName~`: argument `~to!string(i) ~ " (" ~ arg.staticType~` `~arg.name~`) is not present') : `~arg.name~`)`; + } + + /* + if(outputted) + args ~= ","; + args ~= "callback"; + */ + + script ~= `'` ~ func.originalName ~ `'`; + script ~= ":"; + script ~= `function(`~args~`) {`; + if(obj.length) + script ~= ` + var argumentsObject = { + `~obj~` + }; + return this._serverCall('`~func.name~`', argumentsObject, '`~func.returnType~`');`; + else + script ~= ` + return this._serverCall('`~func.name~`', null, '`~func.returnType~`');`; + + script ~= ` + }`; + } + + script ~= "\n}"; + + // some global stuff to put in + script ~= ` + if(typeof arsdGlobalStuffLoadedForWebDotD == "undefined") { + arsdGlobalStuffLoadedForWebDotD = true; + var oldObjectDotPrototypeDotToString = Object.prototype.toString; + Object.prototype.toString = function() { + if(this.formattedSecondarily) + return this.formattedSecondarily; + + return oldObjectDotPrototypeDotToString.call(this); + } + } + `; + + return script; +} + +template isEnum(alias T) if(is(T)) { + static if (is(T == enum)) + enum bool isEnum = true; + else + enum bool isEnum = false; +} + +// WTF, shouldn't is(T == xxx) already do this? +template isEnum(T) if(!is(T)) { + enum bool isEnum = false; +} + +template isStruct(alias T) if(is(T)) { + static if (is(T == struct)) + enum bool isStruct = true; + else + enum bool isStruct = false; +} + +// WTF +template isStruct(T) if(!is(T)) { + enum bool isStruct = false; +} + + +template isApiObject(alias T) if(is(T)) { + static if (is(T : ApiObject)) + enum bool isApiObject = true; + else + enum bool isApiObject = false; +} + +// WTF +template isApiObject(T) if(!is(T)) { + enum bool isApiObject = false; +} + +template isApiProvider(alias T) if(is(T)) { + static if (is(T : ApiProvider)) + enum bool isApiProvider = true; + else + enum bool isApiProvider = false; +} + +// WTF +template isApiProvider(T) if(!is(T)) { + enum bool isApiProvider = false; +} + + +template Passthrough(T) { + T Passthrough; +} + +template PassthroughType(T) { + alias T PassthroughType; +} + +auto generateGetter(PM, Parent, string member, alias hackToEnsureMultipleFunctionsWithTheSameSignatureGetTheirOwnInstantiations)(string io, Parent instantiation) { + static if(is(PM : ApiObject)) { + auto i = new PM(instantiation, io); + return &__traits(getMember, i, member); + } else { + return &__traits(getMember, instantiation, member); + } +} + + + +immutable(ReflectionInfo*) prepareReflection(alias PM)(Cgi cgi, PM instantiation, ApiObject delegate(string) instantiateObject = null) if(is(PM : ApiProvider) || is(PM: ApiObject) ) { + return prepareReflectionImpl!(PM, PM)(cgi, instantiation, instantiateObject); +} + +immutable(ReflectionInfo*) prepareReflectionImpl(alias PM, alias Parent)(Cgi cgi, Parent instantiation, ApiObject delegate(string) instantiateObject = null) if((is(PM : ApiProvider) || is(PM: ApiObject)) && is(Parent : ApiProvider) ) { + + assert(instantiation !is null); + + ReflectionInfo* reflection = new ReflectionInfo; + reflection.name = PM.stringof; + + static if(is(PM: ApiObject)) + reflection.needsInstantiation = true; + + // derivedMembers is changed from allMembers + foreach(member; __traits(derivedMembers, PM)) { + // FIXME: the filthiest of all hacks... + static if(!__traits(compiles, + !is(typeof(__traits(getMember, PM, member)) == function) && + isEnum!(__traits(getMember, PM, member)))) + continue; // must be a data member or something... + else + // DONE WITH FILTHIEST OF ALL HACKS + + //if(member.length == 0) + // continue; + static if( + !is(typeof(__traits(getMember, PM, member)) == function) && + isEnum!(__traits(getMember, PM, member)) + && member[0] != '_' + ) { + EnumInfo i; + i.name = member; + foreach(m; __traits(allMembers, __traits(getMember, PM, member))) { + i.names ~= m; + i.values ~= cast(int) __traits(getMember, __traits(getMember, PM, member), m); + } + + reflection.enums[member] = i; + + } else static if( + !is(typeof(__traits(getMember, PM, member)) == function) && + isStruct!(__traits(getMember, PM, member)) + && member[0] != '_' + ) { + StructInfo i; + i.name = member; + + typeof(Passthrough!(__traits(getMember, PM, member))) s; + foreach(idx, m; s.tupleof) { + StructMemberInfo mem; + + mem.name = s.tupleof[idx].stringof[2..$]; + mem.staticType = typeof(m).stringof; + + mem.defaultValue = null; // FIXME + + i.members ~= mem; + } + + reflection.structs[member] = i; + } else static if( + is(typeof(__traits(getMember, PM, member)) == function) + && ( + member[0] != '_' && + ( + member.length < 5 || + ( + member[$ - 5 .. $] != "_Page" && + member[$ - 5 .. $] != "_Form") && + !(member.length > 16 && member[$ - 16 .. $] == "_PermissionCheck") + ))) { + FunctionInfo f; + ParameterTypeTuple!(__traits(getMember, PM, member)) fargs; + + f.returnType = ReturnType!(__traits(getMember, PM, member)).stringof; + f.returnTypeIsDocument = is(ReturnType!(__traits(getMember, PM, member)) : Document); + + f.name = toUrlName(member); + f.originalName = member; + + assert(instantiation !is null); + f.dispatcher = generateWrapper!( + generateGetter!(PM, Parent, member, __traits(getMember, PM, member)), + __traits(getMember, PM, member), Parent, member + )(reflection, instantiation); + + //f.uriPath = f.originalName; + + auto names = parameterNamesOf!(__traits(getMember, PM, member)); + + foreach(idx, param; fargs) { + Parameter p; + + p.name = names[idx]; + p.staticType = typeof(fargs[idx]).stringof; + static if( is( typeof(param) == enum )) { + p.type = "select"; + + foreach(opt; __traits(allMembers, typeof(param))) { + p.options ~= opt; + p.optionValues ~= to!string(__traits(getMember, param, opt)); + } + } else static if (is(typeof(param) == bool)) { + p.type = "checkbox"; + } else { + if(p.name.toLower.indexOf("password") != -1) // hack to support common naming convention + p.type = "password"; + else + p.type = "text"; + } + + f.parameters ~= p; + } + + static if(__traits(hasMember, PM, member ~ "_Form")) { + f.createForm = &__traits(getMember, PM, member ~ "_Form"); + } + + reflection.functions[f.name] = f; + // also offer the original name if it doesn't + // conflict + //if(f.originalName !in reflection.functions) + reflection.functions[f.originalName] = f; + } + else static if( + !is(typeof(__traits(getMember, PM, member)) == function) && + isApiObject!(__traits(getMember, PM, member)) && + member[0] != '_' + ) { + reflection.objects[member] = prepareReflectionImpl!( + __traits(getMember, PM, member), Parent) + (cgi, instantiation); + } else static if( // child ApiProviders are like child modules + !is(typeof(__traits(getMember, PM, member)) == function) && + isApiProvider!(__traits(getMember, PM, member)) && + member[0] != '_' + ) { + PassthroughType!(__traits(getMember, PM, member)) i; + i = new typeof(i)(); + auto r = prepareReflection!(__traits(getMember, PM, member))(cgi, i); + reflection.objects[member] = r; + if(toLower(member) !in reflection.objects) // web filenames are often lowercase too + reflection.objects[member.toLower] = r; + } + } + + static if(is(PM: ApiProvider)) { + instantiation.cgi = cgi; + instantiation.reflection = cast(immutable) reflection; + instantiation._initialize(); + } + + return cast(immutable) reflection; +} + +void run(Provider)(Cgi cgi, Provider instantiation, int pathInfoStartingPoint = 0) if(is(Provider : ApiProvider)) { + assert(instantiation !is null); + + immutable(ReflectionInfo)* reflection; + if(instantiation.reflection is null) + prepareReflection!(Provider)(cgi, instantiation); + + reflection = instantiation.reflection; + + instantiation._baseUrl = cgi.scriptName ~ cgi.pathInfo[0 .. pathInfoStartingPoint]; + if(cgi.pathInfo[pathInfoStartingPoint .. $].length <= 1) { + auto document = instantiation._defaultPage(); + if(document !is null) { + instantiation._postProcess(document); + cgi.write(document.toString()); + } + cgi.close(); + return; + } + + string funName = cgi.pathInfo[pathInfoStartingPoint + 1..$]; + + // kinda a hack, but this kind of thing should be available anyway + if(funName == "functions.js") { + cgi.setResponseContentType("text/javascript"); + cgi.write(makeJavascriptApi(reflection, replace(cast(string) cgi.requestUri, "functions.js", ""))); + cgi.close(); + return; + } + + instantiation._initializePerCall(); + + // what about some built in functions? + /* + // Basic integer operations + builtin.opAdd + builtin.opSub + builtin.opMul + builtin.opDiv + + // Basic array operations + builtin.opConcat // use to combine calls easily + builtin.opIndex + builtin.opSlice + builtin.length + + // Basic floating point operations + builtin.round + builtin.floor + builtin.ceil + + // Basic object operations + builtin.getMember + + // Basic functional operations + builtin.filter // use to slice down on stuff to transfer + builtin.map // call a server function on a whole array + builtin.reduce + + // Access to the html items + builtin.getAutomaticForm(method) + */ + + const(FunctionInfo)* fun; + + auto envelopeFormat = cgi.request("envelopeFormat", "document"); + Envelope result; + result.userData = cgi.request("passedThroughUserData"); + + string instantiator; + string objectName; + + try { + // Built-ins + string errorMessage; + if(funName.length > 8 && funName[0..8] == "builtin.") { + funName = funName[8..$]; + switch(funName) { + default: assert(0); + case "getAutomaticForm": + auto mfun = new FunctionInfo; + mfun.returnType = "Form"; + mfun.dispatcher = delegate JSONValue (Cgi cgi, string, in string[][string] sargs, in string format, in string secondaryFormat = null) { + auto rfun = cgi.request("method") in reflection.functions; + if(rfun is null) + throw new NoSuchPageException("no such function " ~ cgi.request("method")); + + auto form = createAutomaticForm(new Document, *rfun); + auto idx = cgi.requestUri.indexOf("builtin.getAutomaticForm"); + form.action = cgi.requestUri[0 .. idx] ~ form.action; // make sure it works across the site + JSONValue v; + v.type = JSON_TYPE.STRING; + v.str = form.toString(); + + return v; + }; + + fun = cast(immutable) mfun; + break; + } + } else { + // User-defined + // FIXME: modules? should be done with dots since slashes is used for api objects + fun = funName in reflection.functions; + if(fun is null) { + auto parts = funName.split("/"); + + const(ReflectionInfo)* currentReflection = reflection; + if(parts.length > 1) + while(parts.length) { + if(parts.length > 1) { + objectName = parts[0]; + auto object = objectName in reflection.objects; + if(object is null) { // || object.instantiate is null) + errorMessage = "no such object: " ~ objectName; + goto noSuchFunction; + } + + currentReflection = *object; + + if(!currentReflection.needsInstantiation) { + parts = parts[1 .. $]; + continue; + } + + auto objectIdentifier = parts[1]; + instantiator = objectIdentifier; + + //obj = object.instantiate(objectIdentifier); + + parts = parts[2 .. $]; + + if(parts.length == 0) { + // gotta run the default function + fun = (to!string(cgi.requestMethod)) in currentReflection.functions; + } + } else { + fun = parts[0] in currentReflection.functions; + if(fun is null) + errorMessage = "no such method in class "~objectName~": " ~ parts[0]; + parts = parts[1 .. $]; + } + } + } + } + + if(fun is null) { + noSuchFunction: + if(errorMessage.length) + throw new NoSuchPageException(errorMessage); + string allFuncs, allObjs; + foreach(n, f; reflection.functions) + allFuncs ~= n ~ "\n"; + foreach(n, f; reflection.objects) + allObjs ~= n ~ "\n"; + throw new NoSuchPageException("no such function " ~ funName ~ "\n functions are:\n" ~ allFuncs ~ "\n\nObjects are:\n" ~ allObjs); + } + + assert(fun !is null); + assert(fun.dispatcher !is null); + assert(cgi !is null); + + result.type = fun.returnType; + + string format = cgi.request("format", reflection.defaultOutputFormat); + string secondaryFormat = cgi.request("secondaryFormat", ""); + if(secondaryFormat.length == 0) secondaryFormat = null; + + JSONValue res; + + if(envelopeFormat == "document" && fun.documentDispatcher !is null) { + res = fun.documentDispatcher(cgi, cgi.requestMethod == Cgi.RequestMethod.POST ? cgi.postArray : cgi.getArray); + envelopeFormat = "html"; + } else + res = fun.dispatcher(cgi, instantiator, cgi.requestMethod == Cgi.RequestMethod.POST ? cgi.postArray : cgi.getArray, format, secondaryFormat); + + //if(cgi) + // cgi.setResponseContentType("application/json"); + result.success = true; + result.result = res; + } + catch (Throwable e) { + result.success = false; + result.errorMessage = e.msg; + result.type = e.classinfo.name; + debug result.dFullString = e.toString(); + + if(envelopeFormat == "document" || envelopeFormat == "html") { + auto ipe = cast(InsufficientParametersException) e; + if(ipe !is null) { + assert(fun !is null); + Form form; + if(0 || fun.createForm !is null) { // FIXME: if 0 + // go ahead and use it to make the form page + auto doc = fun.createForm(cgi.requestMethod == Cgi.RequestMethod.POST ? cgi.post : cgi.get); + } else { + Parameter[] params = fun.parameters.dup; + foreach(i, p; fun.parameters) { + string value = ""; + if(p.name in cgi.get) + value = cgi.get[p.name]; + if(p.name in cgi.post) + value = cgi.post[p.name]; + params[i].value = value; + } + + form = createAutomaticForm(new Document, *fun);// params, beautify(fun.originalName)); + foreach(k, v; cgi.get) + form.setValue(k, v); + form.setValue("envelopeFormat", envelopeFormat); + + auto n = form.getElementById("function-name"); + if(n) + n.innerText = beautify(fun.originalName); + } + + assert(form !is null); + result.result.str = form.toString(); + } else { + if(instantiation._errorFunction !is null) { + auto document = instantiation._errorFunction(e); + if(document is null) + goto gotnull; + result.result.str = (document.toString()); + } else { + gotnull: + auto document = new Document; + auto code = document.createElement("pre"); + code.innerText = e.toString(); + + result.result.str = (code.toString()); + } + } + } + } finally { + switch(envelopeFormat) { + case "redirect": + auto redirect = cgi.request("_arsd_redirect_location", cgi.referrer); + + // FIXME: is this safe? it'd make XSS super easy + // add result to url + + if(!result.success) + goto case "none"; + + cgi.setResponseLocation(redirect, false); + break; + case "json": + // this makes firefox ugly + //cgi.setResponseContentType("application/json"); + auto json = toJsonValue(result); + cgi.write(toJSON(&json)); + break; + case "none": + cgi.setResponseContentType("text/plain"); + + if(result.success) { + if(result.result.type == JSON_TYPE.STRING) { + cgi.write(result.result.str); + } else { + cgi.write(toJSON(&result.result)); + } + } else { + cgi.write(result.errorMessage); + } + break; + case "document": + case "html": + default: + cgi.setResponseContentType("text/html"); + + if(result.result.type == JSON_TYPE.STRING) { + auto returned = result.result.str; + + if((fun !is null) && envelopeFormat != "html") { + Document document; + if(fun.returnTypeIsDocument) { + // probably not super efficient... + document = new TemplatedDocument(returned); + } else { + auto e = instantiation._getGenericContainer(); + document = e.parentDocument; + // FIXME: slow, esp if func return element + e.innerHTML = returned; + } + + if(envelopeFormat == "document") + instantiation._postProcess(document); + + returned = document.toString; + } + + cgi.write(returned); + } else + cgi.write(htmlEntitiesEncode(toJSON(&result.result))); + break; + } + + cgi.close(); + } +} + +mixin template FancyMain(T, Args...) { + void fancyMainFunction(Cgi cgi) { //string[] args) { +// auto cgi = new Cgi; + + // there must be a trailing slash for relative links.. + if(cgi.pathInfo.length == 0) { + cgi.setResponseLocation(cgi.requestUri ~ "/"); + cgi.close(); + return; + } + + // FIXME: won't work for multiple objects + T instantiation = new T(); + auto reflection = prepareReflection!(T)(cgi, instantiation); + + run(cgi, instantiation); +/+ + if(args.length > 1) { + string[string][] namedArgs; + foreach(arg; args[2..$]) { + auto lol = arg.indexOf("="); + if(lol == -1) + throw new Exception("use named args for all params"); + //namedArgs[arg[0..lol]] = arg[lol+1..$]; // FIXME + } + + if(!(args[1] in reflection.functions)) { + throw new Exception("No such function"); + } + + //writefln("%s", reflection.functions[args[1]].dispatcher(null, namedArgs, "string")); + } else { ++/ +// } + } + + mixin GenericMain!(fancyMainFunction, Args); +} + +Form createAutomaticForm(Document document, in FunctionInfo func, string[string] fieldTypes = null) { + return createAutomaticForm(document, func.name, func.parameters, beautify(func.originalName), "POST", fieldTypes); +} + +Form createAutomaticForm(Document document, string action, in Parameter[] parameters, string submitText = "Submit", string method = "POST", string[string] fieldTypes = null) { + assert(document !is null); + auto form = cast(Form) document.createElement("form"); + + form.action = action; + + assert(form !is null); + form.method = method; + + + auto fieldset = document.createElement("fieldset"); + auto legend = document.createElement("legend"); + legend.innerText = submitText; + fieldset.appendChild(legend); + + auto table = cast(Table) document.createElement("table"); + assert(table !is null); + + form.appendChild(fieldset); + fieldset.appendChild(table); + + table.appendChild(document.createElement("tbody")); + + static int count = 0; + + foreach(param; parameters) { + Element input; + + string type = param.type; + if(param.name in fieldTypes) + type = fieldTypes[param.name]; + + if(type == "select") { + input = document.createElement("select"); + + foreach(idx, opt; param.options) { + auto option = document.createElement("option"); + option.name = opt; + option.value = param.optionValues[idx]; + + option.innerText = beautify(opt); + + if(option.value == param.value) + option.selected = "selected"; + + input.appendChild(option); + } + + input.name = param.name; + } else if (type == "radio") { + assert(0, "FIXME"); + } else { + if(type.startsWith("textarea")) { + input = document.createElement("textarea"); + input.name = param.name; + input.innerText = param.value; + + auto idx = type.indexOf("-"); + if(idx != -1) { + idx++; + input.rows = type[idx .. $]; + } + } else { + input = document.createElement("input"); + input.type = type; + input.name = param.name; + input.value = param.value; + } + } + + string n = param.name ~ "_auto-form-" ~ to!string(count); + + input.id = n; + + if(type == "hidden") { + form.appendChild(input); + } else { + auto th = document.createElement("th"); + auto label = document.createElement("label"); + label.setAttribute("for", n); + label.innerText = beautify(param.name) ~ ": "; + th.appendChild(label); + + table.appendRow(th, input); + } + + count++; + }; + + auto fmt = document.createElement("select"); + fmt.name = "format"; + fmt.addChild("option", "html").setAttribute("value", "html"); + fmt.addChild("option", "table").setAttribute("value", "table"); + fmt.addChild("option", "json").setAttribute("value", "json"); + fmt.addChild("option", "string").setAttribute("value", "string"); + auto th = table.th(""); + th.addChild("label", "Format:"); + + table.appendRow(th, fmt).className = "format-row"; + + + auto submit = document.createElement("input"); + submit.value = submitText; + submit.type = "submit"; + + table.appendRow(Html(" "), submit); + +// form.setValue("format", reflection.defaultOutputFormat); + + return form; +} + + +/** + * Returns the parameter names of the given function + * + * Params: + * func = the function alias to get the parameter names of + * + * Returns: an array of strings containing the parameter names + */ +/+ +string parameterNamesOf( alias fn )( ) { + string fullName = typeof(&fn).stringof; + + int pos = fullName.lastIndexOf( ')' ); + int end = pos; + int count = 0; + do { + if ( fullName[pos] == ')' ) { + count++; + } else if ( fullName[pos] == '(' ) { + count--; + } + pos--; + } while ( count > 0 ); + + return fullName[pos+2..end]; +} ++/ + + +template parameterNamesOf (alias func) +{ + const parameterNamesOf = parameterNamesOfImpl!(func); +} + + +int indexOfNew(string s, char a) { + foreach(i, c; s) + if(c == a) + return i; + return -1; +} + +/** + * Returns the parameter names of the given function + * + * Params: + * func = the function alias to get the parameter names of + * + * Returns: an array of strings containing the parameter names + */ +private string[] parameterNamesOfImpl (alias func) () +{ + string funcStr = typeof(&func).stringof; + + auto start = funcStr.indexOfNew('('); + auto end = funcStr.indexOfNew(')'); + + const firstPattern = ' '; + const secondPattern = ','; + + funcStr = funcStr[start + 1 .. end]; + + if (funcStr == "") + return null; + + funcStr ~= secondPattern; + + string token; + string[] arr; + + foreach (c ; funcStr) + { + if (c != firstPattern && c != secondPattern) + token ~= c; + + else + { + if (token) + arr ~= token; + + token = null; + } + } + + if (arr.length == 1) + return arr; + + string[] result; + bool skip = false; + + foreach (str ; arr) + { + skip = !skip; + + if (skip) + continue; + + result ~= str; + } + + return result; +} +///////////////////////////////// + +string toHtml(T)(T a) { + string ret; + + static if(is(T : Document)) + ret = a.toString(); + else + static if(isArray!(T)) { + static if(__traits(compiles, typeof(T[0]).makeHtmlArray(a))) + ret = to!string(typeof(T[0]).makeHtmlArray(a)); + else + foreach(v; a) + ret ~= toHtml(v); + } else static if(is(T : Element)) + ret = a.toString(); + else static if(__traits(compiles, a.makeHtmlElement().toString())) + ret = a.makeHtmlElement().toString(); + else static if(is(T == Html)) + ret = a.source; + else + ret = htmlEntitiesEncode(std.array.replace(to!string(a), "\n", "
\n")); + + return ret; +} + +string toJson(T)(T a) { + auto v = toJsonValue(a); + return toJSON(&v); +} + +// FIXME: are the explicit instantiations of this necessary? +JSONValue toJsonValue(T, R = ApiProvider)(T a, string formatToStringAs = null, R api = null) + if(is(R : ApiProvider)) +{ + JSONValue val; + static if(is(T == JSONValue)) { + val = a; + } else static if(__traits(compiles, val = a.makeJsonValue())) { + val = a.makeJsonValue(); + // FIXME: free function to emulate UFCS? + + // FIXME: should we special case something like struct Html? + } else static if(is(T : Element)) { + if(a is null) { + val.type = JSON_TYPE.NULL; + } else { + val.type = JSON_TYPE.STRING; + val.str = a.toString(); + } + } else static if(isIntegral!(T)) { + val.type = JSON_TYPE.INTEGER; + val.integer = to!long(a); + } else static if(isFloatingPoint!(T)) { + val.type = JSON_TYPE.FLOAT; + val.floating = to!real(a); + static assert(0); + } else static if(is(T == void*)) { + val.type = JSON_TYPE.NULL; + } else static if(isPointer!(T)) { + if(a is null) { + val.type = JSON_TYPE.NULL; + } else { + val = toJsonValue!(typeof(*a), R)(*a, formatToStringAs, api); + } + } else static if(is(T == bool)) { + if(a == true) + val.type = JSON_TYPE.TRUE; + if(a == false) + val.type = JSON_TYPE.FALSE; + } else static if(isSomeString!(T)) { + val.type = JSON_TYPE.STRING; + val.str = to!string(a); + } else static if(isAssociativeArray!(T)) { + val.type = JSON_TYPE.OBJECT; + foreach(k, v; a) { + val.object[to!string(k)] = toJsonValue!(typeof(v), R)(v, formatToStringAs, api); + } + } else static if(isArray!(T)) { + val.type = JSON_TYPE.ARRAY; + val.array.length = a.length; + foreach(i, v; a) { + val.array[i] = toJsonValue!(typeof(v), R)(v, formatToStringAs, api); + } + } else static if(is(T == struct)) { // also can do all members of a struct... + val.type = JSON_TYPE.OBJECT; + + foreach(i, member; a.tupleof) { + string name = a.tupleof[i].stringof[2..$]; + static if(a.tupleof[i].stringof[2] != '_') + val.object[name] = toJsonValue!(typeof(member), R)(member, formatToStringAs, api); + } + // HACK: bug in dmd can give debug members in a non-debug build + //static if(__traits(compiles, __traits(getMember, a, member))) + } else { /* our catch all is to just do strings */ + val.type = JSON_TYPE.STRING; + val.str = to!string(a); + // FIXME: handle enums + } + + + // don't want json because it could recurse + if(val.type == JSON_TYPE.OBJECT && formatToStringAs !is null && formatToStringAs != "json") { + JSONValue formatted; + formatted.type = JSON_TYPE.STRING; + + formatAs!(T, R)(a, api, formatted, formatToStringAs, null /* only doing one level of special formatting */); + assert(formatted.type == JSON_TYPE.STRING); + val.object["formattedSecondarily"] = formatted; + } + + return val; +} + +/+ +Document toXml(T)(T t) { + auto xml = new Document; + xml.parse(emptyTag(T.stringof), true, true); + xml.prolog = `` ~ "\n"; + + xml.root = toXmlElement(xml, t); + return xml; +} + +Element toXmlElement(T)(Document document, T t) { + Element val; + static if(is(T == Document)) { + val = t.root; + //} else static if(__traits(compiles, a.makeJsonValue())) { + // val = a.makeJsonValue(); + } else static if(is(T : Element)) { + if(t is null) { + val = document.createElement("value"); + val.innerText = "null"; + val.setAttribute("isNull", "true"); + } else + val = t; + } else static if(is(T == void*)) { + val = document.createElement("value"); + val.innerText = "null"; + val.setAttribute("isNull", "true"); + } else static if(isPointer!(T)) { + if(t is null) { + val = document.createElement("value"); + val.innerText = "null"; + val.setAttribute("isNull", "true"); + } else { + val = toXmlElement(document, *t); + } + } else static if(isAssociativeArray!(T)) { + val = document.createElement("value"); + foreach(k, v; t) { + auto e = document.createElement(to!string(k)); + e.appendChild(toXmlElement(document, v)); + val.appendChild(e); + } + } else static if(isSomeString!(T)) { + val = document.createTextNode(to!string(t)); + } else static if(isArray!(T)) { + val = document.createElement("array"); + foreach(i, v; t) { + auto e = document.createElement("item"); + e.appendChild(toXmlElement(document, v)); + val.appendChild(e); + } + } else static if(is(T == struct)) { // also can do all members of a struct... + val = document.createElement(T.stringof); + foreach(member; __traits(allMembers, T)) { + if(member[0] == '_') continue; // FIXME: skip member functions + auto e = document.createElement(member); + e.appendChild(toXmlElement(document, __traits(getMember, t, member))); + val.appendChild(e); + } + } else { /* our catch all is to just do strings */ + val = document.createTextNode(to!string(t)); + // FIXME: handle enums + } + + return val; +} ++/ + + + +class InsufficientParametersException : Exception { + this(string functionName, string msg) { + super(functionName ~ ": " ~ msg); + } +} + +class InvalidParameterException : Exception { + this(string param, string value, string expected) { + super("bad param: " ~ param ~ ". got: " ~ value ~ ". Expected: " ~expected); + } +} + +void badParameter(alias T)(string expected = "") { + throw new InvalidParameterException(T.stringof, T, expected); +} + +class PermissionDeniedException : Exception { + this(string msg) { + super(msg); + } +} + +class NoSuchPageException : Exception { + this(string msg) { + super(msg); + } +} + +type fromUrlParam(type)(string[] ofInterest) { + type ret; + + // Arrays in a query string are sent as the name repeating... + static if(isArray!(type) && !isSomeString!(type)) { + foreach(a; ofInterest) { + ret ~= to!(ElementType!(type))(a); + } + } + else static if(is(type : Element)) { + auto doc = new Document(ofInterest[$-1], true, true); + + ret = doc.root; + } + /* + else static if(is(type : struct)) { + static assert(0, "struct not supported yet"); + } + */ + else { + // enum should be handled by this too + ret = to!type(ofInterest[$-1]); + } // FIXME: can we support classes? + + + return ret; +} + +WrapperFunction generateWrapper(alias getInstantiation, alias f, alias group, string funName, R)(ReflectionInfo* reflection, R api) if(is(R: ApiProvider)) { + JSONValue wrapper(Cgi cgi, string instantiationIdentifier, in string[][string] sargs, in string format, in string secondaryFormat = null) { + JSONValue returnValue; + returnValue.type = JSON_TYPE.STRING; + + auto instantiation = getInstantiation(instantiationIdentifier, api); + + ParameterTypeTuple!(f) args; + + Throwable t; // the error we see + + // this permission check thing might be removed. It's just there so you can check before + // doing the automatic form... but I think that would be better done some other way. + static if(__traits(hasMember, group, funName ~ "_PermissionCheck")) { + ParameterTypeTuple!(__traits(getMember, group, funName ~ "_PermissionCheck")) argsperm; + + foreach(i, type; ParameterTypeTuple!(__traits(getMember, group, funName ~ "_PermissionCheck"))) { + string name = parameterNamesOf!(__traits(getMember, group, funName ~ "_PermissionCheck"))[i]; + static if(is(type == bool)) { + if(name in sargs) + args[i] = true; + else + args[i] = false; + } else { + if(!(name in sargs)) { + t = new InsufficientParametersException(funName, "arg " ~ name ~ " is not present for permission check"); + goto maybeThrow; + } + argsperm[i] = to!type(sargs[name][$-1]); + } + } + + __traits(getMember, group, funName ~ "_PermissionCheck")(argsperm); + } + // done with arguably useless permission check + + + // Actually calling the function + foreach(i, type; ParameterTypeTuple!(f)) { + string name = parameterNamesOf!(f)[i]; + + // We want to check the named argument first. If it's not there, + // try the positional arguments + string using = name; + if(name !in sargs) + using = "positional-arg-" ~ to!string(i); + + // FIXME: if it's a struct, we should do it's pieces independently here + + static if(is(type == bool)) { + // bool is special cased because HTML checkboxes don't send anything if it isn't checked + if(using in sargs) + args[i] = true; // FIXME: should try looking at the value + else + args[i] = false; + } else { + if(using !in sargs) { + throw new InsufficientParametersException(funName, "arg " ~ name ~ " is not present"); + } + + // We now check the type reported by the client, if there is one + // Right now, only one type is supported: ServerResult, which means + // it's actually a nested function call + + string[] ofInterest = cast(string[]) sargs[using]; // I'm changing the reference, but not the underlying stuff, so this cast is ok + + if(using ~ "-type" in sargs) { + string reportedType = sargs[using ~ "-type"][$-1]; + if(reportedType == "ServerResult") { + + // FIXME: doesn't handle functions that return + // compound types (structs, arrays, etc) + + ofInterest = null; + + string str = sargs[using][$-1]; + int idx = str.indexOf("?"); + string callingName, callingArguments; + if(idx == -1) { + callingName = str; + } else { + callingName = str[0..idx]; + callingArguments = str[idx + 1 .. $]; + } + + // find it in reflection + ofInterest ~= reflection.functions[callingName]. + dispatcher(cgi, null, decodeVariables(callingArguments), "string").str; + } + } + + + args[i] = fromUrlParam!type(ofInterest); + } + } + + static if(!is(ReturnType!f == void)) + ReturnType!(f) ret; + else + void* ret; + + static if(!is(ReturnType!f == void)) + ret = instantiation(args); // version 1 didn't handle exceptions + else + instantiation(args); + + formatAs(ret, api, returnValue, format, secondaryFormat); + + done: + + return returnValue; + } + + return &wrapper; +} + + +void formatAs(T, R)(T ret, R api, ref JSONValue returnValue, string format, string formatJsonToStringAs = null) if(is(R : ApiProvider)) { + + if(api !is null) { + static if(__traits(compiles, api.customFormat(ret, format))) { + auto customFormatted = api.customFormat(ret, format); + if(customFormatted !is null) { + returnValue.str = customFormatted; + return; + } + } + } + switch(format) { + case "html": + // FIXME: should we actually post process here? + /+ + static if(is(typeof(ret) : Document)) { + instantiation._postProcess(ret); + return ret.toString(); + break; + } + static if(__traits(hasMember, group, funName ~ "_Page")) { + auto doc = __traits(getMember, group, funName ~ "_Page")(ret); + instantiation._postProcess(doc); + return doc.toString(); + break; + } + +/ + + returnValue.str = toHtml(ret); + break; + case "string": + static if(__traits(compiles, to!string(ret))) + returnValue.str = to!string(ret); + else goto badType; + break; + case "json": + returnValue = toJsonValue!(typeof(ret), R)(ret, formatJsonToStringAs, api); + break; + case "table": + auto document = new Document(""); + static if(__traits(compiles, structToTable(document, ret))) + returnValue.str = structToTable(document, ret).toString(); + else + goto badType; + break; + default: + badType: + throw new Exception("Couldn't get result as " ~ format); + } +} + + +private string emptyTag(string rootName) { + return ("<" ~ rootName ~ ">"); +} + + +alias JSONValue delegate(Cgi cgi, string, in string[][string] args, in string format, in string secondaryFormat = null) WrapperFunction; + +string urlToBeauty(string url) { + string u = url.replace("/", ""); + + string ret; + + bool capitalize = true; + foreach(c; u) { + if(capitalize) { + ret ~= ("" ~ c).toUpper; + capitalize = false; + } else { + if(c == '-') { + ret ~= " "; + capitalize = true; + } else + ret ~= c; + } + } + + return ret; +} + +string toUrlName(string name) { + string res; + foreach(c; name) { + if(c >= 'a' && c <= 'z') + res ~= c; + else { + res ~= '-'; + if(c >= 'A' && c <= 'Z') + res ~= c + 0x20; + else + res ~= c; + } + } + return res; +} + +string beautify(string name) { + string n; + n ~= toUpper(name[0..1]); + + dchar last; + foreach(dchar c; name[1..$]) { + if((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + if(last != ' ') + n ~= " "; + } + + if(c == '_') + n ~= " "; + else + n ~= c; + last = c; + } + return n; +} + + + + + + +import std.md5; +import core.stdc.stdlib; +import core.stdc.time; +import std.file; +string getSessionId(Cgi cgi) { + static string token; // FIXME: should this actually be static? it seems wrong + if(token is null) { + if("_sess_id" in cgi.cookies) + token = cgi.cookies["_sess_id"]; + else { + auto tmp = uniform(0, int.max); + token = to!string(tmp); + + cgi.setCookie("_sess_id", token, /*60 * 8 * 1000*/ 0, "/", null, true); + } + } + + return getDigestString(cgi.remoteAddress ~ "\r\n" ~ cgi.userAgent ~ "\r\n" ~ token); +} + +void setLoginCookie(Cgi cgi, string name, string value) { + cgi.setCookie(name, value, 0, "/", null, true); +} + +string htmlTemplateWithData(in string text, in string[string] vars) { + assert(text !is null); + + string newText = text; + + if(vars !is null) + foreach(k, v; vars) { + //assert(k !is null); + //assert(v !is null); + newText = newText.replace("{$" ~ k ~ "}", htmlEntitiesEncode(v).replace("\n", "
")); + } + + return newText; +} + +string htmlTemplate(string filename, string[string] vars) { + return htmlTemplateWithData(readText(filename), vars); +} + +class TemplatedDocument : Document { + const override string toString() { + string s; + if(vars !is null) + s = htmlTemplateWithData(super.toString(), vars); + else + s = super.toString(); + + return s; + } + + public: + string[string] vars; + + this(string src) { + super(); + parse(src, true, true); + } + + this() { } + + void delegate(TemplatedDocument)[] preToStringFilters; + void delegate(ref string)[] postToStringFilters; +} + +void writeDocument(Cgi cgi, TemplatedDocument document) { + foreach(f; document.preToStringFilters) + f(document); + + auto s = document.toString(); + + foreach(f; document.postToStringFilters) + f(s); + + cgi.write(s); +} + +/* Password helpers */ + +string makeSaltedPasswordHash(string userSuppliedPassword, string salt = null) { + if(salt is null) + salt = to!string(uniform(0, int.max)); + + return hashToString(SHA256(salt ~ userSuppliedPassword)) ~ ":" ~ salt; +} + +bool checkPassword(string saltedPasswordHash, string userSuppliedPassword) { + auto parts = saltedPasswordHash.split(":"); + + return makeSaltedPasswordHash(userSuppliedPassword, parts[1]) == saltedPasswordHash; +} + + + +Table structToTable(T)(Document document, T arr, string[] fieldsToSkip = null) if(isArray!(T) && !isAssociativeArray!(T)) { + auto t = cast(Table) document.createElement("table"); + t.border = "1"; + + static if(is(T == string[string][])) { + string[string] allKeys; + foreach(row; arr) { + foreach(k; row.keys) + allKeys[k] = k; + } + + auto sortedKeys = allKeys.keys.sort; + Element tr; + + auto thead = t.addChild("thead"); + auto tbody = t.addChild("tbody"); + + tr = thead.addChild("tr"); + foreach(key; sortedKeys) + tr.addChild("th", key); + + bool odd = true; + foreach(row; arr) { + tr = tbody.addChild("tr"); + foreach(k; sortedKeys) { + tr.addChild("td", k in row ? row[k] : ""); + } + if(odd) + tr.addClass("odd"); + + odd = !odd; + } + } else static if(is(typeof(T[0]) == struct)) { + { + auto thead = t.addChild("thead"); + auto tr = thead.addChild("tr"); + auto s = arr[0]; + foreach(idx, member; s.tupleof) + tr.addChild("th", s.tupleof[idx].stringof[2..$]); + } + + bool odd = true; + auto tbody = t.addChild("tbody"); + foreach(s; arr) { + auto tr = tbody.addChild("tr"); + foreach(member; s.tupleof) { + tr.addChild("td", to!string(member)); + } + + if(odd) + tr.addClass("odd"); + + odd = !odd; + } + } else static assert(0); + + return t; +} + +// this one handles horizontal tables showing just one item +Table structToTable(T)(Document document, T s, string[] fieldsToSkip = null) if(!isArray!(T) || isAssociativeArray!(T)) { + static if(__traits(compiles, s.makeHtmlTable(document))) + return s.makeHtmlTable(document); + else { + + auto t = cast(Table) document.createElement("table"); + + static if(is(T == struct)) { + main: foreach(i, member; s.tupleof) { + string name = s.tupleof[i].stringof[2..$]; + foreach(f; fieldsToSkip) + if(name == f) + continue main; + + string nameS = name.idup; + name = ""; + foreach(idx, c; nameS) { + if(c >= 'A' && c <= 'Z') + name ~= " " ~ c; + else if(c == '_') + name ~= " "; + else + name ~= c; + } + + t.appendRow(t.th(name.capitalize), + to!string(member)); + } + } else static if(is(T == string[string])) { + foreach(k, v; s){ + t.appendRow(t.th(k), v); + } + } else static assert(0); + + return t; + } +} + +debug string javascriptBase = ` + // change this in your script to get fewer error popups + "_debugMode":true,` ~ javascriptBaseImpl; +else string javascriptBase = ` + // change this in your script to get more details in errors + "_debugMode":false,` ~ javascriptBaseImpl; + +enum string javascriptBaseImpl = q{ + "_doRequest": function(url, args, callback, method, async) { + var xmlHttp; + try { + xmlHttp=new XMLHttpRequest(); + } + catch (e) { + try { + xmlHttp=new ActiveXObject("Msxml2.XMLHTTP"); + } + catch (e) { + xmlHttp=new ActiveXObject("Microsoft.XMLHTTP"); + } + } + + if(async) + xmlHttp.onreadystatechange=function() { + if(xmlHttp.readyState==4) { + if(callback) { + callback(xmlHttp.responseText, xmlHttp.responseXML); + } + } + } + + var argString = this._getArgString(args); + if(method == "GET" && url.indexOf("?") == -1) + url = url + "?" + argString; + + xmlHttp.open(method, url, async); + + var a = ""; + + if(method == "POST") { + xmlHttp.setRequestHeader("Content-type","application/x-www-form-urlencoded"); + a = argString; + } else { + xmlHttp.setRequestHeader("Content-type", "text/plain"); + } + + xmlHttp.send(a); + + if(!async && callback) { + return callback(xmlHttp.responseText, xmlHttp.responseXML); + } + return xmlHttp; + }, + + "_raiseError":function(type, message) { + var error = new Error(message); + error.name = type; + throw error; + }, + + "_getUriRelativeToBase":function(name, args) { + var str = name; + var argsStr = this._getArgString(args); + if(argsStr.length) + str += "?" + argsStr; + + return str; + }, + + "_getArgString":function(args) { + var a = ""; + var outputted = false; + var i; // wow Javascript sucks! god damned global loop variables + for(i in args) { + if(outputted) { + a += "&"; + } else outputted = true; + var arg = args[i]; + var argType = ""; + // Make sure the types are all sane + + if(arg && arg._arsdTypeOf && arg._arsdTypeOf == "ServerResult") { + argType = arg._arsdTypeOf; + arg = this._getUriRelativeToBase(arg._serverFunction, arg._serverArguments); + + // this arg is a nested server call + a += encodeURIComponent(i) + "="; + a += encodeURIComponent(arg); + } else if(arg && arg.length && typeof arg != "string") { + // FIXME: are we sure this is actually an array? + + var outputtedHere = false; + for(var idx = 0; idx < arg.length; idx++) { + if(outputtedHere) { + a += "&"; + } else outputtedHere = true; + + // FIXME: ought to be recursive + a += encodeURIComponent(i) + "="; + a += encodeURIComponent(arg[idx]); + } + + } else { + // a regular argument + a += encodeURIComponent(i) + "="; + a += encodeURIComponent(arg); + } + // else if: handle arrays and objects too + + if(argType.length > 0) { + a += "&"; + a += encodeURIComponent(i + "-type") + "="; + a += encodeURIComponent(argType); + } + } + + return a; + }, + + "_onError":function(error) { + throw error; + }, + + /// returns an object that can be used to get the actual response from the server + "_serverCall": function (name, passedArgs, returnType) { + var me = this; // this is the Api object + var args = passedArgs; + return { + // type info metadata + "_arsdTypeOf":"ServerResult", + "_staticType":(typeof returnType == "undefined" ? null : returnType), + + // Info about the thing + "_serverFunction":name, + "_serverArguments":args, + + // lower level implementation + "_get":function(callback, onError, async) { + var resObj = this; + if(args == null) + args = {}; + if(!args.format) + args.format = "json"; + args.envelopeFormat = "json"; + return me._doRequest(me._apiBase + name, args, function(t, xml) { + if(me._debugMode) { + try { + var obj = eval("(" + t + ")"); + } catch(e) { + alert("Bad server json: " + e + + "\nOn page: " + (me._apiBase + name) + + "\nGot:\n" + t); + } + } else { + var obj = eval("(" + t + ")"); + } + + if(obj.success) { + if(typeof callback == "function") + callback(obj.result); + else if(typeof resObj.onSuccess == "function") { + resObj.onSuccess(obj.result); + } else if(typeof me.onSuccess == "function") { // do we really want this? + me.onSuccess(obj.result); + } else { + // can we automatically handle it? + // If it's an element, we should replace innerHTML by ID if possible + // if a callback is given and it's a string, that's an id. Return type of element + // should replace that id. return type of string should be appended + // FIXME: meh just do something here. + } + + return obj.result; + } else { + // how should we handle the error? I guess throwing is better than nothing + // but should there be an error callback too? + var error = new Error(obj.errorMessage); + error.name = obj.type; + error.functionUrl = me._apiBase + name; + error.functionArgs = args; + error.errorMessage = obj.errorMessage; + + // myFunction.caller should be available and checked too + // btw arguments.callee is like this for functions + + if(me._debugMode) { + var ourMessage = obj.type + ": " + obj.errorMessage + + "\nOn: " + me._apiBase + name; + if(args.toSource) + ourMessage += args.toSource(); + if(args.stack) + ourMessage += "\n" + args.stack; + + error.message = ourMessage; + + // alert(ourMessage); + } + + if(onError) // local override first... + return onError(error); + else if(resObj.onError) // then this object + return resObj.onError(error); + else if(me._onError) // then the global object + return me._onError(error); + + throw error; // if all else fails... + } + + // assert(0); // not reached + }, (name.indexOf("get") == 0) ? "GET" : "POST", async); // FIXME: hack: naming convention used to figure out method to use + }, + + // should pop open the thing in HTML format + // "popup":null, // FIXME not implemented + + "onError":null, // null means call the global one + + "onSuccess":null, // a generic callback. generally pass something to get instead. + + "formatSet":false, // is the format overridden? + + // gets the result. Works automatically if you don't pass a callback. + // You can also curry arguments to your callback by listing them here. The + // result is put on the end of the arg list to the callback + "get":function(callbackObj) { + var callback = null; + var errorCb = null; + var callbackThis = null; + if(callbackObj) { + if(typeof callbackObj == "function") + callback = callbackObj; + else { + if(callbackObj.length) { + // array + callback = callbackObj[0]; + + if(callbackObj.length >= 2) + errorCb = callbackObj[1]; + } else { + if(callbackObj.onSuccess) + callback = callbackObj.onSuccess; + if(callbackObj.onError) + errorCb = callbackObj.onError; + if(callbackObj.self) + callbackThis = callbackObj.self; + else + callbackThis = callbackObj; + } + } + } + if(arguments.length > 1) { + var ourArguments = []; + for(var a = 1; a < arguments.length; a++) + ourArguments.push(arguments[a]); + + function cb(obj, xml) { + ourArguments.push(obj); + ourArguments.push(xml); + + // that null is the this object inside the function... can + // we make that work? + return callback.apply(callbackThis, ourArguments); + } + + function cberr(err) { + ourArguments.unshift(err); + + // that null is the this object inside the function... can + // we make that work? + return errorCb.apply(callbackThis, ourArguments); + } + + + this._get(cb, errorCb ? cberr : null, true); + } else { + this._get(callback, errorCb, true); + } + }, + + // If you need a particular format, use this. + "getFormat":function(format /* , same args as get... */) { + this.format(format); + var forwardedArgs = []; + for(var a = 1; a < arguments.length; a++) + forwardedArgs.push(arguments[a]); + this.get.apply(this, forwardedArgs); + }, + + // sets the format of the request so normal get uses it + // myapi.someFunction().format('table').get(...); + // see also: getFormat and getHtml + // the secondaryFormat only makes sense if format is json. It + // sets the format returned by object.toString() in the returned objects. + "format":function(format, secondaryFormat) { + if(args == null) + args = {}; + args.format = format; + + if(typeof secondaryFormat == "string" && secondaryFormat) { + if(format != "json") + me._raiseError("AssertError", "secondaryFormat only works if format == json"); + args.secondaryFormat = secondaryFormat; + } + + this.formatSet = true; + return this; + }, + + "getHtml":function(/* args to get... */) { + this.format("html"); + this.get.apply(this, arguments); + }, + + // FIXME: add post aliases + + // don't use unless you're deploying to localhost or something + "getSync":function() { + function cb(obj) { + // no nothing, we're returning the value below + } + + return this._get(cb, null, false); + }, + // takes the result and appends it as html to the given element + + // FIXME: have a type override + "appendTo":function(what) { + if(!this.formatSet) + this.format("html"); + this.get(me._appendContent(what)); + }, + // use it to replace the content of the given element + "useToReplace":function(what) { + if(!this.formatSet) + this.format("html"); + this.get(me._replaceContent(what)); + }, + // use to replace the given element altogether + "useToReplaceElement":function(what) { + if(!this.formatSet) + this.format("html"); + this.get(me._replaceElement(what)); + }, + "useToFillForm":function(what) { + this.get(me._fillForm(what)); + } + // runAsScript has been removed, use get(eval) instead + // FIXME: might be nice to have an automatic popin function too + }; + }, + + "getAutomaticForm":function(method) { + return this._serverCall("builtin.getAutomaticForm", {"method":method}, "Form"); + }, + + "_fillForm": function(what) { + var e = this._getElement(what); + if(this._isListOfNodes(e)) + alert("FIXME: list of forms not implemented"); + else return function(obj) { + if(e.elements && typeof obj == "object") { + for(i in obj) + if(e.elements[i]) + e.elements[i].value = obj[i]; // FIXME: what about checkboxes, selects, etc? + } else + throw new Error("unsupported response"); + }; + }, + + "_getElement": function(what) { + var e; + if(typeof what == "string") + e = document.getElementById(what); + else + e = what; + + return e; + }, + + "_isListOfNodes": function(what) { + // length is on both arrays and lists, but some elements + // have it too. We disambiguate with getAttribute + return (what && (what.length && !what.getAttribute)) + }, + + // These are some convenience functions to use as callbacks + "_replaceContent": function(what) { + var e = this._getElement(what); + if(this._isListOfNodes(e)) + return function(obj) { + for(var a = 0; a < obj.length; a++) { + if( (e[a].tagName.toLowerCase() == "input" + && + e[a].getAttribute("type") == "text") + || + e[a].tagName.toLowerCase() == "textarea") + { + e[a].value = obj; + } else + e[a].innerHTML = obj; + } + } + else + return function(obj) { + if( (e.tagName.toLowerCase() == "input" + && + e.getAttribute("type") == "text") + || + e.tagName.toLowerCase() == "textarea") + { + e.value = obj; + } else + e.innerHTML = obj; + } + }, + + // note: what must be only a single element, FIXME: could check the static type + "_replaceElement": function(what) { + var e = this._getElement(what); + if(this._isListOfNodes(e)) + throw new Error("Can only replace individual elements since removal from a list may be unstable."); + return function(obj) { + var n = document.createElement("div"); + n.innerHTML = obj; + + if(n.firstChild) { + e.parentNode.replaceChild(n.firstChild, e); + } else { + e.parentNode.removeChild(e); + } + } + }, + + "_appendContent": function(what) { + var e = this._getElement(what); + if(this._isListOfNodes(e)) // FIXME: repeating myself... + return function(obj) { + for(var a = 0; a < e.length; a++) + e[a].innerHTML += obj; + } + else + return function(obj) { + e.innerHTML += obj; + } + }, +}; + + +/* + + + +Note for future: dom.d makes working with html easy, since you can +do various forms of post processing on it to make custom formats +among other things. + +I'm considering adding similar stuff for CSS and Javascript. +dom.d now has some more css support - you can apply a stylesheet +to a document and get the computed style and do some minor changes +programmically. StyleSheet : css file :: Document : html file. + +My css lexer/parser is still pretty crappy though. Also, I'm +not sure it's worth going all the way here. + +I'm doing some of it to support my little browser, but for server +side programs, I'm not sure how useful it is to do this kind of +thing. + +A simple textual macro would be more useful for css than a +struct for it.... I kinda want nested declarations and some +functions (the sass thing from ruby is kinda nice in some ways). + +But I'm fairly meh on it anyway. + + +For javascript, I wouldn't mind having a D style foreach in it. +But is it worth it writing a fancy javascript AST thingy just +for that? + +Aside from that, I don't mind the language with how sparingly I +use it though. Besides, writing: + +CoolApi.doSomething("asds").appendTo('element'); + +really isn't bad anyway. + + +The benefit for html was very easy and big. I'm not so sure about +css and js. +*/