More work on the new completion process. Fixed #55

actypes.d

@@ -25,6 +25,7 @@ import std.stdio;
 import std.array;
 import messages;
 import std.array;
+import std.typecons;
 
 /**
  * Any special information about a variable declaration symbol.
@@ -74,7 +75,7 @@ public:
 	 *     kind = the symbol's completion kind
 	 *     resolvedType = the resolved type of the symbol
 	 */
-	this(string name, CompletionKind kind, ACSymbol* type)
+	this(string name, CompletionKind kind, const(ACSymbol)* type)
 	{
 		this.name = name;
 		this.kind = kind;
@@ -101,7 +102,7 @@ public:
 	 * Symbols that compose this symbol, such as enum members, class variables,
 	 * methods, etc.
 	 */
-	ACSymbol*[] parts;
+	const(ACSymbol)*[] parts;
 
 	/**
 	 * Symbol's name
@@ -111,7 +112,7 @@ public:
 	/**
 	 * The symbol that represents the type.
 	 */
-	ACSymbol* type;
+	const(ACSymbol)* type;
 
 	/**
 	 * Calltip to display if this is a function
@@ -145,16 +146,6 @@ public:
 	{
 		return cast(typeof(return)) parts.filter!(a => a.name == name).array;
 	}
-
-    /**
-     * Sorts the parts array, and the parts array of each part, and so on.
-     */
-    void sortParts()
-    {
-        parts.sort();
-        foreach (p; parts)
-            p.sortParts();
-    }
 }
 
 struct Scope
@@ -198,7 +189,7 @@ struct Scope
 		return s.getSymbolsByName(name);
 	}
 
-	ACSymbol*[] symbols;
+	const(ACSymbol)*[] symbols;
 	ImportInformation[] importInformation;
 	Scope* parent;
 	Scope*[] children;
@@ -211,7 +202,7 @@ struct ImportInformation
 	/// module relative path
 	string modulePath;
 	/// symbols to import from this module
-	string[string] importedSymbols;
+	Tuple!(string, string)[] importedSymbols;
 	/// true if the import is public
 	bool isPublic;
 }
@@ -279,7 +270,6 @@ static this()
 		s.parts ~= sizeof_;
 		s.parts ~= stringof_;
 		s.parts ~= mangleof_;
-        s.sortParts();
 	}
 
 	auto cdouble_ = new ACSymbol("cdouble", CompletionKind.keyword);
@@ -314,7 +304,6 @@ static this()
 		s.parts ~= new ACSymbol("nan", CompletionKind.keyword, s);
 		s.parts ~= sizeof_;
 		s.parts ~= stringof_;
-        s.sortParts();
 	}
 
 	ireal_.parts ~= new ACSymbol("im", CompletionKind.keyword, real_);
@@ -331,8 +320,8 @@ static this()
 		float_, idouble_, ifloat_, ireal_, real_, ucent_, void_];
 }
 
-ACSymbol*[] builtinSymbols;
+const(ACSymbol)*[] builtinSymbols;
-ACSymbol*[] arraySymbols;
+const(ACSymbol)*[] arraySymbols;
-ACSymbol*[] assocArraySymbols;
+const(ACSymbol)*[] assocArraySymbols;
-ACSymbol*[] classSymbols;
+const(ACSymbol)*[] classSymbols;
-ACSymbol*[] structSymbols;
+const(ACSymbol)*[] structSymbols;

astconverter.d

@@ -25,6 +25,7 @@ import std.array;
 import std.conv;
 import std.path;
 import std.range;
+import std.typecons;
 
 import stdx.d.ast;
 import stdx.d.lexer;
@@ -35,6 +36,7 @@ import constants;
 import messages;
 import semantic;
 import stupidlog;
+import modulecache; // circular import
 
 /**
  * First Pass handles the following:
@@ -256,9 +258,7 @@ final class FirstPass : ASTVisitor
 
 	override void visit(ImportDeclaration importDeclaration)
 	{
-		Log.trace(__FUNCTION__, " ImportDeclaration");
+//		Log.trace(__FUNCTION__, " ImportDeclaration");
-
-
 		foreach (single; importDeclaration.singleImports.filter!(
 			a => a !is null && a.identifierChain !is null))
 		{
@@ -273,10 +273,10 @@ final class FirstPass : ASTVisitor
 			importDeclaration.importBindings.singleImport.identifierChain);
 		foreach (bind; importDeclaration.importBindings.importBinds)
 		{
-			if (bind.right == TokenType.invalid)
+			Tuple!(string, string) bindTuple;
-				info.importedSymbols[bind.left.value] = null;
+			bindTuple[0] = bind.left.value.dup;
-			else
+			bindTuple[1] = bind.right == TokenType.invalid ? null : bind.right.value.dup;
-				info.importedSymbols[bind.left.value] = bind.right.value;
+			info.importedSymbols ~= bindTuple;
 		}
 		currentScope.importInformation ~= info;
 	}
@@ -462,7 +462,7 @@ public:
 
 private:
 
-	void assignToScopes(ACSymbol* currentSymbol)
+	void assignToScopes(const(ACSymbol)* currentSymbol)
 	{
 		moduleScope.getScopeByCursor(currentSymbol.location).symbols
 			~= currentSymbol;
@@ -474,7 +474,35 @@ private:
 	{
 		foreach (importInfo; currentScope.importInformation)
 		{
-			// TODO: actually process imports here
+			auto symbols = ModuleCache.getSymbolsInModule(importInfo.modulePath);
+			if (importInfo.importedSymbols.length == 0)
+			{
+				currentScope.symbols ~= symbols;
+				continue;
+			}
+			symbolLoop: foreach (symbol; symbols)
+			{
+				foreach (tup; importInfo.importedSymbols)
+				{
+					if (tup[0] != symbol.name)
+						continue symbolLoop;
+					if (tup[1] !is null)
+					{
+						ACSymbol* s = new ACSymbol(tup[1],
+							symbol.kind, symbol.type);
+						// TODO: Compiler gets confused here, so cast the types.
+						s.parts = cast(typeof(s.parts)) symbol.parts;
+						// TODO: Re-format callTip with new name?
+						s.callTip = symbol.callTip;
+						s.qualifier = symbol.qualifier;
+						s.location = symbol.location;
+						s.symbolFile = symbol.symbolFile;
+						currentScope.symbols ~= s;
+					}
+					else
+						currentScope.symbols ~= symbol;
+				}
+			}
 		}
 	}
 
@@ -509,22 +537,26 @@ private:
 
 	void thirdPass(SemanticSymbol* currentSymbol)
 	{
+//		Log.trace("third pass on ", currentSymbol.acSymbol.name);
 		with (CompletionKind) final switch (currentSymbol.acSymbol.kind)
 		{
 		case className:
 		case interfaceName:
-			// resolve inheritance
+			resolveInheritance(currentSymbol);
 			goto case structName;
 		case structName:
 		case unionName:
+			resolveAliasThis(currentSymbol);
+			resolveMixinTemplates(currentSymbol);
 			break;
 		case variableName:
 		case memberVariableName:
-			break;
 		case functionName:
+//			Log.trace("Resolving type of ", currentSymbol.acSymbol.name);
+			currentSymbol.acSymbol.type = resolveType(currentSymbol.type,
+				currentSymbol.acSymbol.location);
 			break;
 		case enumName:
-			break;
 		case keyword:
 		case enumMember:
 		case packageName:
@@ -535,33 +567,78 @@ private:
 		case aliasName:
 		case templateName:
 		case mixinTemplateName:
+			break;
+		}
+
+		foreach (child; currentSymbol.children)
+			thirdPass(child);
+	}
+
+	void resolveInheritance(SemanticSymbol* currentSymbol)
+	{
+//		Log.trace("Resolving inheritance for ", currentSymbol.acSymbol.name);
+		outer: foreach (string[] base; currentSymbol.baseClasses)
+		{
+			const(ACSymbol)* baseClass;
+			if (base.length == 0)
+				continue;
+			auto symbols = moduleScope.getSymbolsByNameAndCursor(
+				base[0], currentSymbol.acSymbol.location);
+			if (symbols.length == 0)
+				continue;
+			baseClass = symbols[0];
+			foreach (part; base[1..$])
+			{
+				symbols = baseClass.getPartsByName(part);
+				if (symbols.length == 0)
+					continue outer;
+				baseClass = symbols[0];
+			}
+			currentSymbol.acSymbol.parts ~= baseClass.parts;
 		}
 	}
 
-	ACSymbol* resolveType(const Type t)
+	void resolveAliasThis(SemanticSymbol* currentSymbol)
-	in
 	{
-		assert (t !is null);
+		// TODO:
-		assert (t.type2 !is null);
 	}
-	body
+
+	void resolveMixinTemplates(SemanticSymbol* currentSymbol)
 	{
-		ACSymbol* s;
+		// TODO:
+	}
+
+	const(ACSymbol)* resolveType(Type t, size_t location)
+	{
+		if (t is null) return null;
+		if (t.type2 is null) return null;
+		const(ACSymbol)* s;
 		if (t.type2.builtinType != TokenType.invalid)
 			s = convertBuiltinType(t.type2);
 		else if (t.type2.typeConstructor != TokenType.invalid)
-			s = resolveType(t.type2.type);
+			s = resolveType(t.type2.type, location);
 		else if (t.type2.symbol !is null)
 		{
-			if (t.type2.symbol.dot)
+			// TODO: global scoped symbol handling
-				Log.error("TODO: global scoped symbol handling");
 			string[] symbolParts = expandSymbol(
 				t.type2.symbol.identifierOrTemplateChain);
-
+			auto symbols = moduleScope.getSymbolsByNameAndCursor(
+				symbolParts[0], location);
+			if (symbols.length == 0)
+				goto resolveSuffixes;
+			s = symbols[0];
+			foreach (symbolPart; symbolParts[1..$])
+			{
+				auto parts = s.getPartsByName(symbolPart);
+				if (parts.length == 0)
+					goto resolveSuffixes;
+				s = parts[0];
+			}
 		}
+	resolveSuffixes:
 		foreach (suffix; t.typeSuffixes)
 			s = processSuffix(s, suffix);
-		return null;
+		return s;
 	}
 
 	static string[] expandSymbol(const IdentifierOrTemplateChain chain)
@@ -570,6 +647,8 @@ private:
 		for (size_t i = 0; i != chain.identifiersOrTemplateInstances.length; ++i)
 		{
 			auto identOrTemplate = chain.identifiersOrTemplateInstances[i];
+			if (identOrTemplate is null)
+				continue;
 			strings[i] = identOrTemplate.templateInstance is null ?
 				identOrTemplate.identifier.value.dup
 				: identOrTemplate.identifier.value.dup;
@@ -577,14 +656,14 @@ private:
 		return strings;
 	}
 
-	static ACSymbol* processSuffix(ACSymbol* symbol, const TypeSuffix suffix)
+	static const(ACSymbol)* processSuffix(const(ACSymbol)* symbol, const TypeSuffix suffix)
 	{
 		if (suffix.star)
 			return symbol;
 		if (suffix.array || suffix.type)
 		{
 			ACSymbol* s = new ACSymbol;
-			s.parts = arraySymbols;
+			s.parts = suffix.array ? arraySymbols : assocArraySymbols;
 			s.type = symbol;
 			s.qualifier = suffix.array ? SymbolQualifier.array : SymbolQualifier.assocArray;
 			return s;
@@ -597,7 +676,7 @@ private:
 		return null;
 	}
 
-	static ACSymbol* convertBuiltinType(const Type2 type2)
+	static const(ACSymbol)* convertBuiltinType(const Type2 type2)
 	{
 		string stringRepresentation = getTokenValue(type2.builtinType);
 		if (stringRepresentation is null) return null;
@@ -619,6 +698,7 @@ const(ACSymbol)*[] convertAstToSymbols(Module m, string symbolFile)
     SecondPass second = SecondPass(first.rootSymbol, first.moduleScope);
 	second.run();
 	ThirdPass third = ThirdPass(second.rootSymbol, second.moduleScope);
+	third.run();
     return cast(typeof(return)) third.rootSymbol.acSymbol.parts;
 }
 
@@ -634,19 +714,6 @@ const(Scope)* generateAutocompleteTrees(const(Token)[] tokens, string symbolFile
 	return cast(typeof(return)) third.moduleScope;
 }
 
-version(unittest) Module parseTestCode(string code)
-{
-	LexerConfig config;
-	const(Token)[] tokens = byToken(cast(ubyte[]) code, config);
-	Parser p = new Parser;
-	p.fileName = "unittest";
-	p.tokens = tokens;
-	Module m = p.parseModule();
-	assert (p.errorCount == 0);
-	assert (p.warningCount == 0);
-	return m;
-}
-
 private:
 
 string[] iotcToStringArray(const IdentifierOrTemplateChain iotc)
@@ -664,5 +731,18 @@ string[] iotcToStringArray(const IdentifierOrTemplateChain iotc)
 
 private static string convertChainToImportPath(IdentifierChain chain)
 {
-	return to!string(chain.identifiers.map!(a => a.value).join(dirSeparator).array) ~ ".d";
+	return to!string(chain.identifiers.map!(a => a.value.dup).join(dirSeparator).array) ~ ".d";
+}
+
+version(unittest) Module parseTestCode(string code)
+{
+	LexerConfig config;
+	const(Token)[] tokens = byToken(cast(ubyte[]) code, config);
+	Parser p = new Parser;
+	p.fileName = "unittest";
+	p.tokens = tokens;
+	Module m = p.parseModule();
+	assert (p.errorCount == 0);
+	assert (p.warningCount == 0);
+	return m;
 }

modulecache.d

@@ -63,18 +63,24 @@ struct ModuleCache
 	/**
 	 * Adds the given path to the list of directories checked for imports
 	 */
-	static void addImportPath(string path)
+	static void addImportPaths(string[] paths)
 	{
-		if (!exists(path))
+		foreach (path; paths)
-			return;
-		importPaths ~= path;
-		foreach (fileName; dirEntries(path, "*.{d,di}", SpanMode.depth))
 		{
-			Log.info("Loading and caching completions for ", fileName);
+			if (!exists(path))
-			getSymbolsInModule(fileName);
+			{
-			import core.memory;
+				Log.error("Cannot cache modules in ", path, " because it does not exist");
-			GC.collect();
+				continue;
-			GC.minimize();
+			}
+			importPaths ~= path;
+		}
+		foreach (path; paths)
+		{
+			foreach (fileName; dirEntries(path, "*.{d,di}", SpanMode.depth))
+			{
+				Log.info("Loading and caching completions for ", fileName);
+				getSymbolsInModule(fileName);
+			}
 		}
 	}
 
@@ -90,8 +96,15 @@ struct ModuleCache
 		string location = resolveImportLoctation(moduleName);
 		if (location is null)
 			return [];
+
 		if (!needsReparsing(location))
-			return cache[location].symbols;
+		{
+			if (location in cache)
+				return cache[location].symbols;
+			return [];
+		}
+
+		recursionGuard[location] = true;
 
 		const(ACSymbol)*[] symbols;
 		try
@@ -117,6 +130,10 @@ struct ModuleCache
 		getTimes(location, access, modification);
 		CacheEntry c = CacheEntry(symbols, modification);
 		cache[location] = c;
+		recursionGuard[location] = false;
+		import core.memory;
+		GC.collect();
+		GC.minimize();
 		return symbols;
 	}
 
@@ -142,7 +159,7 @@ struct ModuleCache
 			if (filePath.exists())
 				return filePath;
 		}
-		writeln("Could not find ", moduleName);
+		Log.error("Could not find ", moduleName);
 		return null;
 	}
 
@@ -161,6 +178,10 @@ private:
 	 */
 	static bool needsReparsing(string mod)
 	{
+		if (mod !in recursionGuard)
+			return true;
+		if (recursionGuard[mod])
+			return false;
 		if (!exists(mod) || mod !in cache)
 			return true;
 		SysTime access;
@@ -172,6 +193,8 @@ private:
 	// Mapping of file paths to their cached symbols.
 	static CacheEntry[string] cache;
 
+	static bool[string] recursionGuard;
+
 	// Listing of paths to check for imports
 	static string[] importPaths;
 }

semantic.d

@@ -22,6 +22,7 @@ import messages;
 import actypes;
 import stdx.d.ast;
 import stdx.d.lexer;
+import stupidlog;
 
 /**
  * Intermediate form between ACSymbol and the AST classes. Stores enough
@@ -41,6 +42,11 @@ public:
 		acSymbol.symbolFile = symbolFile;
 	}
 
+	~this()
+	{
+		Log.trace(acSymbol.name, " destructor");
+	}
+
 	void addChild(SemanticSymbol* child)
 	{
 		children ~= child;

server.d

@@ -91,8 +91,7 @@ int main(string[] args)
 		Log.info("Sockets shut down.");
 	}
 
-	foreach (path; importPaths)
+	ModuleCache.addImportPaths(importPaths);
-		ModuleCache.addImportPath(path);
 	Log.info("Import directories: ", ModuleCache.getImportPaths());
 
 	ubyte[] buffer = new ubyte[1024 * 1024 * 4]; // 4 megabytes should be enough for anybody...
@@ -150,11 +149,7 @@ int main(string[] args)
 		msgpack.unpack(buffer[size_t.sizeof .. bytesReceived], request);
 		if (request.kind == RequestKind.addImport)
 		{
-			foreach (path; request.importPaths)
+			ModuleCache.addImportPaths(request.importPaths);
-			{
-				ModuleCache.addImportPath(path);
-			}
-
 		}
 		else if (request.kind == RequestKind.clearCache)
 		{