From e1846a9466a83b953998f5c1dbd0a8917474de8e Mon Sep 17 00:00:00 2001 From: Basile Burg Date: Mon, 4 Jul 2016 03:12:12 +0200 Subject: [PATCH] libman, use hashset and hash map to speed up the selection of the items, close #77 --- src/ce_common.pas | 43 ++++- src/ce_libman.pas | 379 ++++++++++++++++++++++++++------------- src/ce_main.pas | 2 +- src/ce_nativeproject.pas | 20 +-- 4 files changed, 304 insertions(+), 140 deletions(-) diff --git a/src/ce_common.pas b/src/ce_common.pas index 94ef077a..38efae52 100644 --- a/src/ce_common.pas +++ b/src/ce_common.pas @@ -15,7 +15,7 @@ uses {$IFNDEF CEBUILD} forms, {$ENDIF} - process, asyncprocess; + process, asyncprocess, ghashmap, ghashset; const exeExt = {$IFDEF WINDOWS} '.exe' {$ELSE} '' {$ENDIF}; @@ -31,6 +31,21 @@ type TCECompiler = (dmd, gdc, ldc); + // function used as hash in gXXX sets & maps + TStringHash = class + class function hash(const key: string; maxBucketsPow2: longint): longint; + end; + + // HashMap for TValue by string + generic TStringHashMap = class(specialize THashmap); + + // function used as objects in gXXX sets & maps + TObjectHash = class + class function hash(key: TObject; maxBucketsPow2: longint): longint; + end; + + generic TObjectHashSet = class(specialize THashSet); + // aliased to get a custom prop inspector TCEPathname = type string; TCEFilename = type string; @@ -280,6 +295,32 @@ implementation uses ce_main; + +class function TStringHash.hash(const key: string; maxBucketsPow2: longint): longint; +var + i: integer; +begin + {$PUSH}{$R-} + result := 5381; + for i:= 1 to key.length do + begin + result := ((result shl 5) + result) + Byte(key[i]); + end; + result := result and (maxBucketsPow2-1); + {$POP} +end; + +class function TObjectHash.hash(key: TObject; maxBucketsPow2: longint): longint; +begin + {$PUSH}{$R-}{$WARNINGS OFF}{$HINTS OFF} + {$IFDEF CPU32} + Result := longint(Pointer(key)) and (maxBucketsPow2 -1); + {$ELSE} + Result := (longInt(Pointer(key)) xor PlongInt(PInteger(&key) + 4)^) and (maxBucketsPow2 -1); + {$ENDIF} + {$POP} +end; + procedure TCEPersistentShortcut.assign(aValue: TPersistent); var src: TCEPersistentShortcut; diff --git a/src/ce_libman.pas b/src/ce_libman.pas index 21ff71ac..708ed205 100644 --- a/src/ce_libman.pas +++ b/src/ce_libman.pas @@ -6,17 +6,17 @@ interface uses Classes, SysUtils, FileUtil, ce_common, ce_writableComponent, ce_dcd, LazFileUtils, - ce_dialogs, ce_projutils, ce_interfaces, ce_dlang; + ce_dialogs, ce_projutils, ce_interfaces, ce_dlang, ghashmap, ghashset; type - // TODO-clibman: improve import analysis, caching, hashmap, backup, update. + // TODO-clibman: improve import analysis, update (currently always full). (** * Information for a module in a library manager entry *) TModuleInfo = class(TCollectionItem) - private + strict private fName: string; fImports: TStringList; procedure setImports(value: TStringList); @@ -32,73 +32,100 @@ type (** - * Represents a D static library. In a project libAlias allows to - * resolve automatically the dependencies of a project. + * Represents a D static library. *) TLibraryItem = class(TCollectionItem) - private + type + TModulesByName = specialize TStringHashMap; + strict private fAlias: string; - fSourcePath: string; + fLibSourcePath: string; fLibFile: string; - fProjFile: string; + fLibProject: string; fEnabled: boolean; fDependencies: TStringList; fModules: TCollection; + fModulesByName: TModulesByName; + fHasValidSourcePath: boolean; + fHasValidLibFile: boolean; + fHasValidLibProject: boolean; procedure setDependencies(value: TStringList); procedure setModules(value: TCollection); - function getModuleInfo(ix: integer): TModuleInfo; + procedure setLibProject(const value: string); + procedure setLibFile(const value: string); + procedure setLibSourcePath(const value: string); + function getModule(value: integer): TModuleInfo; + function getModule(const value: string): TModuleInfo; + function moduleCount: integer; published property libAlias: string read fAlias write fAlias; - property libSourcePath: string read fSourcePath write fSourcePath; - property libFile: string read fLibFile write fLibFile; - property projectFile: string read fProjFile write fProjFile; + property libSourcePath: string read fLibSourcePath write setLibSourcePath; + property libFile: string read fLibFile write setLibFile; + property libProject: string read fLibProject write setLibProject; property enabled: boolean read fEnabled write fEnabled default true; // TODO-clibman: dont forget that these props are not written property dependencies: TStringList read fDependencies write setDependencies stored false; property modules: TCollection read fModules write setModules stored false; + // TODO-cmaintenace: remove this property from 3 update 1 + property projectFile: string read fLibProject write fLibProject stored false; public constructor Create(ACollection: TCollection); override; destructor Destroy; override; procedure updateModulesInfo; function addModuleInfo: TModuleInfo; function hasModule(const value: string): boolean; - property moduleInfos[ix: integer]: TModuleInfo read getModuleInfo; + property hasValidLibFile: boolean read fHasValidLibFile; + property hasValidLibProject: boolean read fHasValidLibProject; + property hasValidLibSourcePath: boolean read fHasValidSourcePath; + property moduleByIndex[value: integer]: TModuleInfo read getModule; end; + TLibraryList = specialize TObjectHashSet; + (** - * Represents all the D libraries present on this system. + * Represents all the D libraries handled by Coedit. *) TLibraryManager = class(TWritableLfmTextComponent) - private - fCol: TCollection; - function getItems(index: integer): TLibraryItem; - procedure setCol(value: TCollection); + type + TItemsByAlias = specialize TStringHashMap; + strict private + fCollection: TCollection; + fItemsByAlias: TItemsByAlias; + function getLibraryByIndex(index: integer): TLibraryItem; + function getLibraryByAlias(const value: string): TLibraryItem; + function getLibraryByImport(const value: string): TLibraryItem; + procedure setCollection(value: TCollection); + procedure updateItemsByAlias; + function getLibrariesCount: integer; published - property libraries: TCollection read fCol write setCol; + property libraries: TCollection read fCollection write setCollection; public constructor create(aOwner: TComponent); override; destructor destroy; override; (** - * The caller gets all the static library files in list if aliases is nil - * otherwise the static library files selected by the aliases. + * The caller gets all the static library files in "list" if "aliases" is nil + * otherwise the static library files selected by "aliases". *) procedure getLibFiles(aliases, list: TStrings); (** - * The caller gets all the paths were are located the library sources in list - * if aliases is nil otherwise the paths where are located the sources of the - * livraries selected by aliases. + * The caller gets all the paths were are located the library sources in "list" + * if "aliases" is nil otherwise the paths where are located the sources of the + * libraries selected by "aliases". *) - procedure getLibSources(aliases, list: TStrings); + procedure getLibSourcePath(aliases, list: TStrings); (** - * The caller gets all the static library files and the source path - * that are required by the specified source code. + * The caller gets static libraries files in "libs" and source paths + * in "paths", as required by the specified "source" code. *) procedure getLibsForSource(source, libs, paths: TStrings); // procedure updateDCD; // find the aliases of the libraries used by this library. procedure updateCrossDependencies; - property items[index: integer]: TLibraryItem read getItems; default; + property librariesCount: integer read getLibrariesCount; + property libraryByIndex[value: integer]: TLibraryItem read getLibraryByIndex; + property libraryByAlias[const value: string]: TLibraryItem read getLibraryByAlias; + property libraryByImport[const value: string]: TLibraryItem read getLibraryByImport; end; const @@ -130,20 +157,32 @@ constructor TLibraryItem.Create(ACollection: TCollection); begin inherited create(ACollection); fModules := TCollection.Create(TModuleInfo); + fModulesByName := TModulesByName.create; fDependencies := TStringList.Create; fEnabled:=true; end; destructor TLibraryItem.Destroy; begin + fModulesByName.Free; fDependencies.Free; fModules.Free; inherited; end; -function TLibraryItem.getModuleInfo(ix: integer): TModuleInfo; +function TLibraryItem.moduleCount: integer; begin - exit(TModuleInfo(fModules.Items[ix])); + exit(fModules.Count); +end; + +function TLibraryItem.getModule(value: integer): TModuleInfo; +begin + exit(TModuleInfo(fModules.Items[value])); +end; + +function TLibraryItem.getModule(const value: string): TModuleInfo; +begin + exit(fModulesByName.GetData(value)); end; function TLibraryItem.addModuleInfo: TModuleInfo; @@ -152,13 +191,8 @@ begin end; function TLibraryItem.hasModule(const value: string): boolean; -var - i: integer; begin - for i:= 0 to fModules.Count-1 do - if getModuleInfo(i).name = value then - exit(true); - exit(false); + exit(fModulesByName.contains(value)); end; procedure TLibraryItem.setModules(value: TCollection); @@ -171,6 +205,30 @@ begin fDependencies.Assign(value); end; +procedure TLibraryItem.setLibProject(const value: string); +begin + if fLibProject = value then + exit; + fLibProject:=value; + fHasValidLibProject:=value.fileExists; +end; + +procedure TLibraryItem.setLibFile(const value: string); +begin + if fLibFile = value then + exit; + fLibFile:=value; + fHasValidLibFile:=value.fileExists or value.dirExists; +end; + +procedure TLibraryItem.setLibSourcePath(const value: string); +begin + if fLibSourcePath = value then + exit; + fLibSourcePath:=value; + fHasValidSourcePath:=value.dirExists; +end; + procedure TLibraryItem.updateModulesInfo; var prj: ICECommonProject; @@ -182,9 +240,11 @@ var i: integer; begin fModules.Clear; - if fProjFile.fileExists then + fModulesByName.Free; + fModulesByName := TModulesByName.create; + if hasValidLibProject then begin - prj := loadProject(fProjFile, true); + prj := loadProject(fLibProject, true); tks := TLexTokenList.Create; str := TStringList.Create; try @@ -198,6 +258,7 @@ begin lex(str.Text, tks, nil, [lxoNoComments]); mdi := addModuleInfo; mdi.name := getModuleName(tks); + fModulesByName.insert(mdi.name, mdi); getImports(tks, mdi.imports); tks.Clear; end; @@ -206,13 +267,13 @@ begin str.Free; prj.getProject.Free; end; - end else if fSourcePath.dirExists then + end else if hasValidLibSourcePath then begin lst := TStringList.Create; str := TStringList.Create; tks := TLexTokenList.Create; try - listFiles(lst, fSourcePath, true); + listFiles(lst, fLibSourcePath, true); for i := 0 to lst.Count-1 do begin fle := lst[i]; @@ -222,6 +283,7 @@ begin lex(str.Text, tks, nil, [lxoNoComments]); mdi := addModuleInfo; mdi.name := getModuleName(tks); + fModulesByName.insert(mdi.name, mdi); getImports(tks, mdi.imports); tks.Clear; end; @@ -242,11 +304,12 @@ var i: integer; begin inherited; - fCol := TCollection.Create(TLibraryItem); + fItemsByAlias:= TItemsByAlias.create; + fCollection := TCollection.Create(TLibraryItem); fname := getCoeditDocPath + libFname; if fname.fileExists then loadFromFile(fname); - if fCol.Count = 0 then + if fCollection.Count = 0 then begin {$IFDEF WINDOWS} fDmdPath := ExeSearch('dmd.exe'); @@ -274,16 +337,16 @@ begin // add phobos if '/usr/include/dmd/phobos'.dirExists then begin - with TLibraryItem(fCol.Add) do begin + with TLibraryItem(fCollection.Add) do begin libAlias := 'phobos'; libFile := ''; libSourcePath := '/usr/include/dmd/phobos'; end; end; - // add druntime (no items - only for DCD) + // add druntime (no libraryByIndex - only for DCD) if '/usr/include/dmd/druntime/import'.dirExists then begin - with TLibraryItem(fCol.Add) do begin + with TLibraryItem(fCollection.Add) do begin libAlias := 'druntime'; libFile := ''; libSourcePath := '/usr/include/dmd/druntime/import'; @@ -310,7 +373,8 @@ begin end; {$ENDIF} end; - if (fCol.Count = 0) and not (getCoeditDocPath + libFname).fileExists then + updateItemsByAlias; + if (fCollection.Count = 0) and not (getCoeditDocPath + libFname).fileExists then begin dlgOkInfo( 'Coedit failed to add "druntime" and "phobos" to the library manager.' @@ -319,10 +383,10 @@ begin end; updateDCD; // - for i := 0 to fCol.Count-1 do + for i := 0 to fCollection.Count-1 do begin - if (items[i].libAlias <> 'phobos') and (items[i].libAlias <> 'druntime') then - items[i].updateModulesInfo; + if (libraryByIndex[i].libAlias <> 'phobos') and (libraryByIndex[i].libAlias <> 'druntime') then + libraryByIndex[i].updateModulesInfo; end; updateCrossDependencies; end; @@ -331,18 +395,53 @@ destructor TLibraryManager.destroy; begin ForceDirectoriesUTF8(getCoeditDocPath); LibMan.saveToFile(getCoeditDocPath + libFname); - fCol.Free; + fItemsByAlias.Free; + fCollection.Free; inherited; end; -procedure TLibraryManager.setCol(value: TCollection); +procedure TLibraryManager.updateItemsByAlias; +var + i: integer; begin - fCol.assign(value); + fItemsByAlias.Free; + fItemsByAlias := TItemsByAlias.create; + for i:= 0 to fCollection.Count-1 do + fItemsByAlias.insert(libraryByIndex[i].libAlias, libraryByIndex[i]); end; -function TLibraryManager.getItems(index: integer): TLibraryItem; +procedure TLibraryManager.setCollection(value: TCollection); begin - exit(TLibraryItem(fCol.Items[index])); + fCollection.assign(value); +end; + +function TLibraryManager.getLibraryByIndex(index: integer): TLibraryItem; +begin + exit(TLibraryItem(fCollection.Items[index])); +end; + +function TLibraryManager.getLibraryByAlias(const value: string): TLibraryItem; +begin + exit(fItemsByAlias.GetData(value)); +end; + +function TLibraryManager.getLibraryByImport(const value: string): TLibraryItem; +var + i: integer; + s: TLibraryItem; +begin + result := nil; + for i := 0 to librariesCount-1 do + begin + s := libraryByIndex[i]; + if s.hasModule(value) then + exit(s); + end; +end; + +function TLibraryManager.getLibrariesCount: integer; +begin + exit(fCollection.Count); end; procedure TLibraryManager.updateDCD; @@ -352,12 +451,12 @@ var i: Integer; begin if not DcdWrapper.available then exit; - // note: new items are directly handled but removed ones still in cache until server restarts. + // note: new libraryByIndex are directly handled but removed ones still in cache until server restarts. str := TStringList.Create; try - for i := 0 to fCol.Count-1 do + for i := 0 to fCollection.Count-1 do begin - itm := TLibraryItem(fCol.Items[i]); + itm := TLibraryItem(fCollection.Items[i]); if itm.enabled then str.Add(itm.libSourcePath); end; @@ -368,32 +467,20 @@ begin end; procedure TLibraryManager.getLibFiles(aliases, list: TStrings); -var - itm: TLibraryItem; - lst: TStringList; - i,j: Integer; - dir: string; -begin - for i := 0 to fCol.Count-1 do + procedure add(lib: TLibraryItem); + var + j: integer; + dir: string; + lst: TstringList; begin - itm := TLibraryItem(fCol.Items[i]); - if (not itm.enabled) or - (aliases.isNotNil and (aliases.IndexOf(itm.libAlias) = -1)) then - continue; - // single items files - if fileExists(itm.libFile) then - begin - if list.IndexOf(itm.libFile) <> -1 then - continue; - list.Add(itm.libFile); - end - // folder of items file - else if itm.libFile.dirExists then + // as a trick a folder can be set as source, this allows to pass + // multiple sources in an automated way. + if lib.libFile.dirExists then begin lst := TStringList.Create; try - dir := itm.libFile; - if itm.libFile[dir.length] = DirectorySeparator then + dir := lib.libFile; + if lib.libFile[dir.length] = DirectorySeparator then dir := dir[1..dir.length-1]; listFiles(lst, dir); for j:= 0 to lst.Count-1 do @@ -405,27 +492,57 @@ begin finally lst.Free; end; + end + else if lib.hasValidLibFile then + list.Add(lib.libFile); + end; +var + lib: TLibraryItem; + i: Integer; +begin + // no selector = all libs + if aliases.isNil then + begin + for i:= 0 to librariesCount-1 do + begin + lib := libraryByIndex[i]; + if lib.enabled then + add(lib); end; + end + // else get selected libs + else for i := 0 to aliases.Count-1 do + begin + lib := libraryByAlias[aliases[i]]; + if lib.isNotNil and lib.enabled then + add(lib); end; end; -procedure TLibraryManager.getLibSources(aliases, list: TStrings); +procedure TLibraryManager.getLibSourcePath(aliases, list: TStrings); var - itm: TLibraryItem; + lib: TLibraryItem; i: Integer; begin - for i := 0 to fCol.Count-1 do + // no selector = all libs + if aliases.isNil then begin - itm := TLibraryItem(fCol.Items[i]); - if (not itm.enabled) or - (aliases.isNotNil and (aliases.IndexOf(itm.libAlias) = -1)) then - continue; - // - if list.IndexOf(itm.libSourcePath) <> -1 then - continue; - if not itm.libSourcePath.dirExists then - continue; - list.Add('-I' + itm.libSourcePath); + for i:= 0 to librariesCount-1 do + begin + lib := libraryByIndex[i]; + if lib.enabled and lib.hasValidLibSourcePath then + list.Add('-I' + lib.libSourcePath); + end; + end + // else get selected libs + else + begin + for i := 0 to aliases.Count-1 do + begin + lib := libraryByAlias[aliases[i]]; + if lib.isNotNil and lib.enabled and lib.hasValidLibSourcePath then + list.Add('-I' + lib.libSourcePath); + end; end; end; @@ -435,30 +552,45 @@ var imp: TStringList; i,j: integer; itm: TLibraryItem; + dep: TLibraryItem; + sel: TLibraryList; begin tks := TLexTokenList.Create; imp := TStringList.Create; + sel := TLibraryList.Create; try lex(source.Text, tks, nil, [lxoNoComments]); getImports(tks, imp); - for i := 0 to fCol.Count-1 do + for i:= 0 to imp.Count-1 do begin - itm := items[i]; - for j := imp.Count-1 downto 0 do if itm.hasModule(imp[j]) then + // get library for import I + itm := libraryByImport[imp[i]]; + if itm.isNotNil then begin - imp.Delete(j); - if (libs.IndexOf(itm.libFile) <> -1) then - continue; - libs.Add(itm.libFile); - paths.Add('-I'+itm.libSourcePath); - if itm.dependencies.Count > 0 then + sel.insert(itm); + // get libraries for import I dependencies + for j:= 0 to itm.dependencies.Count-1 do begin - getLibFiles(itm.dependencies, libs); - getLibSources(itm.dependencies, paths); + dep := libraryByAlias[itm.dependencies[j]]; + if dep.isNotNil then + sel.insert(dep); end; end; end; + // add the library files and the import paths for the selection + if not sel.IsEmpty then with sel.Iterator do + begin + while true do if data.hasValidLibFile then + begin + libs.Add(Data.libFile); + paths.Add('-I' + Data.libSourcePath); + if not Next then + break; + end; + free; + end; finally + sel.Free; tks.Free; imp.Free; end; @@ -466,28 +598,33 @@ end; procedure TLibraryManager.updateCrossDependencies; var - i, j, k, m: integer; + i, j, m: integer; + lib: TLibraryItem; + dep: TLibraryItem; + imp: string; begin - for i := 0 to fCol.Count-1 do + for i := 0 to fCollection.Count-1 do begin - if (items[i].libAlias = 'phobos') or (items[i].libAlias = 'druntime') then + lib := libraryByIndex[i]; + if (lib.libAlias = 'phobos') or (lib.libAlias = 'druntime') then continue; - items[i].dependencies.Clear; - for j := 0 to items[i].modules.Count-1 do - for m := 0 to items[i].moduleInfos[j].imports.Count-1 do + lib.dependencies.Clear; + for j := 0 to lib.modules.Count-1 do + for m := 0 to lib.moduleByIndex[j].imports.Count-1 do begin - for k := 0 to fCol.Count-1 do - begin - if k = i then - continue; - if (items[k].libAlias = 'phobos') or (items[k].libAlias = 'druntime') then - continue; - if items[k].hasModule(items[i].moduleInfos[j].imports[m]) then - begin - items[i].dependencies.Add(items[k].libAlias); - break; - end; - end; + imp := lib.moduleByIndex[j].imports[m]; + // module of the same package so... + if lib.hasModule(imp) then + continue; + dep := libraryByImport[imp]; + // ... this should not happen + if dep = lib then + continue; + // core or std are always handled by sc.ini + if dep.isNil or (dep.libAlias = 'phobos') or (dep.libAlias = 'druntime') then + continue; + // add deps + libraryByIndex[i].dependencies.Add(dep.libAlias); end; end; end; diff --git a/src/ce_main.pas b/src/ce_main.pas index c98e7695..850aea4b 100644 --- a/src/ce_main.pas +++ b/src/ce_main.pas @@ -2288,7 +2288,7 @@ begin else begin LibMan.getLibFiles(nil, dmdproc.Parameters); - LibMan.getLibSources(nil, dmdproc.Parameters); + LibMan.getLibSourcePath(nil, dmdproc.Parameters); end; deleteDups(dmdproc.Parameters); diff --git a/src/ce_nativeproject.pas b/src/ce_nativeproject.pas index c9eeaa9d..f40806cd 100644 --- a/src/ce_nativeproject.pas +++ b/src/ce_nativeproject.pas @@ -341,27 +341,13 @@ begin end; procedure TCENativeProject.doChanged; -{$IFDEF DEBUG} -var - lst: TStringList; -{$ENDIF} begin fModified := true; updateOutFilename; getBaseConfig; subjProjChanged(fProjectSubject, self); - if assigned(fOnChange) then fOnChange(Self); - {$IFDEF DEBUG} - //lst := TStringList.Create; - //try - // lst.Add('---------begin----------'); - // getOpts(lst); - // lst.Add('---------end-----------'); - // DebugLn(lst.Text); - //finally - // lst.Free; - //end; - {$ENDIF} + if assigned(fOnChange) then + fOnChange(Self); end; function TCENativeProject.getConfig(const ix: integer): TCompilerConfiguration; @@ -468,7 +454,7 @@ begin LibMan.getLibFiles(libAliasesPtr, aList); // but always adds -I - LibMan.getLibSources(libAliasesPtr, aList); + LibMan.getLibSourcePath(libAliasesPtr, aList); // config if currentConfiguration.isOverriddenConfiguration then currentConfiguration.getOpts(aList, fBaseConfig)