forks
/
dexed
mirror of https://gitlab.com/basile.b/dexed.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

libman, use hashset and hash map to speed up the selection of the items, close #77

This commit is contained in:
Basile Burg 2016-07-04 03:12:12 +02:00
parent 94eddd0ca8
commit e1846a9466
4 changed files with 304 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/ce_common.pas
View File

@ -15,7 +15,7 @@ uses
{$IFNDEF CEBUILD}
forms,
{$ENDIF}
process, asyncprocess;
process, asyncprocess, ghashmap, ghashset;
const
exeExt = {$IFDEF WINDOWS} '.exe' {$ELSE} '' {$ENDIF};
@ -31,6 +31,21 @@ type
TCECompiler = (dmd, gdc, ldc);
// function used as hash in gXXX sets & maps
TStringHash = class
class function hash(const key: string; maxBucketsPow2: longint): longint;
end;
// HashMap for TValue by string
generic TStringHashMap<TValue> = class(specialize THashmap<string, TValue, TStringHash>);
// function used as objects in gXXX sets & maps
TObjectHash = class
class function hash(key: TObject; maxBucketsPow2: longint): longint;
end;
generic TObjectHashSet<TValue> = class(specialize THashSet<TValue, TObjectHash>);
// aliased to get a custom prop inspector
TCEPathname = type string;
TCEFilename = type string;
@ -280,6 +295,32 @@ implementation
uses
ce_main;
class function TStringHash.hash(const key: string; maxBucketsPow2: longint): longint;
var
i: integer;
begin
{$PUSH}{$R-}
result := 5381;
for i:= 1 to key.length do
begin
result := ((result shl 5) + result) + Byte(key[i]);
end;
result := result and (maxBucketsPow2-1);
{$POP}
end;
class function TObjectHash.hash(key: TObject; maxBucketsPow2: longint): longint;
begin
{$PUSH}{$R-}{$WARNINGS OFF}{$HINTS OFF}
{$IFDEF CPU32}
Result := longint(Pointer(key)) and (maxBucketsPow2 -1);
{$ELSE}
Result := (longInt(Pointer(key)) xor PlongInt(PInteger(&key) + 4)^) and (maxBucketsPow2 -1);
{$ENDIF}
{$POP}
end;
procedure TCEPersistentShortcut.assign(aValue: TPersistent);
var
src: TCEPersistentShortcut;

379
src/ce_libman.pas
View File

@ -6,17 +6,17 @@ interface
uses
Classes, SysUtils, FileUtil, ce_common, ce_writableComponent, ce_dcd, LazFileUtils,
ce_dialogs, ce_projutils, ce_interfaces, ce_dlang;
ce_dialogs, ce_projutils, ce_interfaces, ce_dlang, ghashmap, ghashset;
type
// TODO-clibman: improve import analysis, caching, hashmap, backup, update.
// TODO-clibman: improve import analysis, update (currently always full).
(**
* Information for a module in a library manager entry
*)
TModuleInfo = class(TCollectionItem)
private
strict private
fName: string;
fImports: TStringList;
procedure setImports(value: TStringList);
@ -32,73 +32,100 @@ type
(**
* Represents a D static library. In a project libAlias allows to
* resolve automatically the dependencies of a project.
* Represents a D static library.
*)
TLibraryItem = class(TCollectionItem)
private
type
TModulesByName = specialize TStringHashMap<TModuleInfo>;
strict private
fAlias: string;
fSourcePath: string;
fLibSourcePath: string;
fLibFile: string;
fProjFile: string;
fLibProject: string;
fEnabled: boolean;
fDependencies: TStringList;
fModules: TCollection;
fModulesByName: TModulesByName;
fHasValidSourcePath: boolean;
fHasValidLibFile: boolean;
fHasValidLibProject: boolean;
procedure setDependencies(value: TStringList);
procedure setModules(value: TCollection);
function getModuleInfo(ix: integer): TModuleInfo;
procedure setLibProject(const value: string);
procedure setLibFile(const value: string);
procedure setLibSourcePath(const value: string);
function getModule(value: integer): TModuleInfo;
function getModule(const value: string): TModuleInfo;
function moduleCount: integer;
published
property libAlias: string read fAlias write fAlias;
property libSourcePath: string read fSourcePath write fSourcePath;
property libFile: string read fLibFile write fLibFile;
property projectFile: string read fProjFile write fProjFile;
property libSourcePath: string read fLibSourcePath write setLibSourcePath;
property libFile: string read fLibFile write setLibFile;
property libProject: string read fLibProject write setLibProject;
property enabled: boolean read fEnabled write fEnabled default true;
// TODO-clibman: dont forget that these props are not written
property dependencies: TStringList read fDependencies write setDependencies stored false;
property modules: TCollection read fModules write setModules stored false;
// TODO-cmaintenace: remove this property from 3 update 1
property projectFile: string read fLibProject write fLibProject stored false;
public
constructor Create(ACollection: TCollection); override;
destructor Destroy; override;
procedure updateModulesInfo;
function addModuleInfo: TModuleInfo;
function hasModule(const value: string): boolean;
property moduleInfos[ix: integer]: TModuleInfo read getModuleInfo;
property hasValidLibFile: boolean read fHasValidLibFile;
property hasValidLibProject: boolean read fHasValidLibProject;
property hasValidLibSourcePath: boolean read fHasValidSourcePath;
property moduleByIndex[value: integer]: TModuleInfo read getModule;
end;
TLibraryList = specialize TObjectHashSet<TLibraryItem>;
(**
* Represents all the D libraries present on this system.
* Represents all the D libraries handled by Coedit.
*)
TLibraryManager = class(TWritableLfmTextComponent)
private
fCol: TCollection;
function getItems(index: integer): TLibraryItem;
procedure setCol(value: TCollection);
type
TItemsByAlias = specialize TStringHashMap<TLibraryItem>;
strict private
fCollection: TCollection;
fItemsByAlias: TItemsByAlias;
function getLibraryByIndex(index: integer): TLibraryItem;
function getLibraryByAlias(const value: string): TLibraryItem;
function getLibraryByImport(const value: string): TLibraryItem;
procedure setCollection(value: TCollection);
procedure updateItemsByAlias;
function getLibrariesCount: integer;
published
property libraries: TCollection read fCol write setCol;
property libraries: TCollection read fCollection write setCollection;
public
constructor create(aOwner: TComponent); override;
destructor destroy; override;
(**
* The caller gets all the static library files in list if aliases is nil
* otherwise the static library files selected by the aliases.
* The caller gets all the static library files in "list" if "aliases" is nil
* otherwise the static library files selected by "aliases".
*)
procedure getLibFiles(aliases, list: TStrings);
(**
* The caller gets all the paths were are located the library sources in list
* if aliases is nil otherwise the paths where are located the sources of the
* livraries selected by aliases.
* The caller gets all the paths were are located the library sources in "list"
* if "aliases" is nil otherwise the paths where are located the sources of the
* libraries selected by "aliases".
*)
procedure getLibSources(aliases, list: TStrings);
procedure getLibSourcePath(aliases, list: TStrings);
(**
* The caller gets all the static library files and the source path
* that are required by the specified source code.
* The caller gets static libraries files in "libs" and source paths
* in "paths", as required by the specified "source" code.
*)
procedure getLibsForSource(source, libs, paths: TStrings);
//
procedure updateDCD;
// find the aliases of the libraries used by this library.
procedure updateCrossDependencies;
property items[index: integer]: TLibraryItem read getItems; default;
property librariesCount: integer read getLibrariesCount;
property libraryByIndex[value: integer]: TLibraryItem read getLibraryByIndex;
property libraryByAlias[const value: string]: TLibraryItem read getLibraryByAlias;
property libraryByImport[const value: string]: TLibraryItem read getLibraryByImport;
end;
const
@ -130,20 +157,32 @@ constructor TLibraryItem.Create(ACollection: TCollection);
begin
inherited create(ACollection);
fModules := TCollection.Create(TModuleInfo);
fModulesByName := TModulesByName.create;
fDependencies := TStringList.Create;
fEnabled:=true;
end;
destructor TLibraryItem.Destroy;
begin
fModulesByName.Free;
fDependencies.Free;
fModules.Free;
inherited;
end;
function TLibraryItem.getModuleInfo(ix: integer): TModuleInfo;
function TLibraryItem.moduleCount: integer;
begin
exit(TModuleInfo(fModules.Items[ix]));
exit(fModules.Count);
end;
function TLibraryItem.getModule(value: integer): TModuleInfo;
begin
exit(TModuleInfo(fModules.Items[value]));
end;
function TLibraryItem.getModule(const value: string): TModuleInfo;
begin
exit(fModulesByName.GetData(value));
end;
function TLibraryItem.addModuleInfo: TModuleInfo;
@ -152,13 +191,8 @@ begin
end;
function TLibraryItem.hasModule(const value: string): boolean;
var
i: integer;
begin
for i:= 0 to fModules.Count-1 do
if getModuleInfo(i).name = value then
exit(true);
exit(false);
exit(fModulesByName.contains(value));
end;
procedure TLibraryItem.setModules(value: TCollection);
@ -171,6 +205,30 @@ begin
fDependencies.Assign(value);
end;
procedure TLibraryItem.setLibProject(const value: string);
begin
if fLibProject = value then
exit;
fLibProject:=value;
fHasValidLibProject:=value.fileExists;
end;
procedure TLibraryItem.setLibFile(const value: string);
begin
if fLibFile = value then
exit;
fLibFile:=value;
fHasValidLibFile:=value.fileExists or value.dirExists;
end;
procedure TLibraryItem.setLibSourcePath(const value: string);
begin
if fLibSourcePath = value then
exit;
fLibSourcePath:=value;
fHasValidSourcePath:=value.dirExists;
end;
procedure TLibraryItem.updateModulesInfo;
var
prj: ICECommonProject;
@ -182,9 +240,11 @@ var
i: integer;
begin
fModules.Clear;
if fProjFile.fileExists then
fModulesByName.Free;
fModulesByName := TModulesByName.create;
if hasValidLibProject then
begin
prj := loadProject(fProjFile, true);
prj := loadProject(fLibProject, true);
tks := TLexTokenList.Create;
str := TStringList.Create;
try
@ -198,6 +258,7 @@ begin
lex(str.Text, tks, nil, [lxoNoComments]);
mdi := addModuleInfo;
mdi.name := getModuleName(tks);
fModulesByName.insert(mdi.name, mdi);
getImports(tks, mdi.imports);
tks.Clear;
end;
@ -206,13 +267,13 @@ begin
str.Free;
prj.getProject.Free;
end;
end else if fSourcePath.dirExists then
end else if hasValidLibSourcePath then
begin
lst := TStringList.Create;
str := TStringList.Create;
tks := TLexTokenList.Create;
try
listFiles(lst, fSourcePath, true);
listFiles(lst, fLibSourcePath, true);
for i := 0 to lst.Count-1 do
begin
fle := lst[i];
@ -222,6 +283,7 @@ begin
lex(str.Text, tks, nil, [lxoNoComments]);
mdi := addModuleInfo;
mdi.name := getModuleName(tks);
fModulesByName.insert(mdi.name, mdi);
getImports(tks, mdi.imports);
tks.Clear;
end;
@ -242,11 +304,12 @@ var
i: integer;
begin
inherited;
fCol := TCollection.Create(TLibraryItem);
fItemsByAlias:= TItemsByAlias.create;
fCollection := TCollection.Create(TLibraryItem);
fname := getCoeditDocPath + libFname;
if fname.fileExists then
loadFromFile(fname);
if fCol.Count = 0 then
if fCollection.Count = 0 then
begin
{$IFDEF WINDOWS}
fDmdPath := ExeSearch('dmd.exe');
@ -274,16 +337,16 @@ begin
// add phobos
if '/usr/include/dmd/phobos'.dirExists then
begin
with TLibraryItem(fCol.Add) do begin
with TLibraryItem(fCollection.Add) do begin
libAlias := 'phobos';
libFile := '';
libSourcePath := '/usr/include/dmd/phobos';
end;
end;
// add druntime (no items - only for DCD)
// add druntime (no libraryByIndex - only for DCD)
if '/usr/include/dmd/druntime/import'.dirExists then
begin
with TLibraryItem(fCol.Add) do begin
with TLibraryItem(fCollection.Add) do begin
libAlias := 'druntime';
libFile := '';
libSourcePath := '/usr/include/dmd/druntime/import';
@ -310,7 +373,8 @@ begin
end;
{$ENDIF}
end;
if (fCol.Count = 0) and not (getCoeditDocPath + libFname).fileExists then
updateItemsByAlias;
if (fCollection.Count = 0) and not (getCoeditDocPath + libFname).fileExists then
begin
dlgOkInfo(
'Coedit failed to add "druntime" and "phobos" to the library manager.'
@ -319,10 +383,10 @@ begin
end;
updateDCD;
//
for i := 0 to fCol.Count-1 do
for i := 0 to fCollection.Count-1 do
begin
if (items[i].libAlias <> 'phobos') and (items[i].libAlias <> 'druntime') then
items[i].updateModulesInfo;
if (libraryByIndex[i].libAlias <> 'phobos') and (libraryByIndex[i].libAlias <> 'druntime') then
libraryByIndex[i].updateModulesInfo;
end;
updateCrossDependencies;
end;
@ -331,18 +395,53 @@ destructor TLibraryManager.destroy;
begin
ForceDirectoriesUTF8(getCoeditDocPath);
LibMan.saveToFile(getCoeditDocPath + libFname);
fCol.Free;
fItemsByAlias.Free;
fCollection.Free;
inherited;
end;
procedure TLibraryManager.setCol(value: TCollection);
procedure TLibraryManager.updateItemsByAlias;
var
i: integer;
begin
fCol.assign(value);
fItemsByAlias.Free;
fItemsByAlias := TItemsByAlias.create;
for i:= 0 to fCollection.Count-1 do
fItemsByAlias.insert(libraryByIndex[i].libAlias, libraryByIndex[i]);
end;
function TLibraryManager.getItems(index: integer): TLibraryItem;
procedure TLibraryManager.setCollection(value: TCollection);
begin
exit(TLibraryItem(fCol.Items[index]));
fCollection.assign(value);
end;
function TLibraryManager.getLibraryByIndex(index: integer): TLibraryItem;
begin
exit(TLibraryItem(fCollection.Items[index]));
end;
function TLibraryManager.getLibraryByAlias(const value: string): TLibraryItem;
begin
exit(fItemsByAlias.GetData(value));
end;
function TLibraryManager.getLibraryByImport(const value: string): TLibraryItem;
var
i: integer;
s: TLibraryItem;
begin
result := nil;
for i := 0 to librariesCount-1 do
begin
s := libraryByIndex[i];
if s.hasModule(value) then
exit(s);
end;
end;
function TLibraryManager.getLibrariesCount: integer;
begin
exit(fCollection.Count);
end;
procedure TLibraryManager.updateDCD;
@ -352,12 +451,12 @@ var
i: Integer;
begin
if not DcdWrapper.available then exit;
// note: new items are directly handled but removed ones still in cache until server restarts.
// note: new libraryByIndex are directly handled but removed ones still in cache until server restarts.
str := TStringList.Create;
try
for i := 0 to fCol.Count-1 do
for i := 0 to fCollection.Count-1 do
begin
itm := TLibraryItem(fCol.Items[i]);
itm := TLibraryItem(fCollection.Items[i]);
if itm.enabled then
str.Add(itm.libSourcePath);
end;
@ -368,32 +467,20 @@ begin
end;
procedure TLibraryManager.getLibFiles(aliases, list: TStrings);
var
itm: TLibraryItem;
lst: TStringList;
i,j: Integer;
dir: string;
begin
for i := 0 to fCol.Count-1 do
procedure add(lib: TLibraryItem);
var
j: integer;
dir: string;
lst: TstringList;
begin
itm := TLibraryItem(fCol.Items[i]);
if (not itm.enabled) or
(aliases.isNotNil and (aliases.IndexOf(itm.libAlias) = -1)) then
continue;
// single items files
if fileExists(itm.libFile) then
begin
if list.IndexOf(itm.libFile) <> -1 then
continue;
list.Add(itm.libFile);
end
// folder of items file
else if itm.libFile.dirExists then
// as a trick a folder can be set as source, this allows to pass
// multiple sources in an automated way.
if lib.libFile.dirExists then
begin
lst := TStringList.Create;
try
dir := itm.libFile;
if itm.libFile[dir.length] = DirectorySeparator then
dir := lib.libFile;
if lib.libFile[dir.length] = DirectorySeparator then
dir := dir[1..dir.length-1];
listFiles(lst, dir);
for j:= 0 to lst.Count-1 do
@ -405,27 +492,57 @@ begin
finally
lst.Free;
end;
end
else if lib.hasValidLibFile then
list.Add(lib.libFile);
end;
var
lib: TLibraryItem;
i: Integer;
begin
// no selector = all libs
if aliases.isNil then
begin
for i:= 0 to librariesCount-1 do
begin
lib := libraryByIndex[i];
if lib.enabled then
add(lib);
end;
end
// else get selected libs
else for i := 0 to aliases.Count-1 do
begin
lib := libraryByAlias[aliases[i]];
if lib.isNotNil and lib.enabled then
add(lib);
end;
end;
procedure TLibraryManager.getLibSources(aliases, list: TStrings);
procedure TLibraryManager.getLibSourcePath(aliases, list: TStrings);
var
itm: TLibraryItem;
lib: TLibraryItem;
i: Integer;
begin
for i := 0 to fCol.Count-1 do
// no selector = all libs
if aliases.isNil then
begin
itm := TLibraryItem(fCol.Items[i]);
if (not itm.enabled) or
(aliases.isNotNil and (aliases.IndexOf(itm.libAlias) = -1)) then
continue;
//
if list.IndexOf(itm.libSourcePath) <> -1 then
continue;
if not itm.libSourcePath.dirExists then
continue;
list.Add('-I' + itm.libSourcePath);
for i:= 0 to librariesCount-1 do
begin
lib := libraryByIndex[i];
if lib.enabled and lib.hasValidLibSourcePath then
list.Add('-I' + lib.libSourcePath);
end;
end
// else get selected libs
else
begin
for i := 0 to aliases.Count-1 do
begin
lib := libraryByAlias[aliases[i]];
if lib.isNotNil and lib.enabled and lib.hasValidLibSourcePath then
list.Add('-I' + lib.libSourcePath);
end;
end;
end;
@ -435,30 +552,45 @@ var
imp: TStringList;
i,j: integer;
itm: TLibraryItem;
dep: TLibraryItem;
sel: TLibraryList;
begin
tks := TLexTokenList.Create;
imp := TStringList.Create;
sel := TLibraryList.Create;
try
lex(source.Text, tks, nil, [lxoNoComments]);
getImports(tks, imp);
for i := 0 to fCol.Count-1 do
for i:= 0 to imp.Count-1 do
begin
itm := items[i];
for j := imp.Count-1 downto 0 do if itm.hasModule(imp[j]) then
// get library for import I
itm := libraryByImport[imp[i]];
if itm.isNotNil then
begin
imp.Delete(j);
if (libs.IndexOf(itm.libFile) <> -1) then
continue;
libs.Add(itm.libFile);
paths.Add('-I'+itm.libSourcePath);
if itm.dependencies.Count > 0 then
sel.insert(itm);
// get libraries for import I dependencies
for j:= 0 to itm.dependencies.Count-1 do
begin
getLibFiles(itm.dependencies, libs);
getLibSources(itm.dependencies, paths);
dep := libraryByAlias[itm.dependencies[j]];
if dep.isNotNil then
sel.insert(dep);
end;
end;
end;
// add the library files and the import paths for the selection
if not sel.IsEmpty then with sel.Iterator do
begin
while true do if data.hasValidLibFile then
begin
libs.Add(Data.libFile);
paths.Add('-I' + Data.libSourcePath);
if not Next then
break;
end;
free;
end;
finally
sel.Free;
tks.Free;
imp.Free;
end;
@ -466,28 +598,33 @@ end;
procedure TLibraryManager.updateCrossDependencies;
var
i, j, k, m: integer;
i, j, m: integer;
lib: TLibraryItem;
dep: TLibraryItem;
imp: string;
begin
for i := 0 to fCol.Count-1 do
for i := 0 to fCollection.Count-1 do
begin
if (items[i].libAlias = 'phobos') or (items[i].libAlias = 'druntime') then
lib := libraryByIndex[i];
if (lib.libAlias = 'phobos') or (lib.libAlias = 'druntime') then
continue;
items[i].dependencies.Clear;
for j := 0 to items[i].modules.Count-1 do
for m := 0 to items[i].moduleInfos[j].imports.Count-1 do
lib.dependencies.Clear;
for j := 0 to lib.modules.Count-1 do
for m := 0 to lib.moduleByIndex[j].imports.Count-1 do
begin
for k := 0 to fCol.Count-1 do
begin
if k = i then
continue;
if (items[k].libAlias = 'phobos') or (items[k].libAlias = 'druntime') then
continue;
if items[k].hasModule(items[i].moduleInfos[j].imports[m]) then
begin
items[i].dependencies.Add(items[k].libAlias);
break;
end;
end;
imp := lib.moduleByIndex[j].imports[m];
// module of the same package so...
if lib.hasModule(imp) then
continue;
dep := libraryByImport[imp];
// ... this should not happen
if dep = lib then
continue;
// core or std are always handled by sc.ini
if dep.isNil or (dep.libAlias = 'phobos') or (dep.libAlias = 'druntime') then
continue;
// add deps
libraryByIndex[i].dependencies.Add(dep.libAlias);
end;
end;
end;

2
src/ce_main.pas
View File

@ -2288,7 +2288,7 @@ begin
else
begin
LibMan.getLibFiles(nil, dmdproc.Parameters);
LibMan.getLibSources(nil, dmdproc.Parameters);
LibMan.getLibSourcePath(nil, dmdproc.Parameters);
end;
deleteDups(dmdproc.Parameters);

20
src/ce_nativeproject.pas
View File

@ -341,27 +341,13 @@ begin
end;
procedure TCENativeProject.doChanged;
{$IFDEF DEBUG}
var
lst: TStringList;
{$ENDIF}
begin
fModified := true;
updateOutFilename;
getBaseConfig;
subjProjChanged(fProjectSubject, self);
if assigned(fOnChange) then fOnChange(Self);
{$IFDEF DEBUG}
//lst := TStringList.Create;
//try
// lst.Add('---------begin----------');
// getOpts(lst);
// lst.Add('---------end-----------');
// DebugLn(lst.Text);
//finally
// lst.Free;
//end;
{$ENDIF}
if assigned(fOnChange) then
fOnChange(Self);
end;
function TCENativeProject.getConfig(const ix: integer): TCompilerConfiguration;
@ -468,7 +454,7 @@ begin
LibMan.getLibFiles(libAliasesPtr, aList);
// but always adds -I<path>
LibMan.getLibSources(libAliasesPtr, aList);
LibMan.getLibSourcePath(libAliasesPtr, aList);
// config
if currentConfiguration.isOverriddenConfiguration then
currentConfiguration.getOpts(aList, fBaseConfig)