use of com.python.pydev.analysis.additionalinfo.AbstractAdditionalDependencyInfo.IBufferFiller in project Pydev by fabioz.
the class ReferenceSearches method search.
@Override
public List<ModulesKey> search(IProject project, OrderedMap<String, Set<String>> fieldNameToValues, IProgressMonitor monitor) {
final List<ModulesKey> ret = new ArrayList<ModulesKey>();
AbstractAdditionalDependencyInfo abstractAdditionalDependencyInfo = this.abstractAdditionalDependencyInfo.get();
if (abstractAdditionalDependencyInfo == null) {
Log.log("AbstractAdditionalDependencyInfo already collected!");
return ret;
}
final NullProgressMonitor nullMonitor = new NullProgressMonitor();
Set<String> pythonPathFolders = abstractAdditionalDependencyInfo.getPythonPathFolders();
LinkedBlockingQueue<Command> queue = new LinkedBlockingQueue<>();
int searchers = Runtime.getRuntime().availableProcessors();
// The 'ret' should be filled with the module keys where the tokens are found.
final Object retLock = new Object();
// Create 2 consumers
Thread[] threads = new Thread[searchers];
for (int i = 0; i < searchers; i++) {
Searcher searcher = new Searcher(queue, fieldNameToValues.get(IReferenceSearches.FIELD_CONTENTS), ret, retLock);
// Spawn a thread to do the search while we load the contents.
Thread t = new Thread(searcher);
threads[i] = t;
t.start();
}
try {
PythonPathHelper pythonPathHelper = new PythonPathHelper();
pythonPathHelper.setPythonPath(new ArrayList<String>(pythonPathFolders));
ModulesFoundStructure modulesFound = pythonPathHelper.getModulesFoundStructure(project, nullMonitor);
int totalSteps = modulesFound.regularModules.size() + modulesFound.zipContents.size();
monitor.beginTask("Get modules with token in: " + abstractAdditionalDependencyInfo.getUIRepresentation(), totalSteps);
PyPublicTreeMap<ModulesKey, ModulesKey> keys = new PyPublicTreeMap<>();
// no point in searching dlls.
boolean includeOnlySourceModules = true;
ModulesManager.buildKeysForRegularEntries(nullMonitor, modulesFound, keys, includeOnlySourceModules);
// Get from regular files found
for (ModulesKey entry : keys.values()) {
if (entry instanceof ModulesKeyForFolder) {
continue;
}
if (monitor.isCanceled()) {
break;
}
if (AbstractAdditionalDependencyInfo.DEBUG) {
System.out.println("Loading: " + entry);
}
final File file = entry.file;
try {
queue.put(new Command(entry, new IBufferFiller() {
@Override
public void fillBuffer(FastStringBuffer bufFileContents) {
try (FileInputStream stream = new FileInputStream(file)) {
fill(bufFileContents, stream);
} catch (Exception e) {
Log.log(e);
}
}
}));
} catch (InterruptedException e) {
Log.log(e);
}
}
// Get from zip files found
List<ZipContents> allZipsZipContents = modulesFound.zipContents;
for (ZipContents zipContents : allZipsZipContents) {
keys.clear();
if (monitor.isCanceled()) {
break;
}
ModulesManager.buildKeysForZipContents(keys, zipContents);
try (ZipFile zipFile = new ZipFile(zipContents.zipFile)) {
for (ModulesKey entry : keys.values()) {
if (AbstractAdditionalDependencyInfo.DEBUG) {
System.out.println("Loading: " + entry);
}
if (monitor.isCanceled()) {
break;
}
final ModulesKeyForZip z = (ModulesKeyForZip) entry;
if (!z.isFile) {
continue;
}
queue.put(new Command(entry, new IBufferFiller() {
@Override
public void fillBuffer(FastStringBuffer bufFileContents) {
try (InputStream stream = zipFile.getInputStream(zipFile.getEntry(z.zipModulePath))) {
fill(bufFileContents, stream);
} catch (Exception e) {
Log.log(e);
}
}
}));
}
} catch (Exception e) {
Log.log(e);
}
}
} finally {
for (int i = 0; i < searchers; i++) {
// add it to wait for the thread to finish.
queue.add(new Command());
}
}
int j = 0;
while (true) {
j++;
boolean liveFound = false;
for (Thread t : threads) {
if (t.isAlive()) {
liveFound = true;
break;
}
}
if (liveFound) {
if (j % 50 == 0) {
monitor.setTaskName("Searching references...");
monitor.worked(1);
}
Thread.yield();
} else {
break;
}
}
return ret;
}
Aggregations