use of org.python.pydev.core.ModulesKeyForZip in project Pydev by fabioz.
the class ReferenceSearchesLucene method internalSearch.
private synchronized List<ModulesKey> internalSearch(IProject project, final OrderedMap<String, Set<String>> fieldNameToValues, IProgressMonitor monitor) throws OperationCanceledException {
final List<ModulesKey> ret = new ArrayList<ModulesKey>();
PythonNature nature = PythonNature.getPythonNature(project);
if (nature == null) {
Log.log("Project :" + project + " does not have Python nature configured.");
return ret;
}
// Make sure that its information is synchronized.
AbstractAdditionalDependencyInfo abstractAdditionalDependencyInfo = this.abstractAdditionalDependencyInfo.get();
if (abstractAdditionalDependencyInfo == null) {
Log.log("AbstractAdditionalDependencyInfo already collected!");
return ret;
}
Long lastMtime = projectToLastMtime.get(project);
if (lastMtime == null) {
lastMtime = 0L;
}
long currMtime = nature.getMtime();
if (lastMtime != currMtime) {
projectToLastMtime.put(project, currMtime);
Timer timer = null;
if (DEBUG) {
System.out.println("Curr mtime: " + currMtime + " last time: " + lastMtime);
System.out.println("Start sync: " + project);
timer = new Timer();
}
new InterpreterInfoBuilder().syncInfoToPythonPath(monitor, nature);
if (DEBUG) {
timer.printDiff("Sync time");
}
}
boolean mustCommitChange = false;
final String name = "Search modules with token in: " + abstractAdditionalDependencyInfo.getUIRepresentation();
monitor.beginTask(name, 7);
monitor.setTaskName(name);
DiskCache completeIndex = abstractAdditionalDependencyInfo.completeIndex;
// Note: we should be able to deal with entries already deleted!
boolean applyAllDeletes = false;
String folderToPersist = completeIndex.getFolderToPersist();
Object indexApiLock;
File indexDir = new File(folderToPersist, "lc" + IndexApi.luceneSuffix);
synchronized (lock) {
indexApiLock = indexDirToLock.get(indexDir);
if (indexApiLock == null) {
try {
indexApiLock = new Object();
indexDirToLock.put(indexDir, indexApiLock);
} catch (Exception e) {
Log.log(e);
return ret;
}
}
}
IndexApi indexApi = null;
try {
synchronized (indexApiLock) {
indexApi = new IndexApi(indexDir, applyAllDeletes);
// Key to CompleteIndexKey (has modified time).
final Map<ModulesKey, CompleteIndexKey> indexMap = new HashMap<>();
IDocumentsVisitor visitor = new IDocumentsVisitor() {
@Override
public void visit(DocumentInfo documentInfo) {
ModulesKey keyFromIO = ModulesKey.fromIO(documentInfo.get(FIELD_MODULES_KEY_IO));
String modifiedTime = documentInfo.get(FIELD_MODIFIED_TIME);
indexMap.put(keyFromIO, new CompleteIndexKey(keyFromIO, Long.parseLong(modifiedTime)));
}
};
try {
indexApi.visitAllDocs(visitor, FIELD_MODULES_KEY_IO, FIELD_MODIFIED_TIME);
} catch (IOException e) {
Log.log(e);
}
incrementAndCheckProgress("Visited current index", monitor);
Set<CompleteIndexKey> docsToRemove = new HashSet<>();
Set<CompleteIndexKey> modulesToAdd = new HashSet<>();
Map<File, Set<CompleteIndexKey>> zipModulesToAdd = new HashMap<>();
// Wait for the integrity check before getting the keys!
abstractAdditionalDependencyInfo.waitForIntegrityCheck();
final Map<CompleteIndexKey, CompleteIndexKey> currentKeys = completeIndex.keys();
// from the modules (or have a different time).
for (Entry<ModulesKey, CompleteIndexKey> entryInIndex : indexMap.entrySet()) {
CompleteIndexKey indexModule = entryInIndex.getValue();
CompleteIndexKey currentModule = currentKeys.get(indexModule);
if (currentModule == null || currentModule.key == null || currentModule.key.file == null) {
docsToRemove.add(indexModule);
} else {
// exists, but we also need to check the modified time
boolean changed = currentModule.lastModified != indexModule.lastModified;
if (!changed) {
ModulesKey keyCurrentModule = currentModule.key;
ModulesKey keyIndexModule = indexModule.key;
boolean currentIsZip = keyCurrentModule instanceof ModulesKeyForZip;
boolean indexIsZip = keyIndexModule instanceof ModulesKeyForZip;
changed = currentIsZip != indexIsZip;
if (!changed) {
changed = !currentModule.key.file.equals(indexModule.key.file);
}
}
if (changed) {
// remove and add
docsToRemove.add(indexModule);
add(modulesToAdd, zipModulesToAdd, currentModule);
}
}
}
// --- Progress
incrementAndCheckProgress("Updating for removal", monitor);
// Step 2: add new entries in current and not in the index
for (Entry<CompleteIndexKey, CompleteIndexKey> currentEntry : currentKeys.entrySet()) {
CompleteIndexKey completeIndexKey = currentEntry.getValue();
if (!indexMap.containsKey(completeIndexKey.key)) {
ModulesKey modulesKey = completeIndexKey.key;
if (modulesKey instanceof IModulesKeyForJava || modulesKey.file == null || !modulesKey.file.isFile()) {
// ignore this one (we can't do anything with it).
continue;
}
if (modulesKey instanceof ModulesKeyForZip) {
ModulesKeyForZip modulesKeyForZip = (ModulesKeyForZip) modulesKey;
if (!modulesKeyForZip.isFile) {
// Ignore folders in zips (happens for jython folders which may not have an __init__.py)
continue;
}
}
add(modulesToAdd, zipModulesToAdd, completeIndexKey);
}
}
// --- Progress
incrementAndCheckProgress("Updating for addition", monitor);
Map<String, Collection<String>> fieldToValuesToRemove = new HashMap<>();
Collection<String> lstToRemove = new ArrayList<>(docsToRemove.size());
FastStringBuffer tempBuf = new FastStringBuffer();
for (Iterator<CompleteIndexKey> it = docsToRemove.iterator(); it.hasNext(); ) {
it.next().key.toIO(tempBuf.clear());
lstToRemove.add(tempBuf.toString());
}
incrementAndCheckProgress("Removing outdated entries", monitor);
if (lstToRemove.size() > 0) {
fieldToValuesToRemove.put(FIELD_MODULES_KEY_IO, lstToRemove);
try {
mustCommitChange = true;
if (DEBUG) {
System.out.println("Removing: " + fieldToValuesToRemove);
}
indexApi.removeDocs(fieldToValuesToRemove);
} catch (IOException e) {
Log.log(e);
}
}
incrementAndCheckProgress("Indexing new entries", monitor);
if (modulesToAdd.size() > 0) {
mustCommitChange = true;
for (CompleteIndexKey key : modulesToAdd) {
File f = key.key.file;
if (f.isFile()) {
if (DEBUG) {
System.out.println("Indexing: " + f);
}
try (BufferedReader reader = new BufferedReader(new FileReader(f))) {
indexApi.index(createFieldsToIndex(key, tempBuf), reader, FIELD_CONTENTS);
} catch (Exception e) {
Log.log(e);
}
}
}
}
Set<Entry<File, Set<CompleteIndexKey>>> entrySet = zipModulesToAdd.entrySet();
for (Entry<File, Set<CompleteIndexKey>> entry : entrySet) {
File f = entry.getKey();
if (f.exists()) {
try (ZipFile zipFile = new ZipFile(f, ZipFile.OPEN_READ)) {
Set<CompleteIndexKey> value = entry.getValue();
for (CompleteIndexKey completeIndexKey2 : value) {
ModulesKeyForZip forZip = (ModulesKeyForZip) completeIndexKey2.key;
try (InputStream inputStream = zipFile.getInputStream(zipFile.getEntry(forZip.zipModulePath))) {
InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
mustCommitChange = true;
if (DEBUG) {
System.out.println("Indexing: " + completeIndexKey2);
}
indexApi.index(createFieldsToIndex(completeIndexKey2, tempBuf), reader, FIELD_CONTENTS);
}
}
} catch (Exception e) {
Log.log(e);
}
}
}
incrementAndCheckProgress("Committing result", monitor);
if (mustCommitChange) {
if (DEBUG) {
System.out.println("Commit result");
}
try {
indexApi.commit();
} catch (IOException e) {
Log.log(e);
}
}
// Ok, things should be in-place at this point... let's actually do the search now
incrementAndCheckProgress("Searching index", monitor);
try {
if (DEBUG) {
System.out.println("Searching: " + fieldNameToValues);
}
visitor = new IDocumentsVisitor() {
@Override
public void visit(DocumentInfo documentInfo) {
try {
String modKey = documentInfo.get(FIELD_MODULES_KEY_IO);
String modTime = documentInfo.get(FIELD_MODIFIED_TIME);
if (modKey != null && modTime != null) {
ModulesKey fromIO = ModulesKey.fromIO(modKey);
CompleteIndexKey existing = currentKeys.get(new CompleteIndexKey(fromIO));
// Deal with deleted entries still hanging around.
if (existing != null && existing.lastModified == Long.parseLong(modTime)) {
// Ok, we have a match!
ret.add(existing.key);
}
}
} catch (Exception e) {
Log.log(e);
}
}
};
indexApi.searchWildcard(fieldNameToValues, applyAllDeletes, visitor, null, FIELD_MODULES_KEY_IO, FIELD_MODIFIED_TIME);
} catch (Exception e) {
Log.log(e);
}
}
} catch (Exception e) {
Log.log(e);
} finally {
if (indexApi != null) {
indexApi.dispose();
}
}
return ret;
}
use of org.python.pydev.core.ModulesKeyForZip in project Pydev by fabioz.
the class IOUtils method addAstInfo.
public List<IInfo> addAstInfo(ModulesKey key, boolean generateDelta) throws Exception {
if (key instanceof ModulesKeyForFolder) {
addModulesKeyForFolderToIndex(key, generateDelta);
return new ArrayList<IInfo>(0);
}
boolean isZipModule = key instanceof ModulesKeyForZip;
ModulesKeyForZip modulesKeyForZip = null;
if (isZipModule) {
modulesKeyForZip = (ModulesKeyForZip) key;
}
Object doc;
if (isZipModule) {
doc = FileUtilsFileBuffer.getCustomReturnFromZip(modulesKeyForZip.file, modulesKeyForZip.zipModulePath, null);
} else {
doc = FileUtilsFileBuffer.getCustomReturnFromFile(key.file, true, null);
}
char[] charArray;
int len;
if (doc instanceof IDocument) {
IDocument document = (IDocument) doc;
charArray = document.get().toCharArray();
len = charArray.length;
} else if (doc instanceof FastStringBuffer) {
FastStringBuffer fastStringBuffer = (FastStringBuffer) doc;
// In this case, we can actually get the internal array without doing any copies (and just specifying the len).
charArray = fastStringBuffer.getInternalCharsArray();
len = fastStringBuffer.length();
} else if (doc instanceof String) {
String str = (String) doc;
charArray = str.toCharArray();
len = charArray.length;
} else if (doc instanceof char[]) {
charArray = (char[]) doc;
len = charArray.length;
} else {
throw new RuntimeException("Don't know how to handle: " + doc + " -- " + doc.getClass());
}
SimpleNode node = FastDefinitionsParser.parse(charArray, key.file.getName(), len, key.file);
if (node == null) {
return null;
}
return addAstInfo(node, key, generateDelta);
}
use of org.python.pydev.core.ModulesKeyForZip in project Pydev by fabioz.
the class DiskCache method loadFrom.
/**
* Loads from a reader a string that was acquired from writeTo.
* @param objectsPoolMap
*/
public static DiskCache loadFrom(FastBufferedReader reader, ObjectsPoolMap objectsPoolMap) throws IOException {
DiskCache diskCache = new DiskCache();
FastStringBuffer line = reader.readLine();
if (line.startsWith("-- ")) {
throw new RuntimeException("Unexpected line: " + line);
}
diskCache.folderToPersist = line.toString();
FastStringBuffer buf = new FastStringBuffer();
CompleteIndexKey key = null;
char[] internalCharsArray = line.getInternalCharsArray();
while (true) {
line = reader.readLine();
key = null;
if (line == null || line.startsWith("-- ")) {
if (line != null && line.startsWith("-- END DISKCACHE")) {
return diskCache;
}
throw new RuntimeException("Unexpected line: " + line);
} else {
int length = line.length();
int part = 0;
for (int i = 0; i < length; i++) {
char c = internalCharsArray[i];
if (c == '|') {
switch(part) {
case 0:
key = new CompleteIndexKey(ObjectsInternPool.internLocal(objectsPoolMap, buf.toString()));
diskCache.add(key);
break;
case 1:
key.lastModified = org.python.pydev.shared_core.string.StringUtils.parsePositiveLong(buf);
break;
case 2:
if (buf.length() > 0) {
key.key.file = new File(ObjectsInternPool.internLocal(objectsPoolMap, buf.toString()));
}
break;
case 3:
// regular folder path or zip path
if (buf.endsWith("|^")) {
key.key = new ModulesKeyForFolder(key.key.name, key.key.file);
} else {
key.key = new ModulesKeyForZip(key.key.name, key.key.file, ObjectsInternPool.internLocal(objectsPoolMap, buf.toString()), true);
}
break;
case 4:
// isfile in zip
if (buf.toString().equals("0")) {
((ModulesKeyForZip) key.key).isFile = true;
}
break;
default:
throw new RuntimeException("Unexpected part in line: " + line);
}
part++;
buf.clear();
} else {
buf.append(c);
}
}
// Found end of line... this is the last part and depends on where we stopped previously.
if (buf.length() > 0) {
switch(part) {
case 1:
key.lastModified = StringUtils.parsePositiveLong(buf);
break;
case 2:
// File also written.
key.key.file = new File(ObjectsInternPool.internLocal(objectsPoolMap, buf.toString()));
break;
case 3:
// path in zip
key.key = new ModulesKeyForZip(key.key.name, key.key.file, ObjectsInternPool.internLocal(objectsPoolMap, buf.toString()), true);
break;
case 4:
// isfile in zip
if (buf.toString().equals("0")) {
((ModulesKeyForZip) key.key).isFile = true;
}
break;
}
buf.clear();
}
}
}
}
use of org.python.pydev.core.ModulesKeyForZip in project Pydev by fabioz.
the class ModulesManager method handleLineParts.
private static void handleLineParts(ModulesManager modulesManager, HashMap<Integer, String> intToString, String[] split, int size, ArrayList<ModulesKey> lst) {
if (size > 0 && split[0].length() > 0) {
// Just making sure we have something there.
ModulesKey key;
if (size == 1) {
key = new ModulesKey(split[0], null);
// restore with empty modules.
lst.add(key);
} else if (size == 2) {
key = new ModulesKey(split[0], new File(split[1]));
// restore with empty modules.
lst.add(key);
} else if (size == 3) {
try {
key = new ModulesKeyForFolder(split[0], new File(split[1]));
lst.add(key);
} catch (NumberFormatException e) {
Log.log(e);
}
} else if (size == 4) {
try {
key = new // module name
ModulesKeyForZip(// module name
split[0], // zip file (usually repeated over and over again)
new File(intToString.get(Integer.parseInt(split[1]))), // path in zip
split[2], // is file (false = folder)
split[3].equals("1"));
// restore with empty modules.
lst.add(key);
} catch (NumberFormatException e) {
Log.log(e);
}
}
}
}
use of org.python.pydev.core.ModulesKeyForZip in project Pydev by fabioz.
the class ModulesManager method saveToFile.
@Override
public void saveToFile(File workspaceMetadataFile) {
if (workspaceMetadataFile.exists() && !workspaceMetadataFile.isDirectory()) {
try {
FileUtils.deleteFile(workspaceMetadataFile);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
if (!workspaceMetadataFile.exists()) {
workspaceMetadataFile.mkdirs();
}
File modulesKeysFile = new File(workspaceMetadataFile, "modulesKeys");
File pythonpatHelperFile = new File(workspaceMetadataFile, "pythonpath");
FastStringBuffer buf;
HashMap<String, Integer> commonTokens = new HashMap<String, Integer>();
synchronized (modulesKeysLock) {
buf = new FastStringBuffer(this.modulesKeys.size() * 50);
buf.append(MODULES_MANAGER_V3);
for (Iterator<ModulesKey> iter = this.modulesKeys.keySet().iterator(); iter.hasNext(); ) {
ModulesKey next = iter.next();
buf.append(next.name);
if (next.file != null) {
buf.append("|");
if (next instanceof ModulesKeyForZip) {
ModulesKeyForZip modulesKeyForZip = (ModulesKeyForZip) next;
if (modulesKeyForZip.zipModulePath != null) {
String fileStr = next.file.toString();
Integer t = commonTokens.get(fileStr);
if (t == null) {
t = commonTokens.size();
commonTokens.put(fileStr, t);
}
buf.append(t);
buf.append("|");
buf.append(modulesKeyForZip.zipModulePath);
buf.append("|");
buf.append(modulesKeyForZip.isFile ? '1' : '0');
}
} else if (next instanceof ModulesKeyForFolder) {
ModulesKeyForFolder modulesKeyForFolder = (ModulesKeyForFolder) next;
if (modulesKeyForFolder.file != null) {
buf.append(modulesKeyForFolder.file.toString());
buf.append("|^");
}
} else {
buf.append(next.file.toString());
}
}
buf.append('\n');
}
}
if (commonTokens.size() > 0) {
FastStringBuffer header = new FastStringBuffer(buf.length() + (commonTokens.size() * 50));
header.append(MODULES_MANAGER_V3);
header.append("--COMMON--\n");
for (Map.Entry<String, Integer> entries : commonTokens.entrySet()) {
header.append(entries.getValue());
header.append('=');
header.append(entries.getKey());
header.append('\n');
}
header.append("--END-COMMON--\n");
header.append(buf);
buf = header;
}
FileUtils.writeStrToFile(buf.toString(), modulesKeysFile);
this.pythonPathHelper.saveToFile(pythonpatHelperFile);
}
Aggregations