Examples with ModulesKey org.python.pydev.core.ModulesKey used on opensource projects

Example 1 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class ReferenceSearchesLucene method internalSearch.

private synchronized List<ModulesKey> internalSearch(IProject project, final OrderedMap<String, Set<String>> fieldNameToValues, IProgressMonitor monitor) throws OperationCanceledException {
    final List<ModulesKey> ret = new ArrayList<ModulesKey>();
    PythonNature nature = PythonNature.getPythonNature(project);
    if (nature == null) {
        Log.log("Project :" + project + " does not have Python nature configured.");
        return ret;
    }
    // Make sure that its information is synchronized.
    AbstractAdditionalDependencyInfo abstractAdditionalDependencyInfo = this.abstractAdditionalDependencyInfo.get();
    if (abstractAdditionalDependencyInfo == null) {
        Log.log("AbstractAdditionalDependencyInfo already collected!");
        return ret;
    }
    Long lastMtime = projectToLastMtime.get(project);
    if (lastMtime == null) {
        lastMtime = 0L;
    }
    long currMtime = nature.getMtime();
    if (lastMtime != currMtime) {
        projectToLastMtime.put(project, currMtime);
        Timer timer = null;
        if (DEBUG) {
            System.out.println("Curr mtime: " + currMtime + " last time: " + lastMtime);
            System.out.println("Start sync: " + project);
            timer = new Timer();
        }
        new InterpreterInfoBuilder().syncInfoToPythonPath(monitor, nature);
        if (DEBUG) {
            timer.printDiff("Sync time");
        }
    }
    boolean mustCommitChange = false;
    final String name = "Search modules with token in: " + abstractAdditionalDependencyInfo.getUIRepresentation();
    monitor.beginTask(name, 7);
    monitor.setTaskName(name);
    DiskCache completeIndex = abstractAdditionalDependencyInfo.completeIndex;
    // Note: we should be able to deal with entries already deleted!
    boolean applyAllDeletes = false;
    String folderToPersist = completeIndex.getFolderToPersist();
    Object indexApiLock;
    File indexDir = new File(folderToPersist, "lc" + IndexApi.luceneSuffix);
    synchronized (lock) {
        indexApiLock = indexDirToLock.get(indexDir);
        if (indexApiLock == null) {
            try {
                indexApiLock = new Object();
                indexDirToLock.put(indexDir, indexApiLock);
            } catch (Exception e) {
                Log.log(e);
                return ret;
            }
        }
    }
    IndexApi indexApi = null;
    try {
        synchronized (indexApiLock) {
            indexApi = new IndexApi(indexDir, applyAllDeletes);
            // Key to CompleteIndexKey (has modified time).
            final Map<ModulesKey, CompleteIndexKey> indexMap = new HashMap<>();
            IDocumentsVisitor visitor = new IDocumentsVisitor() {

                @Override
                public void visit(DocumentInfo documentInfo) {
                    ModulesKey keyFromIO = ModulesKey.fromIO(documentInfo.get(FIELD_MODULES_KEY_IO));
                    String modifiedTime = documentInfo.get(FIELD_MODIFIED_TIME);
                    indexMap.put(keyFromIO, new CompleteIndexKey(keyFromIO, Long.parseLong(modifiedTime)));
                }
            };
            try {
                indexApi.visitAllDocs(visitor, FIELD_MODULES_KEY_IO, FIELD_MODIFIED_TIME);
            } catch (IOException e) {
                Log.log(e);
            }
            incrementAndCheckProgress("Visited current index", monitor);
            Set<CompleteIndexKey> docsToRemove = new HashSet<>();
            Set<CompleteIndexKey> modulesToAdd = new HashSet<>();
            Map<File, Set<CompleteIndexKey>> zipModulesToAdd = new HashMap<>();
            // Wait for the integrity check before getting the keys!
            abstractAdditionalDependencyInfo.waitForIntegrityCheck();
            final Map<CompleteIndexKey, CompleteIndexKey> currentKeys = completeIndex.keys();
            // from the modules (or have a different time).
            for (Entry<ModulesKey, CompleteIndexKey> entryInIndex : indexMap.entrySet()) {
                CompleteIndexKey indexModule = entryInIndex.getValue();
                CompleteIndexKey currentModule = currentKeys.get(indexModule);
                if (currentModule == null || currentModule.key == null || currentModule.key.file == null) {
                    docsToRemove.add(indexModule);
                } else {
                    // exists, but we also need to check the modified time
                    boolean changed = currentModule.lastModified != indexModule.lastModified;
                    if (!changed) {
                        ModulesKey keyCurrentModule = currentModule.key;
                        ModulesKey keyIndexModule = indexModule.key;
                        boolean currentIsZip = keyCurrentModule instanceof ModulesKeyForZip;
                        boolean indexIsZip = keyIndexModule instanceof ModulesKeyForZip;
                        changed = currentIsZip != indexIsZip;
                        if (!changed) {
                            changed = !currentModule.key.file.equals(indexModule.key.file);
                        }
                    }
                    if (changed) {
                        // remove and add
                        docsToRemove.add(indexModule);
                        add(modulesToAdd, zipModulesToAdd, currentModule);
                    }
                }
            }
            // --- Progress
            incrementAndCheckProgress("Updating for removal", monitor);
            // Step 2: add new entries in current and not in the index
            for (Entry<CompleteIndexKey, CompleteIndexKey> currentEntry : currentKeys.entrySet()) {
                CompleteIndexKey completeIndexKey = currentEntry.getValue();
                if (!indexMap.containsKey(completeIndexKey.key)) {
                    ModulesKey modulesKey = completeIndexKey.key;
                    if (modulesKey instanceof IModulesKeyForJava || modulesKey.file == null || !modulesKey.file.isFile()) {
                        // ignore this one (we can't do anything with it).
                        continue;
                    }
                    if (modulesKey instanceof ModulesKeyForZip) {
                        ModulesKeyForZip modulesKeyForZip = (ModulesKeyForZip) modulesKey;
                        if (!modulesKeyForZip.isFile) {
                            // Ignore folders in zips (happens for jython folders which may not have an __init__.py)
                            continue;
                        }
                    }
                    add(modulesToAdd, zipModulesToAdd, completeIndexKey);
                }
            }
            // --- Progress
            incrementAndCheckProgress("Updating for addition", monitor);
            Map<String, Collection<String>> fieldToValuesToRemove = new HashMap<>();
            Collection<String> lstToRemove = new ArrayList<>(docsToRemove.size());
            FastStringBuffer tempBuf = new FastStringBuffer();
            for (Iterator<CompleteIndexKey> it = docsToRemove.iterator(); it.hasNext(); ) {
                it.next().key.toIO(tempBuf.clear());
                lstToRemove.add(tempBuf.toString());
            }
            incrementAndCheckProgress("Removing outdated entries", monitor);
            if (lstToRemove.size() > 0) {
                fieldToValuesToRemove.put(FIELD_MODULES_KEY_IO, lstToRemove);
                try {
                    mustCommitChange = true;
                    if (DEBUG) {
                        System.out.println("Removing: " + fieldToValuesToRemove);
                    }
                    indexApi.removeDocs(fieldToValuesToRemove);
                } catch (IOException e) {
                    Log.log(e);
                }
            }
            incrementAndCheckProgress("Indexing new entries", monitor);
            if (modulesToAdd.size() > 0) {
                mustCommitChange = true;
                for (CompleteIndexKey key : modulesToAdd) {
                    File f = key.key.file;
                    if (f.isFile()) {
                        if (DEBUG) {
                            System.out.println("Indexing: " + f);
                        }
                        try (BufferedReader reader = new BufferedReader(new FileReader(f))) {
                            indexApi.index(createFieldsToIndex(key, tempBuf), reader, FIELD_CONTENTS);
                        } catch (Exception e) {
                            Log.log(e);
                        }
                    }
                }
            }
            Set<Entry<File, Set<CompleteIndexKey>>> entrySet = zipModulesToAdd.entrySet();
            for (Entry<File, Set<CompleteIndexKey>> entry : entrySet) {
                File f = entry.getKey();
                if (f.exists()) {
                    try (ZipFile zipFile = new ZipFile(f, ZipFile.OPEN_READ)) {
                        Set<CompleteIndexKey> value = entry.getValue();
                        for (CompleteIndexKey completeIndexKey2 : value) {
                            ModulesKeyForZip forZip = (ModulesKeyForZip) completeIndexKey2.key;
                            try (InputStream inputStream = zipFile.getInputStream(zipFile.getEntry(forZip.zipModulePath))) {
                                InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
                                mustCommitChange = true;
                                if (DEBUG) {
                                    System.out.println("Indexing: " + completeIndexKey2);
                                }
                                indexApi.index(createFieldsToIndex(completeIndexKey2, tempBuf), reader, FIELD_CONTENTS);
                            }
                        }
                    } catch (Exception e) {
                        Log.log(e);
                    }
                }
            }
            incrementAndCheckProgress("Committing result", monitor);
            if (mustCommitChange) {
                if (DEBUG) {
                    System.out.println("Commit result");
                }
                try {
                    indexApi.commit();
                } catch (IOException e) {
                    Log.log(e);
                }
            }
            // Ok, things should be in-place at this point... let's actually do the search now
            incrementAndCheckProgress("Searching index", monitor);
            try {
                if (DEBUG) {
                    System.out.println("Searching: " + fieldNameToValues);
                }
                visitor = new IDocumentsVisitor() {

                    @Override
                    public void visit(DocumentInfo documentInfo) {
                        try {
                            String modKey = documentInfo.get(FIELD_MODULES_KEY_IO);
                            String modTime = documentInfo.get(FIELD_MODIFIED_TIME);
                            if (modKey != null && modTime != null) {
                                ModulesKey fromIO = ModulesKey.fromIO(modKey);
                                CompleteIndexKey existing = currentKeys.get(new CompleteIndexKey(fromIO));
                                // Deal with deleted entries still hanging around.
                                if (existing != null && existing.lastModified == Long.parseLong(modTime)) {
                                    // Ok, we have a match!
                                    ret.add(existing.key);
                                }
                            }
                        } catch (Exception e) {
                            Log.log(e);
                        }
                    }
                };
                indexApi.searchWildcard(fieldNameToValues, applyAllDeletes, visitor, null, FIELD_MODULES_KEY_IO, FIELD_MODIFIED_TIME);
            } catch (Exception e) {
                Log.log(e);
            }
        }
    } catch (Exception e) {
        Log.log(e);
    } finally {
        if (indexApi != null) {
            indexApi.dispose();
        }
    }
    return ret;
}

Example 2 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class AbstractAdditionalDependencyInfo method updateKeysIfNeededAndSave.

/**
 * If info == null we're dealing with project info (otherwise we're dealing with interpreter info).
 *
 * The main difference is that we don't index builtin modules for projects (but maybe we should?). Still,
 * to index builtin modules we have to be a bit more careful, especially on changes (i.e.: when a builtin
 * becomes a source module and vice-versa).
 */
public void updateKeysIfNeededAndSave(PyPublicTreeMap<ModulesKey, ModulesKey> keysFound, InterpreterInfo info, IProgressMonitor monitor) {
    Map<CompleteIndexKey, CompleteIndexKey> keys = this.completeIndex.keys();
    ArrayList<ModulesKey> newKeys = new ArrayList<ModulesKey>();
    ArrayList<ModulesKey> removedKeys = new ArrayList<ModulesKey>();
    // temporary
    CompleteIndexKey tempKey = new CompleteIndexKey((ModulesKey) null);
    boolean isJython = info != null ? info.getInterpreterType() == IInterpreterManager.INTERPRETER_TYPE_JYTHON : true;
    Iterator<ModulesKey> it = keysFound.values().iterator();
    while (it.hasNext()) {
        ModulesKey next = it.next();
        if (next.file != null) {
            // Can be a .pyd or a .py
            long lastModified = FileUtils.lastModified(next.file);
            if (lastModified != 0) {
                tempKey.key = next;
                CompleteIndexKey completeIndexKey = keys.get(tempKey);
                if (completeIndexKey == null) {
                    newKeys.add(next);
                } else {
                    if (completeIndexKey.lastModified != lastModified) {
                        // Just re-add it if the time changed!
                        newKeys.add(next);
                    }
                }
            }
        } else {
            // at this point, it's always a compiled module (forced builtin), so, we can't check if it was modified (just re-add it).
            tempKey.key = next;
            CompleteIndexKey completeIndexKey = keys.get(tempKey);
            if (completeIndexKey == null) {
                // Only add if it's not there already.
                newKeys.add(next);
            }
        }
    }
    Iterator<CompleteIndexKey> it2 = keys.values().iterator();
    while (it2.hasNext()) {
        CompleteIndexKey next = it2.next();
        if (!keysFound.containsKey(next.key)) {
            removedKeys.add(next.key);
        }
    }
    boolean hasNew = newKeys.size() != 0;
    boolean hasRemoved = removedKeys.size() != 0;
    modulesAddedAndRemoved.call(new Tuple(newKeys, removedKeys));
    Set<File> ignoreFiles = new HashSet<File>();
    // Remove first!
    if (hasRemoved) {
        for (ModulesKey removedKey : removedKeys) {
            // Don't generate deltas (we'll save it in the end).
            this.removeInfoFromModule(removedKey.name, false);
        }
    }
    // Add last (a module could be removed/added).
    if (hasNew) {
        FastStringBuffer buffer = new FastStringBuffer();
        int currI = 0;
        int total = newKeys.size();
        IModuleRequestState moduleRequest = new BaseModuleRequest(true);
        for (ModulesKey newKey : newKeys) {
            currI += 1;
            if (monitor.isCanceled()) {
                return;
            }
            if (PythonPathHelper.canAddAstInfoForSourceModule(newKey)) {
                buffer.clear().append("Indexing ").append(currI).append(" of ").append(total).append(" (source module): ").append(newKey.name).append("  (").append(currI).append(" of ").append(total).append(")");
                try {
                    // Don't generate deltas (we'll save it in the end).
                    this.addAstInfo(newKey, false);
                } catch (Exception e) {
                    Log.log(e);
                }
            } else {
                if (info != null) {
                    if (isJython && ignoreFiles.contains(newKey.file)) {
                        continue;
                    }
                    buffer.clear().append("Indexing ").append(currI).append(" of ").append(total).append(" (builtin module): ").append(newKey.name);
                    monitor.setTaskName(buffer.toString());
                    IModule builtinModule = info.getModulesManager().getModule(newKey.name, info.getModulesManager().getNature(), true, moduleRequest);
                    if (builtinModule != null) {
                        if (builtinModule instanceof IAbstractJavaClassModule) {
                            if (newKey.file != null) {
                                ignoreFiles.add(newKey.file);
                            } else {
                                Log.log("Not expecting null file for java class module: " + newKey);
                            }
                            continue;
                        }
                        boolean removeFirst = keys.containsKey(new CompleteIndexKey(newKey));
                        addAstForCompiledModule(builtinModule, info, newKey, removeFirst);
                    }
                }
            }
        }
    }
    if (hasNew || hasRemoved) {
        if (DebugSettings.DEBUG_INTERPRETER_AUTO_UPDATE) {
            org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, StringUtils.format("Additional info modules. Added: %s Removed: %s", newKeys, removedKeys));
        }
        save();
    }
}

Example 3 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class AbstractAdditionalInfoWithBuild method restoreInfoForModuleManager.

/**
 * Restores the info for a module manager
 *
 * @param monitor a monitor to keep track of the progress
 * @param m the module manager
 * @param nature the associated nature (may be null if there is no associated nature -- as is the case when
 * restoring system info).
 *
 * @return the info generated from the module manager
 */
public static AbstractAdditionalTokensInfo restoreInfoForModuleManager(IProgressMonitor monitor, IModulesManager m, String additionalFeedback, AbstractAdditionalTokensInfo info, IPythonNature nature, int grammarVersion) {
    if (monitor == null) {
        monitor = new NullProgressMonitor();
    }
    // TODO: Check if keeping a zip file open makes things faster...
    // Timer timer = new Timer();
    ModulesKey[] allModules = m.getOnlyDirectModules();
    int i = 0;
    FastStringBuffer msgBuffer = new FastStringBuffer();
    for (ModulesKey key : allModules) {
        if (monitor.isCanceled()) {
            return null;
        }
        i++;
        if (PythonPathHelper.canAddAstInfoForSourceModule(key)) {
            if (i % 17 == 0) {
                msgBuffer.clear();
                msgBuffer.append("Creating ");
                msgBuffer.append(additionalFeedback);
                msgBuffer.append(" additional info (");
                msgBuffer.append(i);
                msgBuffer.append(" of ");
                msgBuffer.append(allModules.length);
                msgBuffer.append(") for ");
                msgBuffer.append(key.file.getName());
                monitor.setTaskName(msgBuffer.toString());
                monitor.worked(1);
            }
            try {
                if (info.addAstInfo(key, false) == null) {
                    String str = "Unable to generate ast -- using %s.\nError:%s";
                    ErrorDescription errorDesc = null;
                    throw new RuntimeException(StringUtils.format(str, PyParser.getGrammarVersionStr(grammarVersion), (errorDesc != null && errorDesc.message != null) ? errorDesc.message : "unable to determine"));
                }
            } catch (Throwable e) {
                Log.log(IStatus.ERROR, "Problem parsing the file :" + key.file + ".", e);
            }
        }
    }
    // timer.printDiff("Time to restore additional info");
    return info;
}

Example 4 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class InterpreterInfoBuilder method syncInfoToPythonPath.

public BuilderResult syncInfoToPythonPath(IProgressMonitor monitor, PythonPathHelper pythonPathHelper, AbstractAdditionalDependencyInfo additionalInfo, IModulesManager modulesManager, InterpreterInfo info) {
    if (monitor == null) {
        monitor = new NullProgressMonitor();
    }
    if (DebugSettings.DEBUG_INTERPRETER_AUTO_UPDATE) {
        org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "--- Start run");
    }
    BuilderResult ret = checkEarlyReturn(monitor, info);
    if (ret != BuilderResult.OK) {
        return ret;
    }
    ModulesFoundStructure modulesFound = pythonPathHelper.getModulesFoundStructure(null, monitor);
    ret = checkEarlyReturn(monitor, info);
    if (ret != BuilderResult.OK) {
        return ret;
    }
    PyPublicTreeMap<ModulesKey, ModulesKey> keysFound = ModulesManager.buildKeysFromModulesFound(monitor, modulesFound);
    if (DebugSettings.DEBUG_INTERPRETER_AUTO_UPDATE) {
        org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, StringUtils.format("Found: %s modules", keysFound.size()));
    }
    ret = checkEarlyReturn(monitor, info);
    if (ret != BuilderResult.OK) {
        return ret;
    }
    try {
        if (info != null) {
            String[] builtins = info.getBuiltins();
            // as we have to get the completions for all builtin modules from the shell.
            if (builtins != null) {
                for (int i = 0; i < builtins.length; i++) {
                    String name = builtins[i];
                    final ModulesKey k = new ModulesKey(name, null);
                    // Note that it'll override source modules!
                    keysFound.put(k, k);
                }
            }
        }
        synchronized (additionalInfo.updateKeysLock) {
            // Use a lock (if we have more than one builder updating we could get into a racing condition here).
            // Important: do the diff only after the builtins are added (otherwise the modules manager may become wrong)!
            Tuple<List<ModulesKey>, List<ModulesKey>> diffModules = modulesManager.diffModules(keysFound);
            if (diffModules.o1.size() > 0 || diffModules.o2.size() > 0) {
                if (DebugSettings.DEBUG_INTERPRETER_AUTO_UPDATE) {
                    org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, StringUtils.format("Diff modules. Added: %s Removed: %s", diffModules.o1, diffModules.o2));
                }
                // Update the modules manager itself (just pass all the keys as that should be fast)
                if (modulesManager instanceof SystemModulesManager) {
                    ((SystemModulesManager) modulesManager).updateKeysAndSave(keysFound);
                } else {
                    for (ModulesKey newEntry : diffModules.o1) {
                        modulesManager.addModule(newEntry);
                    }
                    modulesManager.removeModules(diffModules.o2);
                }
            }
            additionalInfo.updateKeysIfNeededAndSave(keysFound, info, monitor);
        }
    } catch (Exception e) {
        Log.log(e);
    }
    if (DebugSettings.DEBUG_INTERPRETER_AUTO_UPDATE) {
        org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "--- End Run");
    }
    return BuilderResult.OK;
}

Example 5 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class ClassHierarchySearchTest method tearDown.

@Override
public void tearDown() throws Exception {
    CompiledModule.COMPILED_MODULES_ENABLED = false;
    SourceModule.TESTING = false;
    ProjectModulesManager projectModulesManager = ((ProjectModulesManager) nature.getAstManager().getModulesManager());
    projectModulesManager.doRemoveSingleModule(new ModulesKey("foo", null));
    projectModulesManager.doRemoveSingleModule(new ModulesKey("foo0", null));
    projectModulesManager.doRemoveSingleModule(new ModulesKey("fooIn1", null));
    projectModulesManager.doRemoveSingleModule(new ModulesKey("fooIn10", null));
    projectModulesManager = ((ProjectModulesManager) nature2.getAstManager().getModulesManager());
    projectModulesManager.doRemoveSingleModule(new ModulesKey("fooIn2", null));
    projectModulesManager.doRemoveSingleModule(new ModulesKey("fooIn20", null));
    FileUtils.deleteDirectoryTree(baseDir);
    FileUtils.deleteDirectoryTree(baseDir2);
    super.tearDown();
}