Examples with InterpreterInfoBuilder com.python.pydev.analysis.system_info_builder.InterpreterInfoBuilder used on opensource projects

Example 1 with InterpreterInfoBuilder

use of com.python.pydev.analysis.system_info_builder.InterpreterInfoBuilder in project Pydev by fabioz.

the class ReferenceSearchesLucene method internalSearch.

private synchronized List<ModulesKey> internalSearch(IProject project, final OrderedMap<String, Set<String>> fieldNameToValues, IProgressMonitor monitor) throws OperationCanceledException {
    final List<ModulesKey> ret = new ArrayList<ModulesKey>();
    PythonNature nature = PythonNature.getPythonNature(project);
    if (nature == null) {
        Log.log("Project :" + project + " does not have Python nature configured.");
        return ret;
    }
    // Make sure that its information is synchronized.
    AbstractAdditionalDependencyInfo abstractAdditionalDependencyInfo = this.abstractAdditionalDependencyInfo.get();
    if (abstractAdditionalDependencyInfo == null) {
        Log.log("AbstractAdditionalDependencyInfo already collected!");
        return ret;
    }
    Long lastMtime = projectToLastMtime.get(project);
    if (lastMtime == null) {
        lastMtime = 0L;
    }
    long currMtime = nature.getMtime();
    if (lastMtime != currMtime) {
        projectToLastMtime.put(project, currMtime);
        Timer timer = null;
        if (DEBUG) {
            System.out.println("Curr mtime: " + currMtime + " last time: " + lastMtime);
            System.out.println("Start sync: " + project);
            timer = new Timer();
        }
        new InterpreterInfoBuilder().syncInfoToPythonPath(monitor, nature);
        if (DEBUG) {
            timer.printDiff("Sync time");
        }
    }
    boolean mustCommitChange = false;
    final String name = "Search modules with token in: " + abstractAdditionalDependencyInfo.getUIRepresentation();
    monitor.beginTask(name, 7);
    monitor.setTaskName(name);
    DiskCache completeIndex = abstractAdditionalDependencyInfo.completeIndex;
    // Note: we should be able to deal with entries already deleted!
    boolean applyAllDeletes = false;
    String folderToPersist = completeIndex.getFolderToPersist();
    Object indexApiLock;
    File indexDir = new File(folderToPersist, "lc" + IndexApi.luceneSuffix);
    synchronized (lock) {
        indexApiLock = indexDirToLock.get(indexDir);
        if (indexApiLock == null) {
            try {
                indexApiLock = new Object();
                indexDirToLock.put(indexDir, indexApiLock);
            } catch (Exception e) {
                Log.log(e);
                return ret;
            }
        }
    }
    IndexApi indexApi = null;
    try {
        synchronized (indexApiLock) {
            indexApi = new IndexApi(indexDir, applyAllDeletes);
            // Key to CompleteIndexKey (has modified time).
            final Map<ModulesKey, CompleteIndexKey> indexMap = new HashMap<>();
            IDocumentsVisitor visitor = new IDocumentsVisitor() {

                @Override
                public void visit(DocumentInfo documentInfo) {
                    ModulesKey keyFromIO = ModulesKey.fromIO(documentInfo.get(FIELD_MODULES_KEY_IO));
                    String modifiedTime = documentInfo.get(FIELD_MODIFIED_TIME);
                    indexMap.put(keyFromIO, new CompleteIndexKey(keyFromIO, Long.parseLong(modifiedTime)));
                }
            };
            try {
                indexApi.visitAllDocs(visitor, FIELD_MODULES_KEY_IO, FIELD_MODIFIED_TIME);
            } catch (IOException e) {
                Log.log(e);
            }
            incrementAndCheckProgress("Visited current index", monitor);
            Set<CompleteIndexKey> docsToRemove = new HashSet<>();
            Set<CompleteIndexKey> modulesToAdd = new HashSet<>();
            Map<File, Set<CompleteIndexKey>> zipModulesToAdd = new HashMap<>();
            // Wait for the integrity check before getting the keys!
            abstractAdditionalDependencyInfo.waitForIntegrityCheck();
            final Map<CompleteIndexKey, CompleteIndexKey> currentKeys = completeIndex.keys();
            // from the modules (or have a different time).
            for (Entry<ModulesKey, CompleteIndexKey> entryInIndex : indexMap.entrySet()) {
                CompleteIndexKey indexModule = entryInIndex.getValue();
                CompleteIndexKey currentModule = currentKeys.get(indexModule);
                if (currentModule == null || currentModule.key == null || currentModule.key.file == null) {
                    docsToRemove.add(indexModule);
                } else {
                    // exists, but we also need to check the modified time
                    boolean changed = currentModule.lastModified != indexModule.lastModified;
                    if (!changed) {
                        ModulesKey keyCurrentModule = currentModule.key;
                        ModulesKey keyIndexModule = indexModule.key;
                        boolean currentIsZip = keyCurrentModule instanceof ModulesKeyForZip;
                        boolean indexIsZip = keyIndexModule instanceof ModulesKeyForZip;
                        changed = currentIsZip != indexIsZip;
                        if (!changed) {
                            changed = !currentModule.key.file.equals(indexModule.key.file);
                        }
                    }
                    if (changed) {
                        // remove and add
                        docsToRemove.add(indexModule);
                        add(modulesToAdd, zipModulesToAdd, currentModule);
                    }
                }
            }
            // --- Progress
            incrementAndCheckProgress("Updating for removal", monitor);
            // Step 2: add new entries in current and not in the index
            for (Entry<CompleteIndexKey, CompleteIndexKey> currentEntry : currentKeys.entrySet()) {
                CompleteIndexKey completeIndexKey = currentEntry.getValue();
                if (!indexMap.containsKey(completeIndexKey.key)) {
                    ModulesKey modulesKey = completeIndexKey.key;
                    if (modulesKey instanceof IModulesKeyForJava || modulesKey.file == null || !modulesKey.file.isFile()) {
                        // ignore this one (we can't do anything with it).
                        continue;
                    }
                    if (modulesKey instanceof ModulesKeyForZip) {
                        ModulesKeyForZip modulesKeyForZip = (ModulesKeyForZip) modulesKey;
                        if (!modulesKeyForZip.isFile) {
                            // Ignore folders in zips (happens for jython folders which may not have an __init__.py)
                            continue;
                        }
                    }
                    add(modulesToAdd, zipModulesToAdd, completeIndexKey);
                }
            }
            // --- Progress
            incrementAndCheckProgress("Updating for addition", monitor);
            Map<String, Collection<String>> fieldToValuesToRemove = new HashMap<>();
            Collection<String> lstToRemove = new ArrayList<>(docsToRemove.size());
            FastStringBuffer tempBuf = new FastStringBuffer();
            for (Iterator<CompleteIndexKey> it = docsToRemove.iterator(); it.hasNext(); ) {
                it.next().key.toIO(tempBuf.clear());
                lstToRemove.add(tempBuf.toString());
            }
            incrementAndCheckProgress("Removing outdated entries", monitor);
            if (lstToRemove.size() > 0) {
                fieldToValuesToRemove.put(FIELD_MODULES_KEY_IO, lstToRemove);
                try {
                    mustCommitChange = true;
                    if (DEBUG) {
                        System.out.println("Removing: " + fieldToValuesToRemove);
                    }
                    indexApi.removeDocs(fieldToValuesToRemove);
                } catch (IOException e) {
                    Log.log(e);
                }
            }
            incrementAndCheckProgress("Indexing new entries", monitor);
            if (modulesToAdd.size() > 0) {
                mustCommitChange = true;
                for (CompleteIndexKey key : modulesToAdd) {
                    File f = key.key.file;
                    if (f.isFile()) {
                        if (DEBUG) {
                            System.out.println("Indexing: " + f);
                        }
                        try (BufferedReader reader = new BufferedReader(new FileReader(f))) {
                            indexApi.index(createFieldsToIndex(key, tempBuf), reader, FIELD_CONTENTS);
                        } catch (Exception e) {
                            Log.log(e);
                        }
                    }
                }
            }
            Set<Entry<File, Set<CompleteIndexKey>>> entrySet = zipModulesToAdd.entrySet();
            for (Entry<File, Set<CompleteIndexKey>> entry : entrySet) {
                File f = entry.getKey();
                if (f.exists()) {
                    try (ZipFile zipFile = new ZipFile(f, ZipFile.OPEN_READ)) {
                        Set<CompleteIndexKey> value = entry.getValue();
                        for (CompleteIndexKey completeIndexKey2 : value) {
                            ModulesKeyForZip forZip = (ModulesKeyForZip) completeIndexKey2.key;
                            try (InputStream inputStream = zipFile.getInputStream(zipFile.getEntry(forZip.zipModulePath))) {
                                InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
                                mustCommitChange = true;
                                if (DEBUG) {
                                    System.out.println("Indexing: " + completeIndexKey2);
                                }
                                indexApi.index(createFieldsToIndex(completeIndexKey2, tempBuf), reader, FIELD_CONTENTS);
                            }
                        }
                    } catch (Exception e) {
                        Log.log(e);
                    }
                }
            }
            incrementAndCheckProgress("Committing result", monitor);
            if (mustCommitChange) {
                if (DEBUG) {
                    System.out.println("Commit result");
                }
                try {
                    indexApi.commit();
                } catch (IOException e) {
                    Log.log(e);
                }
            }
            // Ok, things should be in-place at this point... let's actually do the search now
            incrementAndCheckProgress("Searching index", monitor);
            try {
                if (DEBUG) {
                    System.out.println("Searching: " + fieldNameToValues);
                }
                visitor = new IDocumentsVisitor() {

                    @Override
                    public void visit(DocumentInfo documentInfo) {
                        try {
                            String modKey = documentInfo.get(FIELD_MODULES_KEY_IO);
                            String modTime = documentInfo.get(FIELD_MODIFIED_TIME);
                            if (modKey != null && modTime != null) {
                                ModulesKey fromIO = ModulesKey.fromIO(modKey);
                                CompleteIndexKey existing = currentKeys.get(new CompleteIndexKey(fromIO));
                                // Deal with deleted entries still hanging around.
                                if (existing != null && existing.lastModified == Long.parseLong(modTime)) {
                                    // Ok, we have a match!
                                    ret.add(existing.key);
                                }
                            }
                        } catch (Exception e) {
                            Log.log(e);
                        }
                    }
                };
                indexApi.searchWildcard(fieldNameToValues, applyAllDeletes, visitor, null, FIELD_MODULES_KEY_IO, FIELD_MODIFIED_TIME);
            } catch (Exception e) {
                Log.log(e);
            }
        }
    } catch (Exception e) {
        Log.log(e);
    } finally {
        if (indexApi != null) {
            indexApi.dispose();
        }
    }
    return ret;
}

Example 2 with InterpreterInfoBuilder

use of com.python.pydev.analysis.system_info_builder.InterpreterInfoBuilder in project Pydev by fabioz.

the class AdditionalInterpreterInfoTest method testForcedBuiltinsInAdditionalInfo.

// Not working with lucene searches (test must be fixed).
// 
// public void testCompleteIndex() throws Exception {
// String doc = "class Test:\n" +
// "    class Test2:\n" +
// "        def mmm(self):\n" +
// "            a = mmm1\n"
// +
// "            print mmm1";
// File tempFileAt = FileUtils.getTempFileAt(baseDir, "data_temporary_file_on_additional_interpreter_info_test",
// ".py");
// FileUtils.writeStrToFile(doc, tempFileAt);
// try {
// SourceModule module = AbstractModule.createModuleFromDoc("test", tempFileAt, new Document(
// doc), nature, true);
// info.addAstInfo(module.getAst(), new ModulesKey("test", tempFileAt), false);
// 
// List<ModulesKey> modulesWithTokensStartingWith = null;
// 
// modulesWithTokensStartingWith = info.getModulesWithToken("mmm", null);
// assertEquals(1, modulesWithTokensStartingWith.size());
// 
// modulesWithTokensStartingWith = info.getModulesWithToken("mmm1", null);
// assertEquals(1, modulesWithTokensStartingWith.size());
// 
// modulesWithTokensStartingWith = info.getModulesWithToken("mmm4", null);
// assertEquals(0, modulesWithTokensStartingWith.size());
// 
// synchronized (this) {
// wait(1000);
// }
// 
// doc = "new contents";
// FileUtils.writeStrToFile(doc, tempFileAt);
// 
// info.removeInfoFromModule("test", true);
// info.addAstInfo(new ModulesKey("test", tempFileAt), true);
// modulesWithTokensStartingWith = info.getModulesWithToken("mmm", null);
// assertEquals(0, modulesWithTokensStartingWith.size());
// 
// modulesWithTokensStartingWith = info.getModulesWithToken("contents", null);
// assertEquals(1, modulesWithTokensStartingWith.size());
// } finally {
// tempFileAt.delete();
// }
// }
@SuppressWarnings("unchecked")
public void testForcedBuiltinsInAdditionalInfo() throws Exception {
    IInterpreterManager interpreterManager = getInterpreterManager();
    String defaultInterpreter = interpreterManager.getDefaultInterpreterInfo(false).getExecutableOrJar();
    AbstractAdditionalDependencyInfo additionalSystemInfo = AdditionalSystemInterpreterInfo.getAdditionalSystemInfo(interpreterManager, defaultInterpreter);
    checkItertoolsToken(additionalSystemInfo, false);
    InterpreterInfo defaultInterpreterInfo = (InterpreterInfo) interpreterManager.getDefaultInterpreterInfo(false);
    HashSet<String> set = new HashSet<>(Arrays.asList(defaultInterpreterInfo.getBuiltins()));
    assertTrue(set.contains("itertools"));
    // Now, update the information to contain the builtin tokens!
    new InterpreterInfoBuilder().syncInfoToPythonPath(new NullProgressMonitor(), defaultInterpreterInfo);
    checkItertoolsToken(additionalSystemInfo, true);
    // Remove and re-update to check if it's fixed.
    additionalSystemInfo.removeInfoFromModule("itertools", false);
    checkItertoolsToken(additionalSystemInfo, false);
    new InterpreterInfoBuilder().syncInfoToPythonPath(new NullProgressMonitor(), defaultInterpreterInfo);
    checkItertoolsToken(additionalSystemInfo, true);
    int indexSize = additionalSystemInfo.completeIndex.keys().size();
    AdditionalSystemInterpreterInfo newAdditionalInfo = new AdditionalSystemInterpreterInfo(interpreterManager, defaultInterpreter);
    AdditionalSystemInterpreterInfo.setAdditionalSystemInfo(interpreterManager, defaultInterpreter, newAdditionalInfo);
    newAdditionalInfo.load();
    assertEquals(indexSize, newAdditionalInfo.completeIndex.keys().size());
    final List<ModulesKey> added = new ArrayList<>();
    final List<ModulesKey> removed = new ArrayList<>();
    ICallbackListener listener = new ICallbackListener() {

        @Override
        public Object call(Object obj) {
            Tuple t = (Tuple) obj;
            added.addAll((List<ModulesKey>) t.o1);
            removed.addAll((List<ModulesKey>) t.o2);
            return null;
        }
    };
    AbstractAdditionalDependencyInfo.modulesAddedAndRemoved.registerListener(listener);
    try {
        new InterpreterInfoBuilder().syncInfoToPythonPath(new NullProgressMonitor(), defaultInterpreterInfo, newAdditionalInfo);
    } finally {
        AbstractAdditionalDependencyInfo.modulesAddedAndRemoved.unregisterListener(listener);
    }
    if (added.size() > 0) {
        throw new AssertionError("Expected no modules to be added as we just loaded from a clean save. Found:\n" + StringUtils.join("\n", added));
    }
    if (removed.size() > 0) {
        throw new AssertionError("Expected no modules to be removed as we just loaded from a clean save. Found:\n" + StringUtils.join("\n", removed));
    }
    checkItertoolsToken(newAdditionalInfo, true);
}

Example 3 with InterpreterInfoBuilder

use of com.python.pydev.analysis.system_info_builder.InterpreterInfoBuilder in project Pydev by fabioz.

the class AdditionalProjectInterpreterInfo method getAdditionalInfoForProject.

/**
 * @param project the project we want to get info on
 * @return the additional info for a given project (gotten from the cache with its name)
 * @throws MisconfigurationException
 */
public static AbstractAdditionalDependencyInfo getAdditionalInfoForProject(final IPythonNature nature) throws MisconfigurationException {
    if (nature == null) {
        return null;
    }
    IProject project = nature.getProject();
    if (project == null) {
        return null;
    }
    String name = FileUtilsFileBuffer.getValidProjectName(project);
    synchronized (additionalNatureInfoLock) {
        AbstractAdditionalDependencyInfo info = additionalNatureInfo.get(name);
        if (info == null) {
            info = new AdditionalProjectInterpreterInfo(project);
            additionalNatureInfo.put(name, info);
            if (!info.load()) {
                recreateAllInfo(nature, new NullProgressMonitor());
            } else {
                final AbstractAdditionalDependencyInfo temp = info;
                temp.setWaitForIntegrityCheck(true);
                // Ok, after it's loaded the first time, check the index integrity!
                Job j = new Job("Check index integrity for: " + project.getName()) {

                    @Override
                    protected IStatus run(IProgressMonitor monitor) {
                        try (GlobalFeedbackReporter r = GlobalFeedback.start("Check index integrity...")) {
                            new InterpreterInfoBuilder().syncInfoToPythonPath(monitor, nature);
                        } catch (Exception e) {
                            Log.log(e);
                        } finally {
                            temp.setWaitForIntegrityCheck(false);
                        }
                        return Status.OK_STATUS;
                    }
                };
                j.setPriority(Job.INTERACTIVE);
                j.setSystem(true);
                j.schedule();
            }
        }
        return info;
    }
}