Examples with AbstractAdditionalTokensInfo com.python.pydev.analysis.additionalinfo.AbstractAdditionalTokensInfo used on opensource projects

Example 1 with AbstractAdditionalTokensInfo

use of com.python.pydev.analysis.additionalinfo.AbstractAdditionalTokensInfo in project Pydev by fabioz.

the class AnalysisBuilderRunnable method doAnalysis.

@Override
protected void doAnalysis() {
    if (!nature.startRequests()) {
        return;
    }
    try {
        if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
            org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "doAnalysis() - " + moduleName + " " + this.getAnalysisCauseStr());
        }
        // if the resource is not open, there's not much we can do...
        final IResource r = resource;
        if (r == null) {
            org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Finished analysis -- resource null -- " + moduleName);
            return;
        }
        if (!r.getProject().isOpen()) {
            org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Finished analysis -- project closed -- " + moduleName);
            return;
        }
        AnalysisRunner runner = new AnalysisRunner();
        checkStop();
        IAnalysisPreferences analysisPreferences = new AnalysisPreferences(r);
        boolean makeAnalysis = // just get problems in resources that are in the pythonpath
        runner.canDoAnalysis(document) && PyDevBuilderVisitor.isInPythonPath(r) && analysisPreferences.makeCodeAnalysis();
        boolean anotherVisitorRequiresAnalysis = false;
        for (IExternalCodeAnalysisVisitor visitor : allVisitors) {
            anotherVisitorRequiresAnalysis |= visitor.getRequiresAnalysis();
        }
        if (!makeAnalysis) {
            // let's see if we should do code analysis
            if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
                org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Skipping: !makeAnalysis -- " + moduleName);
            }
            if (!anotherVisitorRequiresAnalysis) {
                AnalysisRunner.deleteMarkers(r);
                return;
            } else {
                // Only delete pydev markers (others will be deleted by the respective visitors later on).
                boolean onlyPydevAnalysisMarkers = true;
                AnalysisRunner.deleteMarkers(r, onlyPydevAnalysisMarkers);
            }
        }
        if (makeAnalysis && onlyRecreateCtxInsensitiveInfo) {
            if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
                org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Skipping: !forceAnalysis && analysisCause == ANALYSIS_CAUSE_BUILDER && " + "PyDevBuilderPrefPage.getAnalyzeOnlyActiveEditor() -- " + moduleName);
            }
            return;
        }
        if (nature == null) {
            Log.log("Finished analysis: null nature -- " + moduleName);
            return;
        }
        AbstractAdditionalTokensInfo info = AdditionalProjectInterpreterInfo.getAdditionalInfoForProject(nature);
        if (info == null) {
            Log.log("Unable to get additional info for: " + r + " -- " + moduleName);
            return;
        }
        if (makeAnalysis && DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
            org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "makeAnalysis:" + makeAnalysis + " " + "analysisCause: " + getAnalysisCauseStr() + " -- " + moduleName);
        }
        checkStop();
        if (isHierarchicallyDerived(r)) {
            if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
                org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Resource marked as derived not analyzed: " + r + " -- " + moduleName);
            }
            // might be already there)
            if (r != null) {
                AnalysisRunner.deleteMarkers(r);
            }
            return;
        }
        // Maybe we can improve that when https://github.com/PyCQA/pylint/pull/1189 is done.
        if (!DocumentChanged.hasDocumentChanged(resource, document)) {
            for (IExternalCodeAnalysisVisitor visitor : allVisitors) {
                visitor.startVisit();
            }
        } else {
            for (IExternalCodeAnalysisVisitor visitor : allVisitors) {
                visitor.deleteMarkers();
            }
            if (!makeAnalysis) {
                return;
            }
        }
        List<MarkerInfo> markersFromCodeAnalysis = null;
        if (makeAnalysis) {
            OccurrencesAnalyzer analyzer = new OccurrencesAnalyzer();
            checkStop();
            SourceModule module = (SourceModule) this.module.call(moduleRequest);
            IMessage[] messages = analyzer.analyzeDocument(nature, module, analysisPreferences, document, this.internalCancelMonitor, DefaultIndentPrefs.get(this.resource));
            checkStop();
            if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
                org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Adding markers for module: " + moduleName);
            // for (IMessage message : messages) {
            // Log.toLogFile(this, message.toString());
            // }
            }
            // last chance to stop...
            checkStop();
            // don't stop after setting to add / remove the markers
            if (r != null) {
                boolean analyzeOnlyActiveEditor = PyDevBuilderPreferences.getAnalyzeOnlyActiveEditor();
                if (forceAnalysis || !analyzeOnlyActiveEditor || (analyzeOnlyActiveEditor && (!PyDevBuilderPreferences.getRemoveErrorsWhenEditorIsClosed() || OpenEditors.isEditorOpenForResource(r)))) {
                    markersFromCodeAnalysis = runner.setMarkers(r, document, messages, this.internalCancelMonitor);
                } else {
                    if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
                        org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Skipped adding markers for module: " + moduleName + " (editor not opened).");
                    }
                }
            }
        }
        // if there are callbacks registered, call them if we still didn't return (mostly for tests)
        for (ICallback<Object, IResource> callback : analysisBuilderListeners) {
            try {
                callback.call(r);
            } catch (Exception e) {
                Log.log(e);
            }
        }
        checkStop();
        for (IExternalCodeAnalysisVisitor visitor : allVisitors) {
            visitor.join();
        }
        checkStop();
        if (r != null) {
            for (IExternalCodeAnalysisVisitor visitor : allVisitors) {
                String problemMarker = visitor.getProblemMarkerId();
                String messageId = visitor.getMessageId();
                List<MarkerInfo> markersFromVisitor = visitor.getMarkers(resource);
                if (markersFromVisitor != null && markersFromVisitor.size() > 0) {
                    Map<Integer, List<MarkerInfo>> lineToMarkerInfo = new HashMap<>();
                    if (markersFromCodeAnalysis != null) {
                        for (MarkerInfo codeAnalysisMarkerInfo : markersFromCodeAnalysis) {
                            List<MarkerInfo> list = lineToMarkerInfo.get(codeAnalysisMarkerInfo.lineStart);
                            if (list == null) {
                                list = new ArrayList<>(2);
                                lineToMarkerInfo.put(codeAnalysisMarkerInfo.lineStart, list);
                            }
                            list.add(codeAnalysisMarkerInfo);
                        }
                    }
                    if (visitor == pyLintVisitor) {
                        // (there's no real point in putting an error twice).
                        for (Iterator<MarkerInfo> visitorMarkerInfoIterator = markersFromVisitor.iterator(); visitorMarkerInfoIterator.hasNext(); ) {
                            MarkerInfo visitorMarkerInfo = visitorMarkerInfoIterator.next();
                            List<MarkerInfo> codeAnalysisMarkers = lineToMarkerInfo.get(visitorMarkerInfo.lineStart);
                            if (codeAnalysisMarkers != null && codeAnalysisMarkers.size() > 0) {
                                for (MarkerInfo codeAnalysisMarker : codeAnalysisMarkers) {
                                    if (codeAnalysisMarker.severity < IMarker.SEVERITY_INFO) {
                                        // Don't consider if it shouldn't be shown.
                                        continue;
                                    }
                                    Map<String, Object> additionalInfo = codeAnalysisMarker.additionalInfo;
                                    if (additionalInfo != null) {
                                        Object analysisType = additionalInfo.get(AnalysisRunner.PYDEV_ANALYSIS_TYPE);
                                        if (analysisType != null && analysisType instanceof Integer) {
                                            String pyLintMessageId = CheckAnalysisErrors.getPyLintMessageIdForPyDevAnalysisType((int) analysisType);
                                            if (pyLintMessageId != null && pyLintMessageId.equals(visitorMarkerInfo.additionalInfo.get(messageId))) {
                                                visitorMarkerInfoIterator.remove();
                                                // Stop the for (we've already removed it).
                                                break;
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                    PyMarkerUtils.replaceMarkers(markersFromVisitor, resource, problemMarker, true, this.internalCancelMonitor);
                } else {
                    visitor.deleteMarkers();
                }
            }
        }
    } catch (OperationCanceledException e) {
        // ok, ignore it
        logOperationCancelled();
    } catch (Exception e) {
        Log.log(e);
    } finally {
        try {
            nature.endRequests();
        } catch (Throwable e) {
            Log.log("Error when analyzing: " + moduleName, e);
        }
        try {
            AnalysisBuilderRunnableFactory.removeFromThreads(key, this);
        } catch (Throwable e) {
            Log.log(e);
        }
        dispose();
    }
}

Example 2 with AbstractAdditionalTokensInfo

use of com.python.pydev.analysis.additionalinfo.AbstractAdditionalTokensInfo in project Pydev by fabioz.

the class RefactoringRenameTestBase method setUp.

/**
 * In the setUp, it initializes the files in the refactoring project
 * @see com.python.pydev.analysis.refactoring.refactorer.refactorings.renamelocal.RefactoringLocalTestBase#setUp()
 */
@Override
public void setUp() throws Exception {
    super.setUp();
    if (filesInRefactoringProject == null) {
        filesInRefactoringProject = PyFileListing.getPyFilesBelow(new File(TestDependent.TEST_COM_REFACTORING_PYSRC_LOC), new NullProgressMonitor(), true).getFoundPyFileInfos();
        ArrayList<Tuple<List<ModulesKey>, IPythonNature>> iFiles = new ArrayList<Tuple<List<ModulesKey>, IPythonNature>>();
        List<ModulesKey> modules = new ArrayList<ModulesKey>();
        iFiles.add(new Tuple<List<ModulesKey>, IPythonNature>(modules, natureRefactoring));
        FastStringBuffer tempBuf = new FastStringBuffer();
        for (PyFileInfo info : filesInRefactoringProject) {
            File f = info.getFile();
            String modName = info.getModuleName(tempBuf);
            ModulesKey modulesKey = new ModulesKey(modName, f);
            modules.add(modulesKey);
            SourceModule mod = (SourceModule) AbstractModule.createModule(modName, f, natureRefactoring, true);
            // also create the additional info so that it can be used for finds
            AbstractAdditionalTokensInfo additionalInfo = AdditionalProjectInterpreterInfo.getAdditionalInfoForProject(natureRefactoring);
            additionalInfo.addAstInfo(mod.getAst(), modulesKey, false);
        }
    // RefactorerFindReferences.FORCED_RETURN = iFiles;
    }
    setUpConfigWorkspaceFiles();
}

Example 3 with AbstractAdditionalTokensInfo

use of com.python.pydev.analysis.additionalinfo.AbstractAdditionalTokensInfo in project Pydev by fabioz.

the class GlobalsTwoPanelElementSelector2 method fillContentProvider.

/**
 * This is the place where we put all the info in the content provider. Note that here we must add
 * ALL the info -- later, we'll filter it based on the active working set.
 */
@Override
protected void fillContentProvider(AbstractContentProvider contentProvider, ItemsFilter itemsFilter, IProgressMonitor progressMonitor) throws CoreException {
    if (itemsFilter instanceof InfoFilter) {
        if (progressMonitor != null) {
            progressMonitor.beginTask("Searching...", this.additionalInfo.size());
        }
        for (AbstractAdditionalTokensInfo additionalInfo : this.additionalInfo) {
            if (progressMonitor != null) {
                if (progressMonitor.isCanceled()) {
                    return;
                } else {
                    progressMonitor.worked(1);
                }
            }
            // no duplicates
            Collection<IInfo> allTokens = new HashSet<IInfo>(additionalInfo.getAllTokens());
            for (IInfo iInfo : allTokens) {
                contentProvider.add(new AdditionalInfoAndIInfo(additionalInfo, iInfo), itemsFilter);
            }
            // Also show to the user the modules available as globals (2.2.3)
            IModulesManager modulesManager = null;
            try {
                if (additionalInfo instanceof AdditionalProjectInterpreterInfo) {
                    AdditionalProjectInterpreterInfo projectInterpreterInfo = (AdditionalProjectInterpreterInfo) additionalInfo;
                    IProject project = projectInterpreterInfo.getProject();
                    PythonNature nature = PythonNature.getPythonNature(project);
                    if (nature != null) {
                        ICodeCompletionASTManager astManager = nature.getAstManager();
                        if (astManager != null) {
                            modulesManager = astManager.getModulesManager();
                        }
                    }
                } else if (additionalInfo instanceof AdditionalSystemInterpreterInfo) {
                    AdditionalSystemInterpreterInfo systemInterpreterInfo = (AdditionalSystemInterpreterInfo) additionalInfo;
                    IInterpreterInfo defaultInterpreterInfo = systemInterpreterInfo.getManager().getDefaultInterpreterInfo(false);
                    modulesManager = defaultInterpreterInfo.getModulesManager();
                }
            } catch (Throwable e) {
                Log.log(e);
            }
            if (modulesManager != null) {
                SortedMap<ModulesKey, ModulesKey> allDirectModulesStartingWith = modulesManager.getAllDirectModulesStartingWith("");
                Collection<ModulesKey> values = allDirectModulesStartingWith.values();
                for (ModulesKey modulesKey : values) {
                    contentProvider.add(new AdditionalInfoAndIInfo(additionalInfo, new ModInfo(modulesKey.name, modulesManager.getNature(), modulesKey.file != null ? modulesKey.file.toString() : null, 1, 1)), itemsFilter);
                }
            }
        }
    }
    if (progressMonitor != null) {
        progressMonitor.done();
    }
}

Example 4 with AbstractAdditionalTokensInfo

use of com.python.pydev.analysis.additionalinfo.AbstractAdditionalTokensInfo in project Pydev by fabioz.

the class PyGlobalsBrowser method getFromManagerAndRelatedNatures.

/**
 * Gets it using all the natures that match a given interpreter manager.
 * @throws MisconfigurationException
 */
private static void getFromManagerAndRelatedNatures(String selectedText, IInterpreterManager useManager) {
    AbstractAdditionalTokensInfo additionalSystemInfo;
    try {
        additionalSystemInfo = AdditionalSystemInterpreterInfo.getAdditionalSystemInfo(useManager, useManager.getDefaultInterpreterInfo(true).getExecutableOrJar());
    } catch (MisconfigurationException e) {
        MessageDialog.openError(EditorUtils.getShell(), "Error", "Additional info is not available (default interpreter not configured).");
        handle(e);
        return;
    }
    List<AbstractAdditionalTokensInfo> additionalInfo = new ArrayList<AbstractAdditionalTokensInfo>();
    additionalInfo.add(additionalSystemInfo);
    List<IPythonNature> natures = PythonNature.getPythonNaturesRelatedTo(useManager.getInterpreterType());
    for (IPythonNature nature : natures) {
        AbstractAdditionalDependencyInfo info;
        try {
            info = AdditionalProjectInterpreterInfo.getAdditionalInfoForProject(nature);
            if (info != null) {
                additionalInfo.add(info);
            }
        } catch (MisconfigurationException e) {
            // just go on to the next nature if one is not properly configured.
            handle(e);
        }
    }
    doSelect(natures, additionalInfo, selectedText);
}

Example 5 with AbstractAdditionalTokensInfo

use of com.python.pydev.analysis.additionalinfo.AbstractAdditionalTokensInfo in project Pydev by fabioz.

the class UndefinedVariableQuickFixCreator method createImportQuickProposalsFromMarkerSelectedText.

public static void createImportQuickProposalsFromMarkerSelectedText(IAdaptable projectAdaptable, PySelection ps, int offset, IPythonNature initialNature, List<ICompletionProposalHandle> props, ICodeCompletionASTManager astManager, int start, int end, boolean forceReparseOnApply) throws BadLocationException {
    ps.setSelection(start, end);
    String markerContents = ps.getSelectedText();
    String fullRep = ps.getFullRepAfterSelection();
    IModulesManager projectModulesManager = astManager.getModulesManager();
    IModulesManager[] managersInvolved = projectModulesManager.getManagersInvolved(true);
    boolean doIgnoreImportsStartingWithUnder = AnalysisPreferences.doIgnoreImportsStartingWithUnder(projectAdaptable);
    // Use a single buffer to create all the strings
    FastStringBuffer buffer = new FastStringBuffer();
    // Helper so that we don't add the same module multiple times.
    Set<Tuple<String, String>> mods = new HashSet<Tuple<String, String>>();
    for (IModulesManager iModulesManager : managersInvolved) {
        Set<String> allModules = iModulesManager.getAllModuleNames(false, markerContents.toLowerCase());
        // when an undefined variable is found, we can:
        // - add an auto import (if it is a class or a method or some global attribute)
        // - declare it as a local or global variable
        // - change its name to some other global or local (mistyped)
        // - create a method or class for it (if it is a call)
        // 1. check if it is some module
        CompareContext compareContext = new CompareContext(iModulesManager.getNature());
        for (String completeName : allModules) {
            FullRepIterable iterable = new FullRepIterable(completeName);
            for (String mod : iterable) {
                if (fullRep.startsWith(mod)) {
                    if (// it does not only start with, but it is equal to it.
                    fullRep.length() == mod.length() || (fullRep.length() > mod.length() && fullRep.charAt(mod.length()) == '.')) {
                        buffer.clear();
                        String realImportRep = buffer.append("import ").append(mod).toString();
                        buffer.clear();
                        String displayString = buffer.append("Import ").append(mod).toString();
                        addProp(props, realImportRep, displayString, IInfo.USE_PACKAGE_ICON, offset, mods, compareContext, forceReparseOnApply);
                    }
                }
                String[] strings = FullRepIterable.headAndTail(mod);
                String packageName = strings[0];
                String importRep = strings[1];
                if (importRep.equals(markerContents)) {
                    if (packageName.length() > 0) {
                        buffer.clear();
                        String realImportRep = buffer.append("from ").append(packageName).append(" ").append("import ").append(strings[1]).toString();
                        buffer.clear();
                        String displayString = buffer.append("Import ").append(importRep).append(" (").append(packageName).append(")").toString();
                        addProp(props, realImportRep, displayString, IInfo.USE_PACKAGE_ICON, offset, mods, compareContext, forceReparseOnApply);
                    } else {
                        buffer.clear();
                        String realImportRep = buffer.append("import ").append(strings[1]).toString();
                        buffer.clear();
                        String displayString = buffer.append("Import ").append(importRep).toString();
                        addProp(props, realImportRep, displayString, IInfo.USE_PACKAGE_ICON, offset, mods, compareContext, forceReparseOnApply);
                    }
                }
            }
        }
    }
    // 2. check if it is some global class or method
    List<AbstractAdditionalTokensInfo> additionalInfo;
    try {
        additionalInfo = AdditionalProjectInterpreterInfo.getAdditionalInfo(initialNature);
    } catch (MisconfigurationException e) {
        return;
    }
    FastStringBuffer tempBuf = new FastStringBuffer();
    for (AbstractAdditionalTokensInfo info : additionalInfo) {
        Collection<IInfo> tokensEqualTo = info.getTokensEqualTo(markerContents, AbstractAdditionalTokensInfo.TOP_LEVEL);
        for (IInfo found : tokensEqualTo) {
            // there always is a declaring module
            String name = found.getName();
            String declPackage = found.getDeclaringModuleName();
            String declPackageWithoutInit = declPackage;
            if (declPackageWithoutInit.endsWith(".__init__")) {
                declPackageWithoutInit = declPackageWithoutInit.substring(0, declPackageWithoutInit.length() - 9);
            }
            declPackageWithoutInit = AnalysisPreferences.removeImportsStartingWithUnderIfNeeded(declPackageWithoutInit, tempBuf, doIgnoreImportsStartingWithUnder);
            buffer.clear();
            String importDeclaration = buffer.append("from ").append(declPackageWithoutInit).append(" import ").append(name).toString();
            buffer.clear();
            String displayImport = buffer.append("Import ").append(name).append(" (").append(declPackage).append(")").toString();
            addProp(props, importDeclaration, displayImport, found.getType(), offset, mods, new CompareContext(found.getNature()), forceReparseOnApply);
        }
    }
}