Examples with PyFileInfo org.python.pydev.ast.listing_utils.PyFileListing.PyFileInfo used on opensource projects

Example 1 with PyFileInfo

use of org.python.pydev.ast.listing_utils.PyFileListing.PyFileInfo in project Pydev by fabioz.

the class RefactoringRenameTestBase method setUp.

/**
 * In the setUp, it initializes the files in the refactoring project
 * @see com.python.pydev.analysis.refactoring.refactorer.refactorings.renamelocal.RefactoringLocalTestBase#setUp()
 */
@Override
public void setUp() throws Exception {
    super.setUp();
    if (filesInRefactoringProject == null) {
        filesInRefactoringProject = PyFileListing.getPyFilesBelow(new File(TestDependent.TEST_COM_REFACTORING_PYSRC_LOC), new NullProgressMonitor(), true).getFoundPyFileInfos();
        ArrayList<Tuple<List<ModulesKey>, IPythonNature>> iFiles = new ArrayList<Tuple<List<ModulesKey>, IPythonNature>>();
        List<ModulesKey> modules = new ArrayList<ModulesKey>();
        iFiles.add(new Tuple<List<ModulesKey>, IPythonNature>(modules, natureRefactoring));
        FastStringBuffer tempBuf = new FastStringBuffer();
        for (PyFileInfo info : filesInRefactoringProject) {
            File f = info.getFile();
            String modName = info.getModuleName(tempBuf);
            ModulesKey modulesKey = new ModulesKey(modName, f);
            modules.add(modulesKey);
            SourceModule mod = (SourceModule) AbstractModule.createModule(modName, f, natureRefactoring, true);
            // also create the additional info so that it can be used for finds
            AbstractAdditionalTokensInfo additionalInfo = AdditionalProjectInterpreterInfo.getAdditionalInfoForProject(natureRefactoring);
            additionalInfo.addAstInfo(mod.getAst(), modulesKey, false);
        }
    // RefactorerFindReferences.FORCED_RETURN = iFiles;
    }
    setUpConfigWorkspaceFiles();
}

Example 2 with PyFileInfo

use of org.python.pydev.ast.listing_utils.PyFileListing.PyFileInfo in project Pydev by fabioz.

the class NatureGroup method calculateChildren.

@Override
protected void calculateChildren() throws MisconfigurationException {
    ICodeCompletionASTManager astManager = nature.getAstManager();
    if (astManager == null) {
        addLeaf("AST manager == null (should happen only in the plugin initialization) -- skipping other checks.");
        return;
    }
    IModulesManager projectModulesManager = astManager.getModulesManager();
    if (projectModulesManager == null) {
        addLeaf("Modules manager == null (should happen only in the plugin initialization) -- skipping other checks.");
        return;
    }
    PythonPathHelper pythonPathHelper = (PythonPathHelper) projectModulesManager.getPythonPathHelper();
    if (pythonPathHelper == null) {
        addLeaf("PythonPathHelper == null (should happen only in the plugin initialization) -- skipping other checks.");
        return;
    }
    List<String> pythonpath = pythonPathHelper.getPythonpath();
    for (String s : pythonpath) {
        addLeaf("PYTHONPATH: " + s);
    }
    HashSet<ModulesKey> expectedModuleNames = new HashSet<ModulesKey>();
    for (String string : pythonpath) {
        File file = new File(string);
        if (file.isDirectory()) {
            // TODO: Handle zip file modules!
            Collection<PyFileInfo> modulesBelow = PythonPathHelper.getModulesBelow(file, new NullProgressMonitor(), pythonpath).getFoundPyFileInfos();
            for (PyFileInfo fileInfo : modulesBelow) {
                File moduleFile = fileInfo.getFile();
                String modName = pythonPathHelper.resolveModule(FileUtils.getFileAbsolutePath(moduleFile), true, nature.getProject());
                if (modName != null) {
                    expectedModuleNames.add(new ModulesKey(modName, moduleFile));
                } else {
                    if (PythonPathHelper.isValidModuleLastPart(StringUtils.stripExtension((moduleFile.getName())))) {
                        addLeaf(StringUtils.format("Unable to resolve module: %s (gotten null module name)", moduleFile));
                    }
                }
            }
        } else {
            if (!file.exists()) {
                addLeaf(StringUtils.format("File %s is referenced in the pythonpath but does not exist.", file));
            } else {
                addLeaf(org.python.pydev.shared_core.string.StringUtils.format("File %s not handled (TODO: Fix zip files support in the viewer).", file));
            }
        }
    }
    IntegrityInfo info = new IntegrityInfo();
    info.nature = nature;
    ModulesKey[] onlyDirectModules = projectModulesManager.getOnlyDirectModules();
    TreeSet<ModulesKey> inModulesManager = new TreeSet<ModulesKey>(Arrays.asList(onlyDirectModules));
    Set<String> allAdditionalInfoModuleNames = new TreeSet<String>();
    List<Tuple<AbstractAdditionalTokensInfo, IPythonNature>> additionalInfoAndNature = AdditionalProjectInterpreterInfo.getAdditionalInfoAndNature(nature, false, false, false);
    AbstractAdditionalTokensInfo additionalProjectInfo;
    if (additionalInfoAndNature.size() == 0) {
        addChild(new LeafElement(this, "No additional infos found (1 expected) -- skipping other checks."));
        return;
    } else {
        if (additionalInfoAndNature.size() > 1) {
            addChild(new LeafElement(this, StringUtils.format("%s additional infos found (only 1 expected) -- continuing checks but analysis may be wrong.", additionalInfoAndNature.size())));
        }
        additionalProjectInfo = additionalInfoAndNature.get(0).o1;
        allAdditionalInfoModuleNames.addAll(additionalProjectInfo.getAllModulesWithTokens());
    }
    for (ModulesKey key : inModulesManager) {
        if (!expectedModuleNames.contains(key)) {
            info.modulesNotInDisk.add(key);
            addChild(new LeafElement(this, StringUtils.format("%s exists in memory but not in the disk.", key)));
        }
    }
    ModulesKey tempKey = new ModulesKey(null, null);
    for (String s : allAdditionalInfoModuleNames) {
        tempKey.name = s;
        if (!expectedModuleNames.contains(tempKey)) {
            info.additionalModulesNotInDisk.add(s);
            addChild(new LeafElement(this, StringUtils.format("%s exists in the additional info but not in the disk.", s)));
        }
    }
    for (ModulesKey key : expectedModuleNames) {
        boolean isInModulesManager = inModulesManager.contains(key);
        if (!isInModulesManager) {
            info.modulesNotInMemory.add(key);
            addChild(new LeafElement(this, StringUtils.format("%s exists in the disk but not in memory.", key)));
        }
        if (!allAdditionalInfoModuleNames.contains(key.name)) {
            try {
                AbstractModule mod = AbstractModule.createModule(key.name, key.file, info.nature, true);
                if (!(mod instanceof SourceModule)) {
                    continue;
                }
                SourceModule module = (SourceModule) mod;
                if (module == null || module.getAst() == null) {
                    addChild(new LeafElement(this, StringUtils.format("Warning: cannot parse: %s - %s (so, it's ok not having additional info on it)", key.name, key.file)));
                } else {
                    try {
                        Iterator<ASTEntry> innerEntriesForAST = AbstractAdditionalDependencyInfo.getInnerEntriesForAST(module.getAst()).o2;
                        if (innerEntriesForAST.hasNext()) {
                            info.moduleNotInAdditionalInfo.add(module);
                            addChild(new LeafElement(this, StringUtils.format("The additional info index of the module: %s is not updated.", key.name)));
                        }
                    } catch (Exception e) {
                        addChild(new LeafElement(this, StringUtils.format("Unexpected error happened on: %s - %s: %s", key.name, key.file, e.getMessage())));
                    }
                }
            } catch (IOException e) {
                // OK, it cannot be parsed, so, we cannot generate its info
                addChild(new LeafElement(this, StringUtils.format("Warning: cannot parse: %s - %s (so, it's ok not having additional info on it)", key.name, key.file)));
            }
        }
    }
    // modules manager
    if (info.modulesNotInDisk.size() > 0) {
        for (ModulesKey m : info.modulesNotInDisk) {
            addChild(new LeafElement(this, StringUtils.format("FIX: Removing from modules manager: %s", m)));
        }
        projectModulesManager.removeModules(info.modulesNotInDisk);
    }
    for (ModulesKey key : info.modulesNotInMemory) {
        addChild(new LeafElement(this, "FIX: Adding to modules manager: " + key));
        projectModulesManager.addModule(key);
    }
    // additional info
    for (String s : info.additionalModulesNotInDisk) {
        addChild(new LeafElement(this, StringUtils.format("FIX: Removing from additional info: %s", s)));
        additionalProjectInfo.removeInfoFromModule(s, true);
    }
    for (SourceModule mod : info.moduleNotInAdditionalInfo) {
        addChild(new LeafElement(this, StringUtils.format("FIX: Adding to additional info: %s", mod.getName())));
        additionalProjectInfo.addAstInfo(mod.getAst(), mod.getModulesKey(), true);
    }
}

Example 3 with PyFileInfo

use of org.python.pydev.ast.listing_utils.PyFileListing.PyFileInfo in project Pydev by fabioz.

the class NewProjectNameAndLocationWizardPage method validatePage.

/**
 * Returns whether this page's controls currently all contain valid
 * values.
 *
 * @return <code>true</code> if all controls are valid, and
 *   <code>false</code> if at least one is invalid
 */
protected boolean validatePage() {
    IWorkspace workspace = ResourcesPlugin.getWorkspace();
    String projectFieldContents = getProjectNameFieldValue();
    if (projectFieldContents.equals("")) {
        // $NON-NLS-1$
        setErrorMessage(null);
        setMessage("Project name is empty");
        return false;
    }
    IStatus nameStatus = workspace.validateName(projectFieldContents, IResource.PROJECT);
    if (!nameStatus.isOK()) {
        setErrorMessage(nameStatus.getMessage());
        return false;
    }
    String locationFieldContents = getProjectLocationFieldValue();
    if (locationFieldContents.equals("")) {
        // $NON-NLS-1$
        setErrorMessage(null);
        setMessage("Project location is empty");
        return false;
    }
    // $NON-NLS-1$
    IPath path = new Path("");
    if (!path.isValidPath(locationFieldContents)) {
        setErrorMessage("Project location is not valid");
        return false;
    }
    // commented out. See comments on https://sourceforge.net/tracker/?func=detail&atid=577329&aid=1798364&group_id=85796
    // if (!useDefaults
    // && Platform.getLocation().isPrefixOf(
    // new Path(locationFieldContents))) {
    // setErrorMessage("Default location error");
    // return false;
    // }
    IProject projectHandle = getProjectHandle();
    if (projectHandle.exists()) {
        setErrorMessage("Project already exists");
        return false;
    }
    if (!useDefaults) {
        path = getLocationPath();
        if (path.equals(workspace.getRoot().getLocation())) {
            setErrorMessage("Project location cannot be the workspace location.");
            return false;
        }
    }
    if (isDotProjectFileInLocation()) {
        setErrorMessage(".project found in: " + getLocationPath().toOSString() + " (use the Import Project wizard instead).");
        return false;
    }
    if (getProjectInterpreter() == null) {
        setErrorMessage("Project interpreter not specified");
        return false;
    }
    setErrorMessage(null);
    setMessage(null);
    // Look for existing Python files in the destination folder.
    File locDir = (!useDefaults ? getLocationPath() : getLocationPath().append(projectFieldContents)).toFile();
    if (locDir.exists()) {
        if (!locDir.isDirectory()) {
            setErrorMessage("The location specified exists but is not a directory.");
            return false;
        }
        boolean foundInit = false;
        try {
            Set<String> s = new HashSet<>();
            s.addAll(Arrays.asList(locDir.list()));
            for (String init : FileTypesPreferences.getValidInitFiles()) {
                if (s.contains(init)) {
                    foundInit = true;
                    break;
                }
            }
        } catch (Exception e) {
            setErrorMessage("It was not possible to list the contents of the specified location.");
            return false;
        }
        if (foundInit) {
            setMessage("Project location contains an __init__.py file. Consider using the location's parent folder instead.");
        } else {
            final long initialTime = System.currentTimeMillis();
            IProgressMonitor monitor = new NullProgressMonitor() {

                @Override
                public boolean isCanceled() {
                    // Cancel after 250 millis
                    if (!super.isCanceled()) {
                        if (System.currentTimeMillis() - initialTime > 250) {
                            setCanceled(true);
                        }
                    }
                    return super.isCanceled();
                }
            };
            PyFileListing pyFileListing = PythonPathHelper.getModulesBelow(locDir, monitor, new ArrayList<>());
            if (pyFileListing != null) {
                Collection<PyFileInfo> modulesBelow = pyFileListing.getFoundPyFileInfos();
                if (modulesBelow.size() > 0) {
                    setMessage("Project location contains existing Python files. The created project will include them.");
                }
            }
        }
    }
    return true;
}

Example 4 with PyFileInfo

use of org.python.pydev.ast.listing_utils.PyFileListing.PyFileInfo in project Pydev by fabioz.

the class PyCoverage method refreshCoverageInfo.

/**
 * This method contacts the python server so that we get the information on the files that are below the directory passed as a parameter
 * and stores the information needed on the cache.
 *
 * @param file
 *            should be the root folder from where we want cache info.
 */
public void refreshCoverageInfo(IContainer container, IProgressMonitor monitor) throws CoverageException {
    int exitValue = 0;
    String stdOut = "";
    String stdErr = "";
    cache.clear();
    if (container == null) {
        return;
    }
    try {
        if (!container.exists()) {
            throw new RuntimeException("The directory passed: " + container + " no longer exists.");
        }
        File file = container.getLocation().toFile();
        PyFileListing pyFilesBelow = new PyFileListing();
        if (file.exists()) {
            pyFilesBelow = PyFileListing.getPyFilesBelow(file, monitor, true);
        }
        if (pyFilesBelow.getFoundPyFileInfos().size() == 0) {
            // no files
            return;
        }
        // add the folders to the cache
        boolean added = false;
        for (Iterator<File> it = pyFilesBelow.getFoundFolders().iterator(); it.hasNext(); ) {
            File f = it.next();
            if (!added) {
                cache.addFolder(f);
                added = true;
            } else {
                cache.addFolder(f, f.getParentFile());
            }
        }
        PythonNature nature = PythonNature.getPythonNature(container);
        if (nature == null) {
            throw new RuntimeException("The directory passed: " + container + " does not have an associated nature.");
        }
        AbstractRunner runner = UniversalRunner.getRunner(nature);
        // First, combine the results of the many runs we may have.
        Tuple<String, String> output = runner.runScriptAndGetOutput(PythonRunnerConfig.getCoverageScript(), new String[] { "combine", "--append" }, getCoverageDirLocation(), monitor);
        if (output.o1 != null && output.o1.length() > 0) {
            Log.logInfo(output.o1);
        }
        if (output.o2 != null && output.o2.length() > 0) {
            if (output.o2.startsWith("Coverage.py warning:")) {
                Log.logInfo(output.o2);
            } else {
                Log.log(output.o2);
            }
        }
        // we have to make a process to execute the script. it should look
        // like:
        // coverage.py -r [-m] FILE1 FILE2 ...
        // Report on the statement coverage for the given files. With the -m
        // option, show line numbers of the statements that weren't
        // executed.
        // python coverage.py -r -m files....
        monitor.setTaskName("Starting shell to get info...");
        File coverageDirLocation = getCoverageDirLocation();
        File coverageXmlLocation = new File(coverageDirLocation, "coverage.xml");
        if (coverageXmlLocation.exists()) {
            coverageXmlLocation.delete();
        }
        monitor.worked(1);
        Process p = null;
        try {
            // Will create the coverage.xml file for us to process later on.
            Tuple<Process, String> tup = runner.createProcess(PythonRunnerConfig.getCoverageScript(), new String[] { "--pydev-analyze" }, coverageDirLocation, monitor);
            p = tup.o1;
            String files = "";
            for (Iterator<PyFileInfo> iter = pyFilesBelow.getFoundPyFileInfos().iterator(); iter.hasNext(); ) {
                String fStr = iter.next().getFile().toString();
                files += fStr + "|";
            }
            files += "\r";
            monitor.setTaskName("Writing to shell...");
            // No need to synchronize as we'll waitFor() the process before getting the contents.
            ThreadStreamReader inputStream = new ThreadStreamReader(p.getInputStream(), false);
            inputStream.start();
            ThreadStreamReader errorStream = new ThreadStreamReader(p.getErrorStream(), false);
            errorStream.start();
            monitor.worked(1);
            OutputStream outputStream = p.getOutputStream();
            outputStream.write(files.getBytes());
            outputStream.close();
            monitor.setTaskName("Waiting for process to finish...");
            monitor.worked(1);
            while (true) {
                try {
                    exitValue = p.exitValue();
                    // process finished
                    break;
                } catch (IllegalThreadStateException e) {
                // not finished
                }
                try {
                    Thread.sleep(50);
                } catch (InterruptedException e) {
                // ignore
                }
                monitor.worked(1);
                if (monitor.isCanceled()) {
                    try {
                        p.destroy();
                    } catch (Exception e) {
                        Log.log(e);
                    }
                    break;
                }
            }
            stdOut = inputStream.getAndClearContents().trim();
            stdErr = errorStream.getAndClearContents().trim();
            if (stdOut.length() > 0) {
                Log.log(stdOut);
            }
            if (stdErr.length() > 0) {
                Log.log(stdErr);
            }
            monitor.setTaskName("Getting coverage info...(please wait, this could take a while)");
            monitor.worked(1);
            if (!coverageXmlLocation.exists()) {
                Log.log("Expected file: " + coverageXmlLocation + " to be written to analyze coverage info.");
            } else {
                CoverageXmlInfo.analyze(cache, coverageXmlLocation);
            }
            monitor.setTaskName("Finished");
        } catch (Exception e) {
            if (p != null) {
                p.destroy();
            }
            Log.log(e);
        }
    } catch (Exception e1) {
        Log.log(e1);
        throw new RuntimeException(e1);
    }
    if (exitValue != 0) {
        FastStringBuffer buf = new FastStringBuffer("Error with coverage action (exit value: ", stdErr.length() + stdOut.length() + 40);
        buf.append(exitValue).append(").");
        if (stdOut.length() > 0) {
            buf.append("\nStandard outputt:\n");
            buf.append(stdOut);
        }
        if (stdErr.length() > 0) {
            buf.append("\nError output:\n");
            buf.append(stdErr);
        }
        throw new CoverageException(buf.toString());
    }
}

Example 5 with PyFileInfo

use of org.python.pydev.ast.listing_utils.PyFileListing.PyFileInfo in project Pydev by fabioz.

the class PythonPathHelper method getModulesFoundStructure.

/**
 * Collects the Python modules.
 * <p>
 * Plugins that extend the {@code org.python.pydev.pydev_python_module_resolver} extension point
 * can extend the behavior of this method.  If no such extension exists, the default behavior
 * is to recursively traverse the directories in the PYTHONPATH.
 *
 * @param project the project scope, can be {@code null} to represent a system-wide collection.
 * @param monitor a project monitor, can be {@code null}.
 * @return a {@link ModulesFoundStructure} containing the encountered modules.
 */
public ModulesFoundStructure getModulesFoundStructure(IProject project, IProgressMonitor monitor) {
    if (monitor == null) {
        monitor = new NullProgressMonitor();
    }
    IPythonModuleResolver[] pythonModuleResolvers = getPythonModuleResolvers();
    if (pythonModuleResolvers.length > 0) {
        List<IPath> searchPaths = this.searchPaths;
        for (IPythonModuleResolver finder : pythonModuleResolvers) {
            Collection<IPath> modulesAndZips = finder.findAllModules(project, monitor);
            if (modulesAndZips == null) {
                continue;
            }
            ModulesFoundStructure modulesFoundStructure = new ModulesFoundStructure();
            for (IPath moduleOrZip : modulesAndZips) {
                File moduleOrZipFile = moduleOrZip.toFile();
                if (FileTypesPreferences.isValidZipFile(moduleOrZip.toOSString())) {
                    ModulesFoundStructure.ZipContents zipContents = getFromZip(moduleOrZipFile, monitor);
                    if (zipContents != null) {
                        modulesFoundStructure.zipContents.add(zipContents);
                    }
                } else {
                    String qualifiedName = finder.resolveModule(project, moduleOrZip, searchPaths);
                    if (qualifiedName != null && !qualifiedName.isEmpty()) {
                        modulesFoundStructure.regularModules.put(moduleOrZipFile, qualifiedName);
                    }
                }
            }
            return modulesFoundStructure;
        }
    }
    // The default behavior is to recursively traverse the directories in the PYTHONPATH to
    // collect all encountered Python modules.
    ModulesFoundStructure ret = new ModulesFoundStructure();
    List<String> pythonpathList = getPythonpath();
    FastStringBuffer tempBuf = new FastStringBuffer();
    for (Iterator<String> iter = pythonpathList.iterator(); iter.hasNext(); ) {
        String element = iter.next();
        if (monitor.isCanceled()) {
            break;
        }
        // the slow part is getting the files... not much we can do (I think).
        File root = new File(element);
        PyFileListing below = getModulesBelow(root, monitor, pythonpathList);
        if (below != null) {
            Iterator<PyFileInfo> e1 = below.getFoundPyFileInfos().iterator();
            while (e1.hasNext()) {
                PyFileInfo pyFileInfo = e1.next();
                File file = pyFileInfo.getFile();
                String modName = pyFileInfo.getModuleName(tempBuf);
                if (isValidModuleLastPart(FullRepIterable.getLastPart(modName))) {
                    // Only override if the new name is < than the previous name
                    // (a file may be found multiple times depending on the pythonpath).
                    String existing = ret.regularModules.get(file);
                    if (existing != null) {
                        if (existing.length() < modName.length()) {
                            ret.regularModules.put(file, modName);
                        }
                    } else {
                        ret.regularModules.put(file, modName);
                    }
                }
            }
        } else {
            // ok, it was null, so, maybe this is not a folder, but zip file with java classes...
            ModulesFoundStructure.ZipContents zipContents = getFromZip(root, monitor);
            if (zipContents != null) {
                ret.zipContents.add(zipContents);
            }
        }
    }
    return ret;
}