Examples with PythonPathHelper org.python.pydev.ast.codecompletion.revisited.PythonPathHelper used on opensource projects

Example 6 with PythonPathHelper

use of org.python.pydev.ast.codecompletion.revisited.PythonPathHelper in project Pydev by fabioz.

the class InterpreterInfoBuilder method syncInfoToPythonPath.

public BuilderResult syncInfoToPythonPath(IProgressMonitor monitor, IPythonNature nature) {
    ICodeCompletionASTManager astManager = nature.getAstManager();
    if (astManager == null) {
        return BuilderResult.MUST_SYNCH_LATER;
    }
    PythonPathHelper pythonPathHelper = (PythonPathHelper) astManager.getModulesManager().getPythonPathHelper();
    if (pythonPathHelper == null) {
        return BuilderResult.OK;
    }
    AbstractAdditionalDependencyInfo additionalInfo;
    try {
        additionalInfo = AdditionalProjectInterpreterInfo.getAdditionalInfoForProject(nature);
        IModulesManager modulesManager = astManager.getModulesManager();
        return this.syncInfoToPythonPath(monitor, pythonPathHelper, additionalInfo, modulesManager, null);
    } catch (MisconfigurationException e) {
        Log.log(e);
        return BuilderResult.OK;
    }
}

Example 7 with PythonPathHelper

use of org.python.pydev.ast.codecompletion.revisited.PythonPathHelper in project Pydev by fabioz.

the class AdditionalInfoIntegrityChecker method checkIntegrity.

public static IntegrityInfo checkIntegrity(IPythonNature nature, IProgressMonitor monitor, boolean fix) throws MisconfigurationException {
    IntegrityInfo info = new IntegrityInfo();
    StringBuffer buffer = info.desc;
    info.nature = nature;
    info.modulesManager = nature.getAstManager().getModulesManager();
    info.additionalProjectInfo = (AdditionalProjectInterpreterInfo) AdditionalProjectInterpreterInfo.getAdditionalInfoForProject(nature);
    if (info.additionalProjectInfo == null) {
        buffer.append(StringUtils.format("Unable to get additional project info for: %s (gotten null)", nature.getProject()));
        info.allOk = false;
    }
    PythonPathHelper pythonPathHelper = (PythonPathHelper) info.modulesManager.getPythonPathHelper();
    List<String> pythonpath = pythonPathHelper.getPythonpath();
    buffer.append(org.python.pydev.shared_core.string.StringUtils.format("Checking the integrity of the project: %s\n\n", nature.getProject().getName()));
    buffer.append("Pythonpath:\n");
    for (String string : pythonpath) {
        buffer.append(string);
        buffer.append("\n");
    }
    buffer.append("\n");
    HashSet<ModulesKey> expectedModuleNames = new HashSet<ModulesKey>();
    for (String string : pythonpath) {
        File file = new File(string);
        if (file.exists() && file.isDirectory()) {
            // TODO: Handle zip file modules!
            Collection<PyFileInfo> modulesBelow = PythonPathHelper.getModulesBelow(file, monitor, pythonpath).getFoundPyFileInfos();
            for (PyFileInfo fileInfo : modulesBelow) {
                File moduleFile = fileInfo.getFile();
                String modName = pythonPathHelper.resolveModule(FileUtils.getFileAbsolutePath(moduleFile), true, nature.getProject());
                if (modName != null) {
                    expectedModuleNames.add(new ModulesKey(modName, moduleFile));
                    buffer.append(StringUtils.format("Found module: %s - %s\n", modName, moduleFile));
                } else {
                    if (PythonPathHelper.isValidModuleLastPart(StringUtils.stripExtension((moduleFile.getName())))) {
                        info.allOk = false;
                        buffer.append(StringUtils.format("Unable to resolve module: %s (gotten null module name)\n", moduleFile));
                    }
                }
            }
        } else {
            info.allOk = false;
            buffer.append(StringUtils.format("File %s is referenced in the pythonpath but does not exist.", file));
        }
    }
    check(expectedModuleNames, info, fix);
    return info;
}

Example 8 with PythonPathHelper

use of org.python.pydev.ast.codecompletion.revisited.PythonPathHelper in project Pydev by fabioz.

the class PythonModuleResolverExtensionPointTest method testResolvePathWithResolver.

/**
 * Checks to see that module resolver extensions can be used in resolveModule().
 */
public void testResolvePathWithResolver() {
    PythonPathHelper helper = new PythonPathHelper();
    boolean isPython3Test = false;
    String path = TestDependent.getCompletePythonLib(true, isPython3Test) + "|" + TestDependent.TEST_PYSRC_TESTING_LOC;
    helper.setPythonPath(path);
    final IPath stubbedModulePath1 = Path.fromOSString(FileUtils.getFileAbsolutePath("/this/is/a/path/to/a/file1.py"));
    final IPath stubbedModulePath2 = Path.fromOSString(FileUtils.getFileAbsolutePath("/this/is/a/path/to/a/file2.py"));
    final IPath stubbedNegativeCase = Path.fromOSString(FileUtils.getFileAbsolutePath("/this/is/a/path/to/another/file.py"));
    setTestingModuleResolver(new IPythonModuleResolver() {

        @Override
        public String resolveModule(IProject project, IPath moduleLocation, List<IPath> baseLocations) {
            if (moduleLocation.equals(stubbedModulePath1)) {
                return "stubbed.library";
            }
            if (moduleLocation.equals(stubbedModulePath2)) {
                return "stubbed.second_library";
            }
            if (moduleLocation.equals(stubbedNegativeCase)) {
                return "";
            }
            // Otherwise, delegate.
            return null;
        }

        @Override
        public Collection<IPath> findAllModules(IProject project, IProgressMonitor monitor) {
            throw new UnsupportedOperationException();
        }
    });
    IProject project = null;
    // Check normal resolution:
    assertEquals("stubbed.library", helper.resolveModule(stubbedModulePath1.toOSString(), project));
    assertEquals("stubbed.second_library", helper.resolveModule(stubbedModulePath2.toOSString(), project));
    // Check to see that delegation also works:
    assertEquals("unittest", helper.resolveModule(TestDependent.PYTHON2_LIB + "unittest.py", project));
    // Check the negative case:
    assertNull(helper.resolveModule(stubbedNegativeCase.toOSString(), project));
}

Example 9 with PythonPathHelper

use of org.python.pydev.ast.codecompletion.revisited.PythonPathHelper in project Pydev by fabioz.

the class ReferenceSearches method search.

@Override
public List<ModulesKey> search(IProject project, OrderedMap<String, Set<String>> fieldNameToValues, IProgressMonitor monitor) {
    final List<ModulesKey> ret = new ArrayList<ModulesKey>();
    AbstractAdditionalDependencyInfo abstractAdditionalDependencyInfo = this.abstractAdditionalDependencyInfo.get();
    if (abstractAdditionalDependencyInfo == null) {
        Log.log("AbstractAdditionalDependencyInfo already collected!");
        return ret;
    }
    final NullProgressMonitor nullMonitor = new NullProgressMonitor();
    Set<String> pythonPathFolders = abstractAdditionalDependencyInfo.getPythonPathFolders();
    LinkedBlockingQueue<Command> queue = new LinkedBlockingQueue<>();
    int searchers = Runtime.getRuntime().availableProcessors();
    // The 'ret' should be filled with the module keys where the tokens are found.
    final Object retLock = new Object();
    // Create 2 consumers
    Thread[] threads = new Thread[searchers];
    for (int i = 0; i < searchers; i++) {
        Searcher searcher = new Searcher(queue, fieldNameToValues.get(IReferenceSearches.FIELD_CONTENTS), ret, retLock);
        // Spawn a thread to do the search while we load the contents.
        Thread t = new Thread(searcher);
        threads[i] = t;
        t.start();
    }
    try {
        PythonPathHelper pythonPathHelper = new PythonPathHelper();
        pythonPathHelper.setPythonPath(new ArrayList<String>(pythonPathFolders));
        ModulesFoundStructure modulesFound = pythonPathHelper.getModulesFoundStructure(project, nullMonitor);
        int totalSteps = modulesFound.regularModules.size() + modulesFound.zipContents.size();
        monitor.beginTask("Get modules with token in: " + abstractAdditionalDependencyInfo.getUIRepresentation(), totalSteps);
        PyPublicTreeMap<ModulesKey, ModulesKey> keys = new PyPublicTreeMap<>();
        // no point in searching dlls.
        boolean includeOnlySourceModules = true;
        ModulesManager.buildKeysForRegularEntries(nullMonitor, modulesFound, keys, includeOnlySourceModules);
        // Get from regular files found
        for (ModulesKey entry : keys.values()) {
            if (entry instanceof ModulesKeyForFolder) {
                continue;
            }
            if (monitor.isCanceled()) {
                break;
            }
            if (AbstractAdditionalDependencyInfo.DEBUG) {
                System.out.println("Loading: " + entry);
            }
            final File file = entry.file;
            try {
                queue.put(new Command(entry, new IBufferFiller() {

                    @Override
                    public void fillBuffer(FastStringBuffer bufFileContents) {
                        try (FileInputStream stream = new FileInputStream(file)) {
                            fill(bufFileContents, stream);
                        } catch (Exception e) {
                            Log.log(e);
                        }
                    }
                }));
            } catch (InterruptedException e) {
                Log.log(e);
            }
        }
        // Get from zip files found
        List<ZipContents> allZipsZipContents = modulesFound.zipContents;
        for (ZipContents zipContents : allZipsZipContents) {
            keys.clear();
            if (monitor.isCanceled()) {
                break;
            }
            ModulesManager.buildKeysForZipContents(keys, zipContents);
            try (ZipFile zipFile = new ZipFile(zipContents.zipFile)) {
                for (ModulesKey entry : keys.values()) {
                    if (AbstractAdditionalDependencyInfo.DEBUG) {
                        System.out.println("Loading: " + entry);
                    }
                    if (monitor.isCanceled()) {
                        break;
                    }
                    final ModulesKeyForZip z = (ModulesKeyForZip) entry;
                    if (!z.isFile) {
                        continue;
                    }
                    queue.put(new Command(entry, new IBufferFiller() {

                        @Override
                        public void fillBuffer(FastStringBuffer bufFileContents) {
                            try (InputStream stream = zipFile.getInputStream(zipFile.getEntry(z.zipModulePath))) {
                                fill(bufFileContents, stream);
                            } catch (Exception e) {
                                Log.log(e);
                            }
                        }
                    }));
                }
            } catch (Exception e) {
                Log.log(e);
            }
        }
    } finally {
        for (int i = 0; i < searchers; i++) {
            // add it to wait for the thread to finish.
            queue.add(new Command());
        }
    }
    int j = 0;
    while (true) {
        j++;
        boolean liveFound = false;
        for (Thread t : threads) {
            if (t.isAlive()) {
                liveFound = true;
                break;
            }
        }
        if (liveFound) {
            if (j % 50 == 0) {
                monitor.setTaskName("Searching references...");
                monitor.worked(1);
            }
            Thread.yield();
        } else {
            break;
        }
    }
    return ret;
}

Example 10 with PythonPathHelper

use of org.python.pydev.ast.codecompletion.revisited.PythonPathHelper in project Pydev by fabioz.

the class PyDevBuilderVisitor method isResourceInPythonpathProjectSources.

public boolean isResourceInPythonpathProjectSources(IResource resource, IPythonNature nature, boolean addExternal) throws CoreException, MisconfigurationException {
    Boolean isInProjectPythonpath = (Boolean) memo.get(MODULE_IN_PROJECT_PYTHONPATH + addExternal);
    if (isInProjectPythonpath == null) {
        // This was simply: String moduleName = nature.resolveModuleOnlyInProjectSources(resource, addExternal);
        // Inlined with the code below because nature.getPythonPathNature().getOnlyProjectPythonPathStr was one of
        // the slowest things when doing a full build.
        List<String> onlyProjectPythonPathLst = memo.getOnlyProjectPythonPathStr(nature, addExternal);
        String resourceOSString = SharedCorePlugin.getIResourceOSString(resource);
        String moduleName = null;
        if (resourceOSString != null) {
            ICodeCompletionASTManager astManager = nature.getAstManager();
            if (astManager != null) {
                IModulesManager modulesManager = astManager.getModulesManager();
                if (modulesManager instanceof ProjectModulesManager) {
                    PythonPathHelper pythonPathHelper = ((ProjectModulesManager) modulesManager).getPythonPathHelper();
                    moduleName = pythonPathHelper.resolveModule(resourceOSString, false, onlyProjectPythonPathLst, nature.getProject());
                }
            }
        }
        isInProjectPythonpath = (moduleName != null);
        if (isInProjectPythonpath) {
            setModuleNameInCache(memo, resource, moduleName);
        }
    }
    return isInProjectPythonpath;
}