Examples with ZipContents org.python.pydev.ast.codecompletion.revisited.ModulesFoundStructure.ZipContents used on opensource projects

Example 1 with ZipContents

use of org.python.pydev.ast.codecompletion.revisited.ModulesFoundStructure.ZipContents in project Pydev by fabioz.

the class ModulesManager method buildKeysFromModulesFound.

public static PyPublicTreeMap<ModulesKey, ModulesKey> buildKeysFromModulesFound(IProgressMonitor monitor, ModulesFoundStructure modulesFound) {
    // now, on to actually filling the module keys
    PyPublicTreeMap<ModulesKey, ModulesKey> keys = new PyPublicTreeMap<ModulesKey, ModulesKey>();
    buildKeysForRegularEntries(monitor, modulesFound, keys, false);
    for (ZipContents zipContents : modulesFound.zipContents) {
        if (monitor.isCanceled()) {
            break;
        }
        buildKeysForZipContents(keys, zipContents);
    }
    return keys;
}

Example 2 with ZipContents

use of org.python.pydev.ast.codecompletion.revisited.ModulesFoundStructure.ZipContents in project Pydev by fabioz.

the class PythonPathHelper method getModulesFoundStructure.

/**
 * Collects the Python modules.
 * <p>
 * Plugins that extend the {@code org.python.pydev.pydev_python_module_resolver} extension point
 * can extend the behavior of this method.  If no such extension exists, the default behavior
 * is to recursively traverse the directories in the PYTHONPATH.
 *
 * @param project the project scope, can be {@code null} to represent a system-wide collection.
 * @param monitor a project monitor, can be {@code null}.
 * @return a {@link ModulesFoundStructure} containing the encountered modules.
 */
public ModulesFoundStructure getModulesFoundStructure(IProject project, IProgressMonitor monitor) {
    if (monitor == null) {
        monitor = new NullProgressMonitor();
    }
    IPythonModuleResolver[] pythonModuleResolvers = getPythonModuleResolvers();
    if (pythonModuleResolvers.length > 0) {
        List<IPath> searchPaths = this.searchPaths;
        for (IPythonModuleResolver finder : pythonModuleResolvers) {
            Collection<IPath> modulesAndZips = finder.findAllModules(project, monitor);
            if (modulesAndZips == null) {
                continue;
            }
            ModulesFoundStructure modulesFoundStructure = new ModulesFoundStructure();
            for (IPath moduleOrZip : modulesAndZips) {
                File moduleOrZipFile = moduleOrZip.toFile();
                if (FileTypesPreferences.isValidZipFile(moduleOrZip.toOSString())) {
                    ModulesFoundStructure.ZipContents zipContents = getFromZip(moduleOrZipFile, monitor);
                    if (zipContents != null) {
                        modulesFoundStructure.zipContents.add(zipContents);
                    }
                } else {
                    String qualifiedName = finder.resolveModule(project, moduleOrZip, searchPaths);
                    if (qualifiedName != null && !qualifiedName.isEmpty()) {
                        modulesFoundStructure.regularModules.put(moduleOrZipFile, qualifiedName);
                    }
                }
            }
            return modulesFoundStructure;
        }
    }
    // The default behavior is to recursively traverse the directories in the PYTHONPATH to
    // collect all encountered Python modules.
    ModulesFoundStructure ret = new ModulesFoundStructure();
    List<String> pythonpathList = getPythonpath();
    FastStringBuffer tempBuf = new FastStringBuffer();
    for (Iterator<String> iter = pythonpathList.iterator(); iter.hasNext(); ) {
        String element = iter.next();
        if (monitor.isCanceled()) {
            break;
        }
        // the slow part is getting the files... not much we can do (I think).
        File root = new File(element);
        PyFileListing below = getModulesBelow(root, monitor, pythonpathList);
        if (below != null) {
            Iterator<PyFileInfo> e1 = below.getFoundPyFileInfos().iterator();
            while (e1.hasNext()) {
                PyFileInfo pyFileInfo = e1.next();
                File file = pyFileInfo.getFile();
                String modName = pyFileInfo.getModuleName(tempBuf);
                if (isValidModuleLastPart(FullRepIterable.getLastPart(modName))) {
                    // Only override if the new name is < than the previous name
                    // (a file may be found multiple times depending on the pythonpath).
                    String existing = ret.regularModules.get(file);
                    if (existing != null) {
                        if (existing.length() < modName.length()) {
                            ret.regularModules.put(file, modName);
                        }
                    } else {
                        ret.regularModules.put(file, modName);
                    }
                }
            }
        } else {
            // ok, it was null, so, maybe this is not a folder, but zip file with java classes...
            ModulesFoundStructure.ZipContents zipContents = getFromZip(root, monitor);
            if (zipContents != null) {
                ret.zipContents.add(zipContents);
            }
        }
    }
    return ret;
}

Example 3 with ZipContents

use of org.python.pydev.ast.codecompletion.revisited.ModulesFoundStructure.ZipContents in project Pydev by fabioz.

the class PythonPathHelper method getFromZip.

/**
 * @param root the zip file to analyze
 * @param monitor the monitor, to keep track of what is happening
 * @return a list with the name of the found modules in the jar
 */
protected static ModulesFoundStructure.ZipContents getFromZip(File root, IProgressMonitor monitor) {
    String fileName = root.getName();
    if (root.isFile() && FileTypesPreferences.isValidZipFile(fileName)) {
        // ok, it may be a jar file, so let's get its contents and get the available modules
        // the major difference from handling jars from regular python files is that we don't have to check for __init__.py files
        ModulesFoundStructure.ZipContents zipContents = new ModulesFoundStructure.ZipContents(root);
        // by default it's a zip (for python) -- may change if a .class is found.
        zipContents.zipContentsType = ZipContents.ZIP_CONTENTS_TYPE_PY_ZIP;
        try {
            String zipFileName = root.getName();
            ZipFile zipFile = new ZipFile(root);
            try {
                Enumeration<? extends ZipEntry> entries = zipFile.entries();
                int i = 0;
                FastStringBuffer buffer = new FastStringBuffer();
                // ok, now that we have the zip entries, let's map them to modules
                while (entries.hasMoreElements()) {
                    ZipEntry entry = entries.nextElement();
                    String name = entry.getName();
                    if (!entry.isDirectory()) {
                        if (isValidFileMod(name) || name.endsWith(".class")) {
                            if (name.endsWith(".class")) {
                                zipContents.zipContentsType = ZipContents.ZIP_CONTENTS_TYPE_JAR;
                            }
                            // it is a valid python file
                            if (i % 15 == 0) {
                                if (monitor.isCanceled()) {
                                    return null;
                                }
                                buffer.clear();
                                monitor.setTaskName(buffer.append("Found in ").append(zipFileName).append(" module ").append(name).toString());
                                monitor.worked(1);
                            }
                            zipContents.pyFilesLowerToRegular.put(name.toLowerCase(), name);
                        }
                    } else {
                        // !isDirectory
                        zipContents.pyfoldersLower.add(name.toLowerCase());
                    }
                    i++;
                }
            } finally {
                zipFile.close();
            }
            // now, on to actually filling the structure if we have a zip file (just add the ones that are actually under
            // the pythonpath)
            zipContents.consolidatePythonpathInfo(monitor);
            return zipContents;
        } catch (Exception e) {
            // that's ok, it is probably not a zip file after all...
            Log.log(e);
        }
    }
    return null;
}

Example 4 with ZipContents

use of org.python.pydev.ast.codecompletion.revisited.ModulesFoundStructure.ZipContents in project Pydev by fabioz.

the class ReferenceSearches method search.

@Override
public List<ModulesKey> search(IProject project, OrderedMap<String, Set<String>> fieldNameToValues, IProgressMonitor monitor) {
    final List<ModulesKey> ret = new ArrayList<ModulesKey>();
    AbstractAdditionalDependencyInfo abstractAdditionalDependencyInfo = this.abstractAdditionalDependencyInfo.get();
    if (abstractAdditionalDependencyInfo == null) {
        Log.log("AbstractAdditionalDependencyInfo already collected!");
        return ret;
    }
    final NullProgressMonitor nullMonitor = new NullProgressMonitor();
    Set<String> pythonPathFolders = abstractAdditionalDependencyInfo.getPythonPathFolders();
    LinkedBlockingQueue<Command> queue = new LinkedBlockingQueue<>();
    int searchers = Runtime.getRuntime().availableProcessors();
    // The 'ret' should be filled with the module keys where the tokens are found.
    final Object retLock = new Object();
    // Create 2 consumers
    Thread[] threads = new Thread[searchers];
    for (int i = 0; i < searchers; i++) {
        Searcher searcher = new Searcher(queue, fieldNameToValues.get(IReferenceSearches.FIELD_CONTENTS), ret, retLock);
        // Spawn a thread to do the search while we load the contents.
        Thread t = new Thread(searcher);
        threads[i] = t;
        t.start();
    }
    try {
        PythonPathHelper pythonPathHelper = new PythonPathHelper();
        pythonPathHelper.setPythonPath(new ArrayList<String>(pythonPathFolders));
        ModulesFoundStructure modulesFound = pythonPathHelper.getModulesFoundStructure(project, nullMonitor);
        int totalSteps = modulesFound.regularModules.size() + modulesFound.zipContents.size();
        monitor.beginTask("Get modules with token in: " + abstractAdditionalDependencyInfo.getUIRepresentation(), totalSteps);
        PyPublicTreeMap<ModulesKey, ModulesKey> keys = new PyPublicTreeMap<>();
        // no point in searching dlls.
        boolean includeOnlySourceModules = true;
        ModulesManager.buildKeysForRegularEntries(nullMonitor, modulesFound, keys, includeOnlySourceModules);
        // Get from regular files found
        for (ModulesKey entry : keys.values()) {
            if (entry instanceof ModulesKeyForFolder) {
                continue;
            }
            if (monitor.isCanceled()) {
                break;
            }
            if (AbstractAdditionalDependencyInfo.DEBUG) {
                System.out.println("Loading: " + entry);
            }
            final File file = entry.file;
            try {
                queue.put(new Command(entry, new IBufferFiller() {

                    @Override
                    public void fillBuffer(FastStringBuffer bufFileContents) {
                        try (FileInputStream stream = new FileInputStream(file)) {
                            fill(bufFileContents, stream);
                        } catch (Exception e) {
                            Log.log(e);
                        }
                    }
                }));
            } catch (InterruptedException e) {
                Log.log(e);
            }
        }
        // Get from zip files found
        List<ZipContents> allZipsZipContents = modulesFound.zipContents;
        for (ZipContents zipContents : allZipsZipContents) {
            keys.clear();
            if (monitor.isCanceled()) {
                break;
            }
            ModulesManager.buildKeysForZipContents(keys, zipContents);
            try (ZipFile zipFile = new ZipFile(zipContents.zipFile)) {
                for (ModulesKey entry : keys.values()) {
                    if (AbstractAdditionalDependencyInfo.DEBUG) {
                        System.out.println("Loading: " + entry);
                    }
                    if (monitor.isCanceled()) {
                        break;
                    }
                    final ModulesKeyForZip z = (ModulesKeyForZip) entry;
                    if (!z.isFile) {
                        continue;
                    }
                    queue.put(new Command(entry, new IBufferFiller() {

                        @Override
                        public void fillBuffer(FastStringBuffer bufFileContents) {
                            try (InputStream stream = zipFile.getInputStream(zipFile.getEntry(z.zipModulePath))) {
                                fill(bufFileContents, stream);
                            } catch (Exception e) {
                                Log.log(e);
                            }
                        }
                    }));
                }
            } catch (Exception e) {
                Log.log(e);
            }
        }
    } finally {
        for (int i = 0; i < searchers; i++) {
            // add it to wait for the thread to finish.
            queue.add(new Command());
        }
    }
    int j = 0;
    while (true) {
        j++;
        boolean liveFound = false;
        for (Thread t : threads) {
            if (t.isAlive()) {
                liveFound = true;
                break;
            }
        }
        if (liveFound) {
            if (j % 50 == 0) {
                monitor.setTaskName("Searching references...");
                monitor.worked(1);
            }
            Thread.yield();
        } else {
            break;
        }
    }
    return ret;
}