Examples with PyPublicTreeMap org.python.pydev.ast.codecompletion.revisited.PyPublicTreeMap used on opensource projects

Example 1 with PyPublicTreeMap

use of org.python.pydev.ast.codecompletion.revisited.PyPublicTreeMap in project Pydev by fabioz.

the class ReferenceSearches method search.

@Override
public List<ModulesKey> search(IProject project, OrderedMap<String, Set<String>> fieldNameToValues, IProgressMonitor monitor) {
    final List<ModulesKey> ret = new ArrayList<ModulesKey>();
    AbstractAdditionalDependencyInfo abstractAdditionalDependencyInfo = this.abstractAdditionalDependencyInfo.get();
    if (abstractAdditionalDependencyInfo == null) {
        Log.log("AbstractAdditionalDependencyInfo already collected!");
        return ret;
    }
    final NullProgressMonitor nullMonitor = new NullProgressMonitor();
    Set<String> pythonPathFolders = abstractAdditionalDependencyInfo.getPythonPathFolders();
    LinkedBlockingQueue<Command> queue = new LinkedBlockingQueue<>();
    int searchers = Runtime.getRuntime().availableProcessors();
    // The 'ret' should be filled with the module keys where the tokens are found.
    final Object retLock = new Object();
    // Create 2 consumers
    Thread[] threads = new Thread[searchers];
    for (int i = 0; i < searchers; i++) {
        Searcher searcher = new Searcher(queue, fieldNameToValues.get(IReferenceSearches.FIELD_CONTENTS), ret, retLock);
        // Spawn a thread to do the search while we load the contents.
        Thread t = new Thread(searcher);
        threads[i] = t;
        t.start();
    }
    try {
        PythonPathHelper pythonPathHelper = new PythonPathHelper();
        pythonPathHelper.setPythonPath(new ArrayList<String>(pythonPathFolders));
        ModulesFoundStructure modulesFound = pythonPathHelper.getModulesFoundStructure(project, nullMonitor);
        int totalSteps = modulesFound.regularModules.size() + modulesFound.zipContents.size();
        monitor.beginTask("Get modules with token in: " + abstractAdditionalDependencyInfo.getUIRepresentation(), totalSteps);
        PyPublicTreeMap<ModulesKey, ModulesKey> keys = new PyPublicTreeMap<>();
        // no point in searching dlls.
        boolean includeOnlySourceModules = true;
        ModulesManager.buildKeysForRegularEntries(nullMonitor, modulesFound, keys, includeOnlySourceModules);
        // Get from regular files found
        for (ModulesKey entry : keys.values()) {
            if (entry instanceof ModulesKeyForFolder) {
                continue;
            }
            if (monitor.isCanceled()) {
                break;
            }
            if (AbstractAdditionalDependencyInfo.DEBUG) {
                System.out.println("Loading: " + entry);
            }
            final File file = entry.file;
            try {
                queue.put(new Command(entry, new IBufferFiller() {

                    @Override
                    public void fillBuffer(FastStringBuffer bufFileContents) {
                        try (FileInputStream stream = new FileInputStream(file)) {
                            fill(bufFileContents, stream);
                        } catch (Exception e) {
                            Log.log(e);
                        }
                    }
                }));
            } catch (InterruptedException e) {
                Log.log(e);
            }
        }
        // Get from zip files found
        List<ZipContents> allZipsZipContents = modulesFound.zipContents;
        for (ZipContents zipContents : allZipsZipContents) {
            keys.clear();
            if (monitor.isCanceled()) {
                break;
            }
            ModulesManager.buildKeysForZipContents(keys, zipContents);
            try (ZipFile zipFile = new ZipFile(zipContents.zipFile)) {
                for (ModulesKey entry : keys.values()) {
                    if (AbstractAdditionalDependencyInfo.DEBUG) {
                        System.out.println("Loading: " + entry);
                    }
                    if (monitor.isCanceled()) {
                        break;
                    }
                    final ModulesKeyForZip z = (ModulesKeyForZip) entry;
                    if (!z.isFile) {
                        continue;
                    }
                    queue.put(new Command(entry, new IBufferFiller() {

                        @Override
                        public void fillBuffer(FastStringBuffer bufFileContents) {
                            try (InputStream stream = zipFile.getInputStream(zipFile.getEntry(z.zipModulePath))) {
                                fill(bufFileContents, stream);
                            } catch (Exception e) {
                                Log.log(e);
                            }
                        }
                    }));
                }
            } catch (Exception e) {
                Log.log(e);
            }
        }
    } finally {
        for (int i = 0; i < searchers; i++) {
            // add it to wait for the thread to finish.
            queue.add(new Command());
        }
    }
    int j = 0;
    while (true) {
        j++;
        boolean liveFound = false;
        for (Thread t : threads) {
            if (t.isAlive()) {
                liveFound = true;
                break;
            }
        }
        if (liveFound) {
            if (j % 50 == 0) {
                monitor.setTaskName("Searching references...");
                monitor.worked(1);
            }
            Thread.yield();
        } else {
            break;
        }
    }
    return ret;
}

Example 2 with PyPublicTreeMap

use of org.python.pydev.ast.codecompletion.revisited.PyPublicTreeMap in project Pydev by fabioz.

the class TreeIO method loadTreeFrom.

public static PyPublicTreeMap<String, Set<IInfo>> loadTreeFrom(final FastBufferedReader reader, final Map<Integer, String> dictionary, FastStringBuffer buf, ObjectsPoolMap objectsPoolMap, IPythonNature nature) throws IOException {
    PyPublicTreeMap<String, Set<IInfo>> tree = new PyPublicTreeMap<String, Set<IInfo>>();
    final int size = StringUtils.parsePositiveInt(reader.readLine());
    try {
        final Entry[] entries = new Entry[size];
        // note: the path (2nd int in record) is optional
        for (int iEntry = 0; iEntry < size; iEntry++) {
            buf.clear();
            FastStringBuffer readLine = reader.readLine();
            if (readLine == null || readLine.startsWith("-- ")) {
                throw new RuntimeException("Unexpected line: " + readLine);
            }
            char[] internalCharsArray = readLine.getInternalCharsArray();
            int length = readLine.length();
            String key = null;
            String infoName = null;
            String path = null;
            String file = null;
            int line = 0;
            int col = 0;
            int i = 0;
            OUT: for (; i < length; i++) {
                char c = internalCharsArray[i];
                switch(c) {
                    case '|':
                        key = ObjectsInternPool.internLocal(objectsPoolMap, buf.toString());
                        buf.clear();
                        i++;
                        break OUT;
                    default:
                        buf.appendResizeOnExc(c);
                }
            }
            int hashSize = 0;
            OUT2: for (; i < length; i++) {
                char c = internalCharsArray[i];
                switch(c) {
                    case '|':
                        hashSize = StringUtils.parsePositiveInt(buf);
                        buf.clear();
                        i++;
                        break OUT2;
                    default:
                        buf.appendResizeOnExc(c);
                }
            }
            HashSet<IInfo> set = new HashSet<IInfo>(hashSize);
            for (; i < length; i++) {
                char c = internalCharsArray[i];
                switch(c) {
                    case NAME_CHAR_SEPARATOR:
                        infoName = ObjectsInternPool.internLocal(objectsPoolMap, buf.toString());
                        buf.clear();
                        break;
                    case PATH_CHAR_SEPARATOR:
                        path = dictionary.get(StringUtils.parsePositiveInt(buf));
                        buf.clear();
                        break;
                    case FILE_CHAR_SEPARATOR:
                        file = dictionary.get(StringUtils.parsePositiveInt(buf));
                        buf.clear();
                        break;
                    case LINE_CHAR_SEPARATOR:
                        line = StringUtils.parsePositiveInt(buf);
                        buf.clear();
                        break;
                    case COL_CHAR_SEPARATOR:
                        col = StringUtils.parsePositiveInt(buf);
                        buf.clear();
                        break;
                    case END_IINFO_SEPARATOR:
                        int dictKey = StringUtils.parsePositiveInt(buf);
                        byte type = (byte) dictKey;
                        // leave only the 3 least significant bits there (this is the type -- value from 0 - 8).
                        type &= 0x07;
                        // the entry in the dict doesn't have the least significant bits there.
                        dictKey = (dictKey >> 3);
                        buf.clear();
                        String moduleDeclared = dictionary.get(dictKey);
                        if (moduleDeclared == null) {
                            throw new AssertionError("Unable to find key: " + dictKey);
                        }
                        if (infoName == null) {
                            throw new AssertionError("Info name may not be null. Line: " + line);
                        }
                        switch(type) {
                            case IInfo.CLASS_WITH_IMPORT_TYPE:
                                set.add(new ClassInfo(infoName, moduleDeclared, path, false, nature, file, line, col));
                                break;
                            case IInfo.METHOD_WITH_IMPORT_TYPE:
                                set.add(new FuncInfo(infoName, moduleDeclared, path, false, nature, file, line, col));
                                break;
                            case IInfo.ATTRIBUTE_WITH_IMPORT_TYPE:
                                set.add(new AttrInfo(infoName, moduleDeclared, path, false, nature, file, line, col));
                                break;
                            case IInfo.NAME_WITH_IMPORT_TYPE:
                                set.add(new NameInfo(infoName, moduleDeclared, path, false, nature, file, line, col));
                                break;
                            case IInfo.MOD_IMPORT_TYPE:
                                set.add(new ModInfo(infoName, false, nature, file, line, col));
                                break;
                            default:
                                Log.log("Unexpected type: " + type);
                        }
                        file = null;
                        infoName = null;
                        path = null;
                        file = null;
                        line = 0;
                        col = 0;
                        break;
                    default:
                        buf.appendResizeOnExc(c);
                }
            }
            entries[iEntry] = new MapEntry(key, set);
        }
        tree.buildFromSorted(size, new Iterator() {

            private int iNext;

            @Override
            public boolean hasNext() {
                return iNext > size;
            }

            @Override
            public Object next() {
                Object o = entries[iNext];
                iNext++;
                return o;
            }

            @Override
            public void remove() {
            }
        }, null, null);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    }
    return tree;
}