Examples with ModulesKey org.python.pydev.core.ModulesKey used on opensource projects

Example 56 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class ReferenceSearches method search.

@Override
public List<ModulesKey> search(IProject project, OrderedMap<String, Set<String>> fieldNameToValues, IProgressMonitor monitor) {
    final List<ModulesKey> ret = new ArrayList<ModulesKey>();
    AbstractAdditionalDependencyInfo abstractAdditionalDependencyInfo = this.abstractAdditionalDependencyInfo.get();
    if (abstractAdditionalDependencyInfo == null) {
        Log.log("AbstractAdditionalDependencyInfo already collected!");
        return ret;
    }
    final NullProgressMonitor nullMonitor = new NullProgressMonitor();
    Set<String> pythonPathFolders = abstractAdditionalDependencyInfo.getPythonPathFolders();
    LinkedBlockingQueue<Command> queue = new LinkedBlockingQueue<>();
    int searchers = Runtime.getRuntime().availableProcessors();
    // The 'ret' should be filled with the module keys where the tokens are found.
    final Object retLock = new Object();
    // Create 2 consumers
    Thread[] threads = new Thread[searchers];
    for (int i = 0; i < searchers; i++) {
        Searcher searcher = new Searcher(queue, fieldNameToValues.get(IReferenceSearches.FIELD_CONTENTS), ret, retLock);
        // Spawn a thread to do the search while we load the contents.
        Thread t = new Thread(searcher);
        threads[i] = t;
        t.start();
    }
    try {
        PythonPathHelper pythonPathHelper = new PythonPathHelper();
        pythonPathHelper.setPythonPath(new ArrayList<String>(pythonPathFolders));
        ModulesFoundStructure modulesFound = pythonPathHelper.getModulesFoundStructure(project, nullMonitor);
        int totalSteps = modulesFound.regularModules.size() + modulesFound.zipContents.size();
        monitor.beginTask("Get modules with token in: " + abstractAdditionalDependencyInfo.getUIRepresentation(), totalSteps);
        PyPublicTreeMap<ModulesKey, ModulesKey> keys = new PyPublicTreeMap<>();
        // no point in searching dlls.
        boolean includeOnlySourceModules = true;
        ModulesManager.buildKeysForRegularEntries(nullMonitor, modulesFound, keys, includeOnlySourceModules);
        // Get from regular files found
        for (ModulesKey entry : keys.values()) {
            if (entry instanceof ModulesKeyForFolder) {
                continue;
            }
            if (monitor.isCanceled()) {
                break;
            }
            if (AbstractAdditionalDependencyInfo.DEBUG) {
                System.out.println("Loading: " + entry);
            }
            final File file = entry.file;
            try {
                queue.put(new Command(entry, new IBufferFiller() {

                    @Override
                    public void fillBuffer(FastStringBuffer bufFileContents) {
                        try (FileInputStream stream = new FileInputStream(file)) {
                            fill(bufFileContents, stream);
                        } catch (Exception e) {
                            Log.log(e);
                        }
                    }
                }));
            } catch (InterruptedException e) {
                Log.log(e);
            }
        }
        // Get from zip files found
        List<ZipContents> allZipsZipContents = modulesFound.zipContents;
        for (ZipContents zipContents : allZipsZipContents) {
            keys.clear();
            if (monitor.isCanceled()) {
                break;
            }
            ModulesManager.buildKeysForZipContents(keys, zipContents);
            try (ZipFile zipFile = new ZipFile(zipContents.zipFile)) {
                for (ModulesKey entry : keys.values()) {
                    if (AbstractAdditionalDependencyInfo.DEBUG) {
                        System.out.println("Loading: " + entry);
                    }
                    if (monitor.isCanceled()) {
                        break;
                    }
                    final ModulesKeyForZip z = (ModulesKeyForZip) entry;
                    if (!z.isFile) {
                        continue;
                    }
                    queue.put(new Command(entry, new IBufferFiller() {

                        @Override
                        public void fillBuffer(FastStringBuffer bufFileContents) {
                            try (InputStream stream = zipFile.getInputStream(zipFile.getEntry(z.zipModulePath))) {
                                fill(bufFileContents, stream);
                            } catch (Exception e) {
                                Log.log(e);
                            }
                        }
                    }));
                }
            } catch (Exception e) {
                Log.log(e);
            }
        }
    } finally {
        for (int i = 0; i < searchers; i++) {
            // add it to wait for the thread to finish.
            queue.add(new Command());
        }
    }
    int j = 0;
    while (true) {
        j++;
        boolean liveFound = false;
        for (Thread t : threads) {
            if (t.isAlive()) {
                liveFound = true;
                break;
            }
        }
        if (liveFound) {
            if (j % 50 == 0) {
                monitor.setTaskName("Searching references...");
                monitor.worked(1);
            }
            Thread.yield();
        } else {
            break;
        }
    }
    return ret;
}

Example 57 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class AdditionalProjectInterpreterInfo method getModulesWithToken.

/**
 * @param token the token we want to search for (must be an exact match). Only tokens which are valid identifiers
 * may be searched (i.e.: no dots in it or anything alike).
 *
 * @return List<ModulesKey> a list with all the modules that contains the passed token.
 *
 * Note: if it's a name with dots, we'll split it and search for each one.
 */
public List<ModulesKey> getModulesWithToken(String token, IProgressMonitor monitor) throws OperationCanceledException {
    NullProgressMonitor nullMonitor = new NullProgressMonitor();
    if (monitor == null) {
        monitor = nullMonitor;
    }
    int length = token.length();
    if (token == null || length == 0) {
        return new ArrayList<>();
    }
    for (int i = 0; i < length; i++) {
        char c = token.charAt(i);
        if (!Character.isJavaIdentifierPart(c) && c != '.') {
            throw new RuntimeException(StringUtils.format("Token: %s is not a valid token to search for.", token));
        }
    }
    StringUtils.checkTokensValidForWildcardQuery(token);
    OrderedMap<String, Set<String>> fieldNameToValues = new OrderedMap<>();
    Set<String> split = new HashSet<>();
    for (String s : StringUtils.splitForIndexMatching(token)) {
        // We need to search in lowercase (we only index case-insensitive).
        split.add(s.toLowerCase());
    }
    fieldNameToValues.put(IReferenceSearches.FIELD_CONTENTS, split);
    List<ModulesKey> search = getReferenceSearches().search(project, fieldNameToValues, monitor);
    // }
    return search;
}

Example 58 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class ClassHierarchySearchTest method assertIsIn.

private HierarchyNodeModel assertIsIn(String name, String modName, List<HierarchyNodeModel> parents) {
    FastStringBuffer available = new FastStringBuffer();
    for (HierarchyNodeModel model : parents) {
        available.append(model.name).append(" - ").append(model.moduleName);
        if (model.name.equals(name)) {
            if (modName == null) {
                return model;
            } else if (model.moduleName.equals(modName) || model.moduleName.startsWith(modName)) {
                return model;
            }
        }
    }
    try {
        RefactorerFindReferences references = new RefactorerFindReferences();
        RefactoringRequest request = new RefactoringRequest(null, null, nature);
        request.qualifier = name;
        List<Tuple<List<ModulesKey>, IPythonNature>> findPossibleReferences = references.findPossibleReferences(request);
        String errorMsg = "Unable to find node with name:" + name + " mod:" + modName + "\nAvailable:" + available + "\n\nPythonpath: " + nature.getPythonPathNature().getOnlyProjectPythonPathStr(true) + "\n" + "Found possible references: " + StringUtils.join("\n", findPossibleReferences);
        fail(errorMsg);
    } catch (CoreException e) {
        throw new RuntimeException(e);
    }
    return null;
}