Examples with ModulesFoundStructure org.python.pydev.ast.codecompletion.revisited.ModulesFoundStructure used on opensource projects

Example 1 with ModulesFoundStructure

use of org.python.pydev.ast.codecompletion.revisited.ModulesFoundStructure in project Pydev by fabioz.

the class InterpreterInfoBuilder method syncInfoToPythonPath.

public BuilderResult syncInfoToPythonPath(IProgressMonitor monitor, PythonPathHelper pythonPathHelper, AbstractAdditionalDependencyInfo additionalInfo, IModulesManager modulesManager, InterpreterInfo info) {
    if (monitor == null) {
        monitor = new NullProgressMonitor();
    }
    if (DebugSettings.DEBUG_INTERPRETER_AUTO_UPDATE) {
        org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "--- Start run");
    }
    BuilderResult ret = checkEarlyReturn(monitor, info);
    if (ret != BuilderResult.OK) {
        return ret;
    }
    ModulesFoundStructure modulesFound = pythonPathHelper.getModulesFoundStructure(null, monitor);
    ret = checkEarlyReturn(monitor, info);
    if (ret != BuilderResult.OK) {
        return ret;
    }
    PyPublicTreeMap<ModulesKey, ModulesKey> keysFound = ModulesManager.buildKeysFromModulesFound(monitor, modulesFound);
    if (DebugSettings.DEBUG_INTERPRETER_AUTO_UPDATE) {
        org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, StringUtils.format("Found: %s modules", keysFound.size()));
    }
    ret = checkEarlyReturn(monitor, info);
    if (ret != BuilderResult.OK) {
        return ret;
    }
    try {
        if (info != null) {
            String[] builtins = info.getBuiltins();
            // as we have to get the completions for all builtin modules from the shell.
            if (builtins != null) {
                for (int i = 0; i < builtins.length; i++) {
                    String name = builtins[i];
                    final ModulesKey k = new ModulesKey(name, null);
                    // Note that it'll override source modules!
                    keysFound.put(k, k);
                }
            }
        }
        synchronized (additionalInfo.updateKeysLock) {
            // Use a lock (if we have more than one builder updating we could get into a racing condition here).
            // Important: do the diff only after the builtins are added (otherwise the modules manager may become wrong)!
            Tuple<List<ModulesKey>, List<ModulesKey>> diffModules = modulesManager.diffModules(keysFound);
            if (diffModules.o1.size() > 0 || diffModules.o2.size() > 0) {
                if (DebugSettings.DEBUG_INTERPRETER_AUTO_UPDATE) {
                    org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, StringUtils.format("Diff modules. Added: %s Removed: %s", diffModules.o1, diffModules.o2));
                }
                // Update the modules manager itself (just pass all the keys as that should be fast)
                if (modulesManager instanceof SystemModulesManager) {
                    ((SystemModulesManager) modulesManager).updateKeysAndSave(keysFound);
                } else {
                    for (ModulesKey newEntry : diffModules.o1) {
                        modulesManager.addModule(newEntry);
                    }
                    modulesManager.removeModules(diffModules.o2);
                }
            }
            additionalInfo.updateKeysIfNeededAndSave(keysFound, info, monitor);
        }
    } catch (Exception e) {
        Log.log(e);
    }
    if (DebugSettings.DEBUG_INTERPRETER_AUTO_UPDATE) {
        org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "--- End Run");
    }
    return BuilderResult.OK;
}

Example 2 with ModulesFoundStructure

use of org.python.pydev.ast.codecompletion.revisited.ModulesFoundStructure in project Pydev by fabioz.

the class PythonModuleResolverExtensionPointTest method testGetModulesFoundStructureWithResolver.

/**
 * Checks that participants can explicitly curtail the results collected by
 * {@link PythonPathHelper#getModulesFoundStructure(IProject, IProgressMonitor)}.
 */
public void testGetModulesFoundStructureWithResolver() {
    PythonPathHelper helper = new PythonPathHelper();
    boolean isPython3Test = false;
    String path = TestDependent.getCompletePythonLib(true, isPython3Test) + "|" + TestDependent.TEST_PYSRC_TESTING_LOC;
    helper.setPythonPath(path);
    setTestingModuleResolver(new IPythonModuleResolver() {

        @Override
        public String resolveModule(IProject project, IPath moduleLocation, List<IPath> baseLocations) {
            if (moduleLocation.equals(Path.fromOSString("/root/x/y.py"))) {
                return "x.y";
            }
            return null;
        }

        @Override
        public Collection<IPath> findAllModules(IProject project, IProgressMonitor monitor) {
            List<IPath> modules = new ArrayList<>();
            modules.add(Path.fromOSString("/root/x/y.py"));
            return modules;
        }
    });
    ModulesFoundStructure modulesFoundStructure = helper.getModulesFoundStructure(null, null);
    Map<File, String> regularModules = modulesFoundStructure.regularModules;
    assertEquals(1, regularModules.keySet().size());
    assertEquals(new File("/root/x/y.py"), regularModules.keySet().iterator().next());
    assertEquals("x.y", regularModules.values().iterator().next());
}

Example 3 with ModulesFoundStructure

use of org.python.pydev.ast.codecompletion.revisited.ModulesFoundStructure in project Pydev by fabioz.

the class ReferenceSearches method search.

@Override
public List<ModulesKey> search(IProject project, OrderedMap<String, Set<String>> fieldNameToValues, IProgressMonitor monitor) {
    final List<ModulesKey> ret = new ArrayList<ModulesKey>();
    AbstractAdditionalDependencyInfo abstractAdditionalDependencyInfo = this.abstractAdditionalDependencyInfo.get();
    if (abstractAdditionalDependencyInfo == null) {
        Log.log("AbstractAdditionalDependencyInfo already collected!");
        return ret;
    }
    final NullProgressMonitor nullMonitor = new NullProgressMonitor();
    Set<String> pythonPathFolders = abstractAdditionalDependencyInfo.getPythonPathFolders();
    LinkedBlockingQueue<Command> queue = new LinkedBlockingQueue<>();
    int searchers = Runtime.getRuntime().availableProcessors();
    // The 'ret' should be filled with the module keys where the tokens are found.
    final Object retLock = new Object();
    // Create 2 consumers
    Thread[] threads = new Thread[searchers];
    for (int i = 0; i < searchers; i++) {
        Searcher searcher = new Searcher(queue, fieldNameToValues.get(IReferenceSearches.FIELD_CONTENTS), ret, retLock);
        // Spawn a thread to do the search while we load the contents.
        Thread t = new Thread(searcher);
        threads[i] = t;
        t.start();
    }
    try {
        PythonPathHelper pythonPathHelper = new PythonPathHelper();
        pythonPathHelper.setPythonPath(new ArrayList<String>(pythonPathFolders));
        ModulesFoundStructure modulesFound = pythonPathHelper.getModulesFoundStructure(project, nullMonitor);
        int totalSteps = modulesFound.regularModules.size() + modulesFound.zipContents.size();
        monitor.beginTask("Get modules with token in: " + abstractAdditionalDependencyInfo.getUIRepresentation(), totalSteps);
        PyPublicTreeMap<ModulesKey, ModulesKey> keys = new PyPublicTreeMap<>();
        // no point in searching dlls.
        boolean includeOnlySourceModules = true;
        ModulesManager.buildKeysForRegularEntries(nullMonitor, modulesFound, keys, includeOnlySourceModules);
        // Get from regular files found
        for (ModulesKey entry : keys.values()) {
            if (entry instanceof ModulesKeyForFolder) {
                continue;
            }
            if (monitor.isCanceled()) {
                break;
            }
            if (AbstractAdditionalDependencyInfo.DEBUG) {
                System.out.println("Loading: " + entry);
            }
            final File file = entry.file;
            try {
                queue.put(new Command(entry, new IBufferFiller() {

                    @Override
                    public void fillBuffer(FastStringBuffer bufFileContents) {
                        try (FileInputStream stream = new FileInputStream(file)) {
                            fill(bufFileContents, stream);
                        } catch (Exception e) {
                            Log.log(e);
                        }
                    }
                }));
            } catch (InterruptedException e) {
                Log.log(e);
            }
        }
        // Get from zip files found
        List<ZipContents> allZipsZipContents = modulesFound.zipContents;
        for (ZipContents zipContents : allZipsZipContents) {
            keys.clear();
            if (monitor.isCanceled()) {
                break;
            }
            ModulesManager.buildKeysForZipContents(keys, zipContents);
            try (ZipFile zipFile = new ZipFile(zipContents.zipFile)) {
                for (ModulesKey entry : keys.values()) {
                    if (AbstractAdditionalDependencyInfo.DEBUG) {
                        System.out.println("Loading: " + entry);
                    }
                    if (monitor.isCanceled()) {
                        break;
                    }
                    final ModulesKeyForZip z = (ModulesKeyForZip) entry;
                    if (!z.isFile) {
                        continue;
                    }
                    queue.put(new Command(entry, new IBufferFiller() {

                        @Override
                        public void fillBuffer(FastStringBuffer bufFileContents) {
                            try (InputStream stream = zipFile.getInputStream(zipFile.getEntry(z.zipModulePath))) {
                                fill(bufFileContents, stream);
                            } catch (Exception e) {
                                Log.log(e);
                            }
                        }
                    }));
                }
            } catch (Exception e) {
                Log.log(e);
            }
        }
    } finally {
        for (int i = 0; i < searchers; i++) {
            // add it to wait for the thread to finish.
            queue.add(new Command());
        }
    }
    int j = 0;
    while (true) {
        j++;
        boolean liveFound = false;
        for (Thread t : threads) {
            if (t.isAlive()) {
                liveFound = true;
                break;
            }
        }
        if (liveFound) {
            if (j % 50 == 0) {
                monitor.setTaskName("Searching references...");
                monitor.worked(1);
            }
            Thread.yield();
        } else {
            break;
        }
    }
    return ret;
}