Examples with ModulesKey org.python.pydev.core.ModulesKey used on opensource projects

Example 16 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class SystemModulesManager method load.

@Override
public void load() throws IOException {
    final File workspaceMetadataFile = getIoDirectory();
    ModulesManager.loadFromFile(this, workspaceMetadataFile);
    DeltaSaver<ModulesKey> d = this.deltaSaver = new DeltaSaver<ModulesKey>(this.getIoDirectory(), "v1_sys_astdelta", readFromFileMethod, toFileMethod);
    // process the current deltas (clears current deltas automatically and saves it when the processing is concluded)
    d.processDeltas(this);
}

Example 17 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class SystemModulesManager method getBuiltinModule.

@Override
public AbstractModule getBuiltinModule(String name, boolean dontSearchInit, IModuleRequestState moduleRequest) {
    AbstractModule n = null;
    // check for supported builtins these don't have files associated.
    // they are the first to be passed because the user can force a module to be builtin, because there
    // is some information that is only useful when you have builtins, such as os and wxPython (those can
    // be source modules, but they have so much runtime info that it is almost impossible to get useful information
    // from statically analyzing them).
    String[] builtins = getBuiltins();
    if (builtins == null || this.info == null) {
        // still on startup
        return null;
    }
    // for temporary access (so that we don't generate many instances of it)
    ModulesKey keyForCacheAccess = new ModulesKey(null, null);
    // A different choice for users that want more complete information on the libraries they're dealing
    // with is using predefined modules. Those will
    File predefinedModule = this.info.getPredefinedModule(name, moduleRequest);
    boolean found = predefinedModule != null && predefinedModule.exists();
    if (!found && !name.endsWith(".__init__")) {
        final String nameWithInit = name + ".__init__";
        predefinedModule = this.info.getPredefinedModule(nameWithInit, moduleRequest);
        found = predefinedModule != null && predefinedModule.exists();
        if (found) {
            name = nameWithInit;
        }
    }
    if (found) {
        keyForCacheAccess.name = name;
        keyForCacheAccess.file = predefinedModule;
        n = cachePredefined.getObj(keyForCacheAccess, this);
        if ((n instanceof PredefinedSourceModule)) {
            PredefinedSourceModule predefinedSourceModule = (PredefinedSourceModule) n;
            if (predefinedSourceModule.isSynched()) {
                return n;
            }
        // otherwise (not PredefinedSourceModule or not synched), just keep going to create
        // it as a predefined source module
        }
        boolean tryToParse = true;
        Long lastModified = null;
        if (predefinedFilesNotParsedToTimestamp == null) {
            predefinedFilesNotParsedToTimestamp = new HashMap<File, Long>();
        } else {
            Long lastTimeChanged = predefinedFilesNotParsedToTimestamp.get(predefinedModule);
            if (lastTimeChanged != null) {
                lastModified = FileUtils.lastModified(predefinedModule);
                if (lastTimeChanged.equals(lastModified)) {
                    tryToParse = false;
                } else {
                    predefinedFilesNotParsedToTimestamp.remove(predefinedModule);
                }
            }
        }
        if (tryToParse) {
            IDocument doc;
            try {
                doc = FileUtilsFileBuffer.getDocFromFile(predefinedModule);
                IGrammarVersionProvider provider = new IGrammarVersionProvider() {

                    @Override
                    public int getGrammarVersion() throws MisconfigurationException {
                        // Always Python 3.0 here
                        return IGrammarVersionProvider.LATEST_GRAMMAR_PY3_VERSION;
                    }

                    @Override
                    public AdditionalGrammarVersionsToCheck getAdditionalGrammarVersions() throws MisconfigurationException {
                        return null;
                    }
                };
                ParseOutput obj = PyParser.reparseDocument(new PyParser.ParserInfo(doc, provider, name, predefinedModule));
                if (obj.error != null) {
                    if (lastModified == null) {
                        lastModified = FileUtils.lastModified(predefinedModule);
                    }
                    predefinedFilesNotParsedToTimestamp.put(predefinedModule, lastModified);
                    Log.log("Unable to parse: " + predefinedModule, obj.error);
                } else if (obj.ast != null) {
                    SimpleNode ast = (SimpleNode) obj.ast;
                    if ("builtins".equals(name) || "__builtin__".equals(name)) {
                        // None/False/True must be added as they're not there by default.
                        if (ast instanceof Module) {
                            Module module = (Module) ast;
                            PyAstFactory astFactory = new PyAstFactory(new AdapterPrefs("\n", info.getModulesManager().getNature()));
                            module.body = ArrayUtils.concatArrays(module.body, new stmtType[] { astFactory.createAssign(astFactory.createStoreName("None"), astFactory.createNone()), astFactory.createAssign(astFactory.createStoreName("False"), astFactory.createFalse()), astFactory.createAssign(astFactory.createStoreName("True"), astFactory.createTrue()), astFactory.createAssign(astFactory.createStoreName("__builtins__"), astFactory.createName("Any")) });
                        }
                    }
                    n = new PredefinedSourceModule(name, predefinedModule, ast, obj.error);
                    cachePredefined.add(keyForCacheAccess, n, this);
                    // doAddSingleModule(keyForCacheAccess, n);
                    return n;
                }
            // keep on going
            } catch (Throwable e) {
                Log.log(e);
            }
        }
    }
    boolean foundStartingWithBuiltin = false;
    FastStringBuffer buffer = null;
    for (int i = 0; i < builtins.length; i++) {
        String forcedBuiltin = builtins[i];
        if (name.startsWith(forcedBuiltin)) {
            if (name.length() > forcedBuiltin.length() && name.charAt(forcedBuiltin.length()) == '.') {
                foundStartingWithBuiltin = true;
                keyForCacheAccess.name = name;
                n = cache.getObj(keyForCacheAccess, this);
                if (n == null && dontSearchInit == false) {
                    if (buffer == null) {
                        buffer = new FastStringBuffer();
                    } else {
                        buffer.clear();
                    }
                    keyForCacheAccess.name = buffer.append(name).append(".__init__").toString();
                    n = cache.getObj(keyForCacheAccess, this);
                }
                if (n instanceof EmptyModule || n instanceof SourceModule) {
                    // it is actually found as a source module, so, we have to 'coerce' it to a compiled module
                    n = new CompiledModule(name, this, this.getNature());
                    doAddSingleModule(new ModulesKey(n.getName(), null), n);
                    return n;
                }
            }
            if (name.equals(forcedBuiltin)) {
                keyForCacheAccess.name = name;
                n = cache.getObj(keyForCacheAccess, this);
                if (n == null || n instanceof EmptyModule || n instanceof SourceModule) {
                    // still not created or not defined as compiled module (as it should be)
                    n = new CompiledModule(name, this, this.getNature());
                    doAddSingleModule(new ModulesKey(n.getName(), null), n);
                    return n;
                }
            }
            if (n instanceof CompiledModule) {
                return n;
            }
        }
    }
    if (foundStartingWithBuiltin) {
        if (builtinsNotConsidered.getObj(name) != null) {
            return null;
        }
        // ok, just add it if it is some module that actually exists
        n = new CompiledModule(name, this, this.getNature());
        TokensList globalTokens = n.getGlobalTokens();
        // the module to see its return values (because that's slow)
        if (globalTokens.size() > 0 && contains(globalTokens, "__file__")) {
            doAddSingleModule(new ModulesKey(name, null), n);
            return n;
        } else {
            builtinsNotConsidered.add(name, name);
            return null;
        }
    }
    return null;
}

Example 18 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class NatureGroup method calculateChildren.

@Override
protected void calculateChildren() throws MisconfigurationException {
    ICodeCompletionASTManager astManager = nature.getAstManager();
    if (astManager == null) {
        addLeaf("AST manager == null (should happen only in the plugin initialization) -- skipping other checks.");
        return;
    }
    IModulesManager projectModulesManager = astManager.getModulesManager();
    if (projectModulesManager == null) {
        addLeaf("Modules manager == null (should happen only in the plugin initialization) -- skipping other checks.");
        return;
    }
    PythonPathHelper pythonPathHelper = (PythonPathHelper) projectModulesManager.getPythonPathHelper();
    if (pythonPathHelper == null) {
        addLeaf("PythonPathHelper == null (should happen only in the plugin initialization) -- skipping other checks.");
        return;
    }
    List<String> pythonpath = pythonPathHelper.getPythonpath();
    for (String s : pythonpath) {
        addLeaf("PYTHONPATH: " + s);
    }
    HashSet<ModulesKey> expectedModuleNames = new HashSet<ModulesKey>();
    for (String string : pythonpath) {
        File file = new File(string);
        if (file.isDirectory()) {
            // TODO: Handle zip file modules!
            Collection<PyFileInfo> modulesBelow = PythonPathHelper.getModulesBelow(file, new NullProgressMonitor(), pythonpath).getFoundPyFileInfos();
            for (PyFileInfo fileInfo : modulesBelow) {
                File moduleFile = fileInfo.getFile();
                String modName = pythonPathHelper.resolveModule(FileUtils.getFileAbsolutePath(moduleFile), true, nature.getProject());
                if (modName != null) {
                    expectedModuleNames.add(new ModulesKey(modName, moduleFile));
                } else {
                    if (PythonPathHelper.isValidModuleLastPart(StringUtils.stripExtension((moduleFile.getName())))) {
                        addLeaf(StringUtils.format("Unable to resolve module: %s (gotten null module name)", moduleFile));
                    }
                }
            }
        } else {
            if (!file.exists()) {
                addLeaf(StringUtils.format("File %s is referenced in the pythonpath but does not exist.", file));
            } else {
                addLeaf(org.python.pydev.shared_core.string.StringUtils.format("File %s not handled (TODO: Fix zip files support in the viewer).", file));
            }
        }
    }
    IntegrityInfo info = new IntegrityInfo();
    info.nature = nature;
    ModulesKey[] onlyDirectModules = projectModulesManager.getOnlyDirectModules();
    TreeSet<ModulesKey> inModulesManager = new TreeSet<ModulesKey>(Arrays.asList(onlyDirectModules));
    Set<String> allAdditionalInfoModuleNames = new TreeSet<String>();
    List<Tuple<AbstractAdditionalTokensInfo, IPythonNature>> additionalInfoAndNature = AdditionalProjectInterpreterInfo.getAdditionalInfoAndNature(nature, false, false, false);
    AbstractAdditionalTokensInfo additionalProjectInfo;
    if (additionalInfoAndNature.size() == 0) {
        addChild(new LeafElement(this, "No additional infos found (1 expected) -- skipping other checks."));
        return;
    } else {
        if (additionalInfoAndNature.size() > 1) {
            addChild(new LeafElement(this, StringUtils.format("%s additional infos found (only 1 expected) -- continuing checks but analysis may be wrong.", additionalInfoAndNature.size())));
        }
        additionalProjectInfo = additionalInfoAndNature.get(0).o1;
        allAdditionalInfoModuleNames.addAll(additionalProjectInfo.getAllModulesWithTokens());
    }
    for (ModulesKey key : inModulesManager) {
        if (!expectedModuleNames.contains(key)) {
            info.modulesNotInDisk.add(key);
            addChild(new LeafElement(this, StringUtils.format("%s exists in memory but not in the disk.", key)));
        }
    }
    ModulesKey tempKey = new ModulesKey(null, null);
    for (String s : allAdditionalInfoModuleNames) {
        tempKey.name = s;
        if (!expectedModuleNames.contains(tempKey)) {
            info.additionalModulesNotInDisk.add(s);
            addChild(new LeafElement(this, StringUtils.format("%s exists in the additional info but not in the disk.", s)));
        }
    }
    for (ModulesKey key : expectedModuleNames) {
        boolean isInModulesManager = inModulesManager.contains(key);
        if (!isInModulesManager) {
            info.modulesNotInMemory.add(key);
            addChild(new LeafElement(this, StringUtils.format("%s exists in the disk but not in memory.", key)));
        }
        if (!allAdditionalInfoModuleNames.contains(key.name)) {
            try {
                AbstractModule mod = AbstractModule.createModule(key.name, key.file, info.nature, true);
                if (!(mod instanceof SourceModule)) {
                    continue;
                }
                SourceModule module = (SourceModule) mod;
                if (module == null || module.getAst() == null) {
                    addChild(new LeafElement(this, StringUtils.format("Warning: cannot parse: %s - %s (so, it's ok not having additional info on it)", key.name, key.file)));
                } else {
                    try {
                        Iterator<ASTEntry> innerEntriesForAST = AbstractAdditionalDependencyInfo.getInnerEntriesForAST(module.getAst()).o2;
                        if (innerEntriesForAST.hasNext()) {
                            info.moduleNotInAdditionalInfo.add(module);
                            addChild(new LeafElement(this, StringUtils.format("The additional info index of the module: %s is not updated.", key.name)));
                        }
                    } catch (Exception e) {
                        addChild(new LeafElement(this, StringUtils.format("Unexpected error happened on: %s - %s: %s", key.name, key.file, e.getMessage())));
                    }
                }
            } catch (IOException e) {
                // OK, it cannot be parsed, so, we cannot generate its info
                addChild(new LeafElement(this, StringUtils.format("Warning: cannot parse: %s - %s (so, it's ok not having additional info on it)", key.name, key.file)));
            }
        }
    }
    // modules manager
    if (info.modulesNotInDisk.size() > 0) {
        for (ModulesKey m : info.modulesNotInDisk) {
            addChild(new LeafElement(this, StringUtils.format("FIX: Removing from modules manager: %s", m)));
        }
        projectModulesManager.removeModules(info.modulesNotInDisk);
    }
    for (ModulesKey key : info.modulesNotInMemory) {
        addChild(new LeafElement(this, "FIX: Adding to modules manager: " + key));
        projectModulesManager.addModule(key);
    }
    // additional info
    for (String s : info.additionalModulesNotInDisk) {
        addChild(new LeafElement(this, StringUtils.format("FIX: Removing from additional info: %s", s)));
        additionalProjectInfo.removeInfoFromModule(s, true);
    }
    for (SourceModule mod : info.moduleNotInAdditionalInfo) {
        addChild(new LeafElement(this, StringUtils.format("FIX: Adding to additional info: %s", mod.getName())));
        additionalProjectInfo.addAstInfo(mod.getAst(), mod.getModulesKey(), true);
    }
}

Example 19 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class PySearchIndexQuery method run.

@Override
public IStatus run(IProgressMonitor monitor) throws OperationCanceledException {
    SearchIndexResult searchResult = (SearchIndexResult) getSearchResult();
    // Remove all so that we don't get duplicates on a search refresh.
    searchResult.removeAll();
    StringMatcherWithIndexSemantics stringMatcher = createStringMatcher();
    Set<String> moduleNamesFilter = scopeAndData.getModuleNamesFilter();
    OrderedMap<String, Set<String>> fieldNameToValues = new OrderedMap<>();
    if (moduleNamesFilter != null && !moduleNamesFilter.isEmpty()) {
        fieldNameToValues.put(IReferenceSearches.FIELD_MODULE_NAME, moduleNamesFilter);
    }
    Set<String> split = makeTextFieldPatternsToSearchFromText();
    fieldNameToValues.put(IReferenceSearches.FIELD_CONTENTS, split);
    final List<IPythonNature> pythonNatures = PyScopeAndData.getPythonNatures(scopeAndData);
    monitor.beginTask("Search indexes", pythonNatures.size());
    try {
        for (IPythonNature nature : pythonNatures) {
            AbstractAdditionalDependencyInfo info;
            try {
                info = AdditionalProjectInterpreterInfo.getAdditionalInfoForProject(nature);
            } catch (MisconfigurationException e) {
                Log.log(e);
                continue;
            }
            IReferenceSearches referenceSearches = info.getReferenceSearches();
            List<ModulesKey> search = referenceSearches.search(nature.getProject(), fieldNameToValues, new SubProgressMonitor(monitor, 1));
            IFile workspaceFile;
            for (ModulesKey modulesKey : search) {
                File file = modulesKey.file;
                if (file == null || !file.exists()) {
                    Log.logInfo(StringUtils.format("Ignoring: %s. File no longer exists.", file));
                }
                workspaceFile = FindWorkspaceFiles.getWorkspaceFile(file, nature.getProject());
                if (workspaceFile == null) {
                    Log.logInfo(StringUtils.format("Ignoring: %s. Unable to resolve to a file in the Eclipse workspace.", file));
                    continue;
                }
                IDocument doc = FileUtilsFileBuffer.getDocFromResource(workspaceFile);
                String text = doc.get();
                createMatches(doc, text, stringMatcher, workspaceFile, searchResult, modulesKey);
            }
        }
    } finally {
        monitor.done();
    }
    return Status.OK_STATUS;
}

Example 20 with ModulesKey

use of org.python.pydev.core.ModulesKey in project Pydev by fabioz.

the class SearchIndexQueryTest method testSearchQuery.

public void testSearchQuery() throws Exception {
    PySearchIndexQuery query = new PySearchIndexQuery("my");
    String text = "rara\nmy\nnomyno\nmy";
    IDocument doc = new Document(text);
    IFile f = new AdditionalInfoFileStub("stub") {

        @Override
        public long getModificationStamp() {
            return 0;
        }
    };
    AbstractTextSearchResult searchResult = new PySearchResult(null);
    query.createMatches(doc, text, query.createStringMatcher(), f, searchResult, new ModulesKey("my", null));
    assertEquals(2, searchResult.getMatchCount());
}