Examples with AbstractModule org.python.pydev.ast.codecompletion.revisited.modules.AbstractModule used on opensource projects

Example 1 with AbstractModule

use of org.python.pydev.ast.codecompletion.revisited.modules.AbstractModule in project Pydev by fabioz.

the class ForcedLibGroup method calculateChildren.

@Override
protected void calculateChildren() throws MisconfigurationException {
    SystemModulesManager m = (SystemModulesManager) this.interpreterInfo.getModulesManager();
    AbstractModule builtinModule = m.getBuiltinModule(forcedLib, true, new BaseModuleRequest(true));
    TokensList globalTokens = builtinModule.getGlobalTokens();
    ArrayList<LeafElement> lst = new ArrayList<LeafElement>();
    for (IterTokenEntry entry : globalTokens) {
        IToken iToken = entry.getToken();
        lst.add(new LeafElement(this, iToken.getRepresentation()));
    }
    Collections.sort(lst, new Comparator<LeafElement>() {

        @Override
        public int compare(LeafElement o1, LeafElement o2) {
            return o1.toString().compareTo(o2.toString());
        }
    });
    for (LeafElement leafElement : lst) {
        addChild(leafElement);
    }
}

Example 2 with AbstractModule

use of org.python.pydev.ast.codecompletion.revisited.modules.AbstractModule in project Pydev by fabioz.

the class ModulesManager method addModule.

@Override
public IModule addModule(final ModulesKey key) {
    AbstractModule ret = AbstractModule.createEmptyModule(key);
    doAddSingleModule(key, ret);
    return ret;
}

Example 3 with AbstractModule

use of org.python.pydev.ast.codecompletion.revisited.modules.AbstractModule in project Pydev by fabioz.

the class ModulesManager method getModule.

/**
 * This method returns the module that corresponds to the path passed as a parameter.
 *
 * @param name the name of the module we're looking for  (e.g.: mod1.mod2)
 * @param dontSearchInit is used in a negative form because initially it was isLookingForRelative, but
 * it actually defines if we should look in __init__ modules too, so, the name matches the old signature.
 *
 * NOTE: isLookingForRelative description was: when looking for relative imports, we don't check for __init__
 * @return the module represented by this name
 */
protected IModule getModule(boolean acceptCompiledModule, String name, IPythonNature nature, boolean dontSearchInit, IModuleRequestState moduleRequest) {
    synchronized (lockTemporaryModules) {
        SortedMap<Integer, IModule> map = temporaryModules.get(name);
        if (map != null && map.size() > 0) {
            if (DEBUG_TEMPORARY_MODULES) {
                System.out.println("Returning temporary module: " + name);
            }
            return map.get(map.lastKey());
        }
    }
    AbstractModule n = null;
    ModulesKey keyForCacheAccess = new ModulesKey(null, null);
    if (!dontSearchInit) {
        if (n == null) {
            keyForCacheAccess.name = (String) StringUtils.join(".", new String[] { name, "__init__" }, null);
            n = cache.getObj(keyForCacheAccess, this);
            if (n != null) {
                name = keyForCacheAccess.name;
            }
        }
    }
    if (n == null) {
        keyForCacheAccess.name = name;
        n = cache.getObj(keyForCacheAccess, this);
    }
    if (n instanceof SourceModule) {
        // ok, module exists, let's check if it is synched with the filesystem version...
        SourceModule s = (SourceModule) n;
        if (!s.isSynched()) {
            // change it for an empty and proceed as usual.
            n = (AbstractModule) addModule(createModulesKey(s.getName(), s.getFile()));
        }
    }
    if (n instanceof EmptyModule) {
        EmptyModule e = (EmptyModule) n;
        if (e.f != null) {
            if (!e.f.exists()) {
                // if the file does not exist anymore, just remove it.
                keyForCacheAccess.name = name;
                keyForCacheAccess.file = e.f;
                doRemoveSingleModule(keyForCacheAccess);
                n = null;
            } else {
                // file exists
                n = checkOverride(name, nature, n);
                if (n instanceof EmptyModule) {
                    // ok, handle case where the file is actually from a zip file...
                    if (e instanceof EmptyModuleForZip) {
                        EmptyModuleForZip emptyModuleForZip = (EmptyModuleForZip) e;
                        if (emptyModuleForZip.pathInZip.endsWith(".class") || !emptyModuleForZip.isFile) {
                            // handle java class... (if it's a class or a folder in a jar)
                            try {
                                if (createModuleFromJar != null) {
                                    n = createModuleFromJar.call(emptyModuleForZip, nature);
                                    n = decorateModule(n, nature);
                                } else {
                                    n = null;
                                }
                            } catch (Throwable e1) {
                                Log.log("Unable to create module from jar (note: JDT is required for Jython development): " + emptyModuleForZip + " project: " + (nature != null ? nature.getProject() : "null"), e1);
                                n = null;
                            }
                        } else if (FileTypesPreferences.isValidDll(emptyModuleForZip.pathInZip)) {
                            // .pyd
                            n = new CompiledModule(name, this, nature);
                            n = decorateModule(n, nature);
                        } else if (PythonPathHelper.isValidSourceFile(emptyModuleForZip.pathInZip)) {
                            // handle python file from zip... we have to create it getting the contents from the zip file
                            try {
                                IDocument doc = FileUtilsFileBuffer.getDocFromZip(emptyModuleForZip.f, emptyModuleForZip.pathInZip);
                                // NOTE: The nature (and so the grammar to be used) must be defined by this modules
                                // manager (and not by the initial caller)!!
                                n = AbstractModule.createModuleFromDoc(name, emptyModuleForZip.f, doc, this.getNature(), false);
                                SourceModule zipModule = (SourceModule) n;
                                zipModule.zipFilePath = emptyModuleForZip.pathInZip;
                                n = decorateModule(n, nature);
                            } catch (Exception exc1) {
                                Log.log(exc1);
                                n = null;
                            }
                        }
                    } else if (e instanceof EmptyModuleForFolder) {
                        try {
                            n = new InitFromDirModule(name, e.f, new Module(new stmtType[0]), null, this.getNature());
                            n = decorateModule(n, nature);
                        } catch (Exception e1) {
                            Log.log(e1);
                            n = null;
                        }
                    } else {
                        // regular case... just go on and create it.
                        try {
                            // NOTE: The nature (and so the grammar to be used) must be defined by this modules
                            // manager (and not by the initial caller)!!
                            n = AbstractModule.createModule(name, e.f, this.getNature(), true);
                            n = decorateModule(n, nature);
                        } catch (IOException exc) {
                            keyForCacheAccess.name = name;
                            keyForCacheAccess.file = e.f;
                            doRemoveSingleModule(keyForCacheAccess);
                            n = null;
                        } catch (MisconfigurationException exc) {
                            Log.log(exc);
                            n = null;
                        }
                    }
                }
            }
        } else {
            // ok, it does not have a file associated, so, we treat it as a builtin (this can happen in java jars)
            n = checkOverride(name, nature, n);
            if (n instanceof EmptyModule) {
                if (acceptCompiledModule) {
                    n = new CompiledModule(name, this, nature);
                    n = decorateModule(n, nature);
                } else {
                    return null;
                }
            }
        }
        if (n != null) {
            doAddSingleModule(createModulesKey(name, e.f), n);
        } else {
            Log.log(("The module " + name + " could not be found nor created!"));
        }
    }
    if (n instanceof EmptyModule) {
        throw new RuntimeException("Should not be an empty module anymore: " + n);
    }
    if (n instanceof SourceModule) {
        SourceModule sourceModule = (SourceModule) n;
        // now, here's a catch... it may be a bootstrap module...
        if (sourceModule.isBootstrapModule()) {
            // if it's a bootstrap module, we must replace it for the related compiled module.
            n = new CompiledModule(name, this, nature);
            n = decorateModule(n, nature);
        }
    }
    return n;
}

Example 4 with AbstractModule

use of org.python.pydev.ast.codecompletion.revisited.modules.AbstractModule in project Pydev by fabioz.

the class ModulesManager method decorateModule.

/**
 * Called after the creation of any module. Used as a workaround for filling tokens that are in no way
 * available in the code-completion through the regular inspection.
 *
 * The django objects class is the reason why this happens... It's structure for the creation on a model class
 * follows no real patterns for the creation of the 'objects' attribute in the class, and thus, we have no
 * real generic way of discovering it (actually, even by looking at the class definition this is very obscure),
 * so, the solution found is creating the objects by decorating the module with that info.
 */
private AbstractModule decorateModule(AbstractModule n, IPythonNature nature) {
    if (n instanceof SourceModule) {
        if ("django.db.models.base".equals(n.getName())) {
            SourceModule sourceModule = (SourceModule) n;
            SimpleNode ast = sourceModule.getAst();
            boolean found = false;
            Module module = (Module) ast;
            stmtType[] body = module.body;
            for (SimpleNode node : body) {
                if (node instanceof ClassDef && "Model".equals(NodeUtils.getRepresentationString(node))) {
                    found = true;
                    Object[][] metaclassAttrs = new Object[][] { { "objects", NodeUtils.makeAttribute("django.db.models.manager.Manager()") }, { "DoesNotExist", new Name("Exception", Name.Load, false) }, { "MultipleObjectsReturned", new Name("Exception", Name.Load, false) } };
                    ClassDef classDef = (ClassDef) node;
                    stmtType[] newBody = new stmtType[classDef.body.length + metaclassAttrs.length];
                    System.arraycopy(classDef.body, 0, newBody, metaclassAttrs.length, classDef.body.length);
                    int i = 0;
                    for (Object[] objAndType : metaclassAttrs) {
                        // Note that the line/col is important so that we correctly acknowledge it inside the "class Model" scope.
                        Name name = new Name((String) objAndType[0], Name.Store, false);
                        name.beginColumn = classDef.beginColumn + 4;
                        name.beginLine = classDef.beginLine + 1;
                        newBody[i] = new Assign(new exprType[] { name }, (exprType) objAndType[1], null);
                        newBody[i].beginColumn = classDef.beginColumn + 4;
                        newBody[i].beginLine = classDef.beginLine + 1;
                        i += 1;
                    }
                    classDef.body = newBody;
                    break;
                }
            }
            if (found) {
                stmtType[] newBody = new stmtType[body.length + 1];
                System.arraycopy(body, 0, newBody, 1, body.length);
                Import importNode = new Import(new aliasType[] { new aliasType(new NameTok("django.db.models.manager", NameTok.ImportModule), null) });
                newBody[0] = importNode;
                module.body = newBody;
            }
        } else if ("django.db.models.manager".equals(n.getName())) {
            SourceModule sourceModule = (SourceModule) n;
            SimpleNode ast = sourceModule.getAst();
            Module module = (Module) ast;
            stmtType[] body = module.body;
            for (SimpleNode node : body) {
                if (node instanceof ClassDef && "Manager".equals(NodeUtils.getRepresentationString(node))) {
                    ClassDef classDef = (ClassDef) node;
                    stmtType[] newBody = null;
                    try {
                        File managerBody = CorePlugin.getBundleInfo().getRelativePath(new Path("pysrc/stubs/_django_manager_body.py"));
                        IDocument doc = FileUtilsFileBuffer.getDocFromFile(managerBody);
                        IGrammarVersionProvider provider = new IGrammarVersionProvider() {

                            @Override
                            public int getGrammarVersion() throws MisconfigurationException {
                                // Always Python 3.0 here
                                return IGrammarVersionProvider.LATEST_GRAMMAR_PY3_VERSION;
                            }

                            @Override
                            public AdditionalGrammarVersionsToCheck getAdditionalGrammarVersions() throws MisconfigurationException {
                                return null;
                            }
                        };
                        ParseOutput obj = PyParser.reparseDocument(new PyParser.ParserInfo(doc, provider, "_django_manager_body", managerBody));
                        Module ast2 = (Module) obj.ast;
                        newBody = ast2.body;
                        for (stmtType b : newBody) {
                            // Note that the line/col is important so that we correctly acknowledge it inside the "class Manager" scope.
                            b.beginColumn = classDef.beginColumn + 4;
                            b.beginLine = classDef.beginLine + 1;
                        }
                        classDef.body = newBody;
                        break;
                    } catch (Exception e) {
                        Log.log(e);
                    }
                }
            }
        } else if ("typing".equals(n.getName())) {
            Module module = (Module) ((SourceModule) n).getAst();
            for (SimpleNode node : module.body) {
                if (node instanceof Assign && ((Assign) node).value instanceof Call && ((Call) ((Assign) node).value).func instanceof Name && "_alias".equals(((Name) ((Call) ((Assign) node).value).func).id) && ((Call) ((Assign) node).value).args.length >= 1) {
                    Assign assign = (Assign) node;
                    assign.value = ((Call) assign.value).args[0];
                }
            }
        }
    }
    return n;
}

Example 5 with AbstractModule

use of org.python.pydev.ast.codecompletion.revisited.modules.AbstractModule in project Pydev by fabioz.

the class SystemModulesManager method getBuiltinModule.

@Override
public AbstractModule getBuiltinModule(String name, boolean dontSearchInit, IModuleRequestState moduleRequest) {
    AbstractModule n = null;
    // check for supported builtins these don't have files associated.
    // they are the first to be passed because the user can force a module to be builtin, because there
    // is some information that is only useful when you have builtins, such as os and wxPython (those can
    // be source modules, but they have so much runtime info that it is almost impossible to get useful information
    // from statically analyzing them).
    String[] builtins = getBuiltins();
    if (builtins == null || this.info == null) {
        // still on startup
        return null;
    }
    // for temporary access (so that we don't generate many instances of it)
    ModulesKey keyForCacheAccess = new ModulesKey(null, null);
    // A different choice for users that want more complete information on the libraries they're dealing
    // with is using predefined modules. Those will
    File predefinedModule = this.info.getPredefinedModule(name, moduleRequest);
    boolean found = predefinedModule != null && predefinedModule.exists();
    if (!found && !name.endsWith(".__init__")) {
        final String nameWithInit = name + ".__init__";
        predefinedModule = this.info.getPredefinedModule(nameWithInit, moduleRequest);
        found = predefinedModule != null && predefinedModule.exists();
        if (found) {
            name = nameWithInit;
        }
    }
    if (found) {
        keyForCacheAccess.name = name;
        keyForCacheAccess.file = predefinedModule;
        n = cachePredefined.getObj(keyForCacheAccess, this);
        if ((n instanceof PredefinedSourceModule)) {
            PredefinedSourceModule predefinedSourceModule = (PredefinedSourceModule) n;
            if (predefinedSourceModule.isSynched()) {
                return n;
            }
        // otherwise (not PredefinedSourceModule or not synched), just keep going to create
        // it as a predefined source module
        }
        boolean tryToParse = true;
        Long lastModified = null;
        if (predefinedFilesNotParsedToTimestamp == null) {
            predefinedFilesNotParsedToTimestamp = new HashMap<File, Long>();
        } else {
            Long lastTimeChanged = predefinedFilesNotParsedToTimestamp.get(predefinedModule);
            if (lastTimeChanged != null) {
                lastModified = FileUtils.lastModified(predefinedModule);
                if (lastTimeChanged.equals(lastModified)) {
                    tryToParse = false;
                } else {
                    predefinedFilesNotParsedToTimestamp.remove(predefinedModule);
                }
            }
        }
        if (tryToParse) {
            IDocument doc;
            try {
                doc = FileUtilsFileBuffer.getDocFromFile(predefinedModule);
                IGrammarVersionProvider provider = new IGrammarVersionProvider() {

                    @Override
                    public int getGrammarVersion() throws MisconfigurationException {
                        // Always Python 3.0 here
                        return IGrammarVersionProvider.LATEST_GRAMMAR_PY3_VERSION;
                    }

                    @Override
                    public AdditionalGrammarVersionsToCheck getAdditionalGrammarVersions() throws MisconfigurationException {
                        return null;
                    }
                };
                ParseOutput obj = PyParser.reparseDocument(new PyParser.ParserInfo(doc, provider, name, predefinedModule));
                if (obj.error != null) {
                    if (lastModified == null) {
                        lastModified = FileUtils.lastModified(predefinedModule);
                    }
                    predefinedFilesNotParsedToTimestamp.put(predefinedModule, lastModified);
                    Log.log("Unable to parse: " + predefinedModule, obj.error);
                } else if (obj.ast != null) {
                    SimpleNode ast = (SimpleNode) obj.ast;
                    if ("builtins".equals(name) || "__builtin__".equals(name)) {
                        // None/False/True must be added as they're not there by default.
                        if (ast instanceof Module) {
                            Module module = (Module) ast;
                            PyAstFactory astFactory = new PyAstFactory(new AdapterPrefs("\n", info.getModulesManager().getNature()));
                            module.body = ArrayUtils.concatArrays(module.body, new stmtType[] { astFactory.createAssign(astFactory.createStoreName("None"), astFactory.createNone()), astFactory.createAssign(astFactory.createStoreName("False"), astFactory.createFalse()), astFactory.createAssign(astFactory.createStoreName("True"), astFactory.createTrue()), astFactory.createAssign(astFactory.createStoreName("__builtins__"), astFactory.createName("Any")) });
                        }
                    }
                    n = new PredefinedSourceModule(name, predefinedModule, ast, obj.error);
                    cachePredefined.add(keyForCacheAccess, n, this);
                    // doAddSingleModule(keyForCacheAccess, n);
                    return n;
                }
            // keep on going
            } catch (Throwable e) {
                Log.log(e);
            }
        }
    }
    boolean foundStartingWithBuiltin = false;
    FastStringBuffer buffer = null;
    for (int i = 0; i < builtins.length; i++) {
        String forcedBuiltin = builtins[i];
        if (name.startsWith(forcedBuiltin)) {
            if (name.length() > forcedBuiltin.length() && name.charAt(forcedBuiltin.length()) == '.') {
                foundStartingWithBuiltin = true;
                keyForCacheAccess.name = name;
                n = cache.getObj(keyForCacheAccess, this);
                if (n == null && dontSearchInit == false) {
                    if (buffer == null) {
                        buffer = new FastStringBuffer();
                    } else {
                        buffer.clear();
                    }
                    keyForCacheAccess.name = buffer.append(name).append(".__init__").toString();
                    n = cache.getObj(keyForCacheAccess, this);
                }
                if (n instanceof EmptyModule || n instanceof SourceModule) {
                    // it is actually found as a source module, so, we have to 'coerce' it to a compiled module
                    n = new CompiledModule(name, this, this.getNature());
                    doAddSingleModule(new ModulesKey(n.getName(), null), n);
                    return n;
                }
            }
            if (name.equals(forcedBuiltin)) {
                keyForCacheAccess.name = name;
                n = cache.getObj(keyForCacheAccess, this);
                if (n == null || n instanceof EmptyModule || n instanceof SourceModule) {
                    // still not created or not defined as compiled module (as it should be)
                    n = new CompiledModule(name, this, this.getNature());
                    doAddSingleModule(new ModulesKey(n.getName(), null), n);
                    return n;
                }
            }
            if (n instanceof CompiledModule) {
                return n;
            }
        }
    }
    if (foundStartingWithBuiltin) {
        if (builtinsNotConsidered.getObj(name) != null) {
            return null;
        }
        // ok, just add it if it is some module that actually exists
        n = new CompiledModule(name, this, this.getNature());
        TokensList globalTokens = n.getGlobalTokens();
        // the module to see its return values (because that's slow)
        if (globalTokens.size() > 0 && contains(globalTokens, "__file__")) {
            doAddSingleModule(new ModulesKey(name, null), n);
            return n;
        } else {
            builtinsNotConsidered.add(name, name);
            return null;
        }
    }
    return null;
}