Examples with PyAstFactory org.python.pydev.parser.jython.ast.factory.PyAstFactory used on opensource projects

Example 1 with PyAstFactory

use of org.python.pydev.parser.jython.ast.factory.PyAstFactory in project Pydev by fabioz.

the class AbstractAdditionalDependencyInfo method addAstForCompiledModule.

private void addAstForCompiledModule(IModule module, InterpreterInfo info, ModulesKey newKey, boolean removeFirst) {
    TokensList globalTokens = module.getGlobalTokens();
    PyAstFactory astFactory = new PyAstFactory(new AdapterPrefs("\n", info.getModulesManager().getNature()));
    List<stmtType> body = new ArrayList<>(globalTokens.size());
    for (IterTokenEntry entry : globalTokens) {
        IToken token = entry.getToken();
        switch(token.getType()) {
            case IToken.TYPE_CLASS:
                body.add(astFactory.createClassDef(token.getRepresentation()));
                break;
            case IToken.TYPE_FUNCTION:
                body.add(astFactory.createFunctionDef(token.getRepresentation()));
                break;
            default:
                Name attr = astFactory.createName(token.getRepresentation());
                // assign to itself just for generation purposes.
                body.add(astFactory.createAssign(attr, attr));
                break;
        }
    }
    // System.out.println("Creating info for: " + module.getName());
    if (removeFirst) {
        removeInfoFromModule(newKey.name, false);
    }
    addAstInfo(astFactory.createModule(body), newKey, false);
}

Example 2 with PyAstFactory

use of org.python.pydev.parser.jython.ast.factory.PyAstFactory in project Pydev by fabioz.

the class PyAstFactoryTest method setUp.

@Override
protected void setUp() throws Exception {
    super.setUp();
    astFactory = new PyAstFactory(new AdapterPrefs("\n", new IGrammarVersionProvider() {

        @Override
        public int getGrammarVersion() throws MisconfigurationException {
            return IGrammarVersionProvider.GRAMMAR_PYTHON_VERSION_2_7;
        }

        @Override
        public AdditionalGrammarVersionsToCheck getAdditionalGrammarVersions() throws MisconfigurationException {
            return null;
        }
    }));
}

Example 3 with PyAstFactory

use of org.python.pydev.parser.jython.ast.factory.PyAstFactory in project Pydev by fabioz.

the class OverrideMethodCompletionProposal method applyOnDocument.

public int applyOnDocument(ITextViewer viewer, IDocument document, char trigger, int stateMask, int offset) {
    IGrammarVersionProvider versionProvider = null;
    IPyEdit edit = null;
    if (viewer instanceof IPySourceViewer) {
        IPySourceViewer pySourceViewer = (IPySourceViewer) viewer;
        versionProvider = edit = pySourceViewer.getEdit();
    } else {
        versionProvider = new IGrammarVersionProvider() {

            @Override
            public int getGrammarVersion() throws MisconfigurationException {
                return IGrammarVersionProvider.LATEST_GRAMMAR_PY3_VERSION;
            }

            @Override
            public AdditionalGrammarVersionsToCheck getAdditionalGrammarVersions() throws MisconfigurationException {
                return null;
            }
        };
    }
    String delimiter = PySelection.getDelimiter(document);
    PyAstFactory factory = new PyAstFactory(new AdapterPrefs(delimiter, versionProvider));
    // Note that the copy won't have a parent.
    stmtType overrideBody = factory.createOverrideBody(this.functionDef, parentClassName, currentClassName);
    FunctionDef functionDef = this.functionDef.createCopy(false);
    functionDef.body = new stmtType[] { overrideBody != null ? overrideBody : new Pass() };
    try {
        MakeAstValidForPrettyPrintingVisitor.makeValid(functionDef);
    } catch (Exception e) {
        Log.log(e);
    }
    IIndentPrefs indentPrefs;
    if (edit != null) {
        indentPrefs = edit.getIndentPrefs();
    } else {
        indentPrefs = DefaultIndentPrefs.get(null);
    }
    String printed = NodeUtils.printAst(indentPrefs, edit, functionDef, delimiter);
    PySelection ps = new PySelection(document, offset);
    try {
        String lineContentsToCursor = ps.getLineContentsToCursor();
        int defIndex = lineContentsToCursor.indexOf("def");
        int defOffset = ps.getLineOffset() + defIndex;
        printed = StringUtils.indentTo(printed, lineContentsToCursor.substring(0, defIndex), false);
        printed = StringUtils.rightTrim(printed);
        this.fLen += offset - defOffset;
        document.replace(defOffset, this.fLen, printed);
        return defOffset + printed.length();
    } catch (BadLocationException x) {
    // ignore
    }
    return -1;
}

Example 4 with PyAstFactory

use of org.python.pydev.parser.jython.ast.factory.PyAstFactory in project Pydev by fabioz.

the class SystemModulesManager method getBuiltinModule.

@Override
public AbstractModule getBuiltinModule(String name, boolean dontSearchInit, IModuleRequestState moduleRequest) {
    AbstractModule n = null;
    // check for supported builtins these don't have files associated.
    // they are the first to be passed because the user can force a module to be builtin, because there
    // is some information that is only useful when you have builtins, such as os and wxPython (those can
    // be source modules, but they have so much runtime info that it is almost impossible to get useful information
    // from statically analyzing them).
    String[] builtins = getBuiltins();
    if (builtins == null || this.info == null) {
        // still on startup
        return null;
    }
    // for temporary access (so that we don't generate many instances of it)
    ModulesKey keyForCacheAccess = new ModulesKey(null, null);
    // A different choice for users that want more complete information on the libraries they're dealing
    // with is using predefined modules. Those will
    File predefinedModule = this.info.getPredefinedModule(name, moduleRequest);
    boolean found = predefinedModule != null && predefinedModule.exists();
    if (!found && !name.endsWith(".__init__")) {
        final String nameWithInit = name + ".__init__";
        predefinedModule = this.info.getPredefinedModule(nameWithInit, moduleRequest);
        found = predefinedModule != null && predefinedModule.exists();
        if (found) {
            name = nameWithInit;
        }
    }
    if (found) {
        keyForCacheAccess.name = name;
        keyForCacheAccess.file = predefinedModule;
        n = cachePredefined.getObj(keyForCacheAccess, this);
        if ((n instanceof PredefinedSourceModule)) {
            PredefinedSourceModule predefinedSourceModule = (PredefinedSourceModule) n;
            if (predefinedSourceModule.isSynched()) {
                return n;
            }
        // otherwise (not PredefinedSourceModule or not synched), just keep going to create
        // it as a predefined source module
        }
        boolean tryToParse = true;
        Long lastModified = null;
        if (predefinedFilesNotParsedToTimestamp == null) {
            predefinedFilesNotParsedToTimestamp = new HashMap<File, Long>();
        } else {
            Long lastTimeChanged = predefinedFilesNotParsedToTimestamp.get(predefinedModule);
            if (lastTimeChanged != null) {
                lastModified = FileUtils.lastModified(predefinedModule);
                if (lastTimeChanged.equals(lastModified)) {
                    tryToParse = false;
                } else {
                    predefinedFilesNotParsedToTimestamp.remove(predefinedModule);
                }
            }
        }
        if (tryToParse) {
            IDocument doc;
            try {
                doc = FileUtilsFileBuffer.getDocFromFile(predefinedModule);
                IGrammarVersionProvider provider = new IGrammarVersionProvider() {

                    @Override
                    public int getGrammarVersion() throws MisconfigurationException {
                        // Always Python 3.0 here
                        return IGrammarVersionProvider.LATEST_GRAMMAR_PY3_VERSION;
                    }

                    @Override
                    public AdditionalGrammarVersionsToCheck getAdditionalGrammarVersions() throws MisconfigurationException {
                        return null;
                    }
                };
                ParseOutput obj = PyParser.reparseDocument(new PyParser.ParserInfo(doc, provider, name, predefinedModule));
                if (obj.error != null) {
                    if (lastModified == null) {
                        lastModified = FileUtils.lastModified(predefinedModule);
                    }
                    predefinedFilesNotParsedToTimestamp.put(predefinedModule, lastModified);
                    Log.log("Unable to parse: " + predefinedModule, obj.error);
                } else if (obj.ast != null) {
                    SimpleNode ast = (SimpleNode) obj.ast;
                    if ("builtins".equals(name) || "__builtin__".equals(name)) {
                        // None/False/True must be added as they're not there by default.
                        if (ast instanceof Module) {
                            Module module = (Module) ast;
                            PyAstFactory astFactory = new PyAstFactory(new AdapterPrefs("\n", info.getModulesManager().getNature()));
                            module.body = ArrayUtils.concatArrays(module.body, new stmtType[] { astFactory.createAssign(astFactory.createStoreName("None"), astFactory.createNone()), astFactory.createAssign(astFactory.createStoreName("False"), astFactory.createFalse()), astFactory.createAssign(astFactory.createStoreName("True"), astFactory.createTrue()), astFactory.createAssign(astFactory.createStoreName("__builtins__"), astFactory.createName("Any")) });
                        }
                    }
                    n = new PredefinedSourceModule(name, predefinedModule, ast, obj.error);
                    cachePredefined.add(keyForCacheAccess, n, this);
                    // doAddSingleModule(keyForCacheAccess, n);
                    return n;
                }
            // keep on going
            } catch (Throwable e) {
                Log.log(e);
            }
        }
    }
    boolean foundStartingWithBuiltin = false;
    FastStringBuffer buffer = null;
    for (int i = 0; i < builtins.length; i++) {
        String forcedBuiltin = builtins[i];
        if (name.startsWith(forcedBuiltin)) {
            if (name.length() > forcedBuiltin.length() && name.charAt(forcedBuiltin.length()) == '.') {
                foundStartingWithBuiltin = true;
                keyForCacheAccess.name = name;
                n = cache.getObj(keyForCacheAccess, this);
                if (n == null && dontSearchInit == false) {
                    if (buffer == null) {
                        buffer = new FastStringBuffer();
                    } else {
                        buffer.clear();
                    }
                    keyForCacheAccess.name = buffer.append(name).append(".__init__").toString();
                    n = cache.getObj(keyForCacheAccess, this);
                }
                if (n instanceof EmptyModule || n instanceof SourceModule) {
                    // it is actually found as a source module, so, we have to 'coerce' it to a compiled module
                    n = new CompiledModule(name, this, this.getNature());
                    doAddSingleModule(new ModulesKey(n.getName(), null), n);
                    return n;
                }
            }
            if (name.equals(forcedBuiltin)) {
                keyForCacheAccess.name = name;
                n = cache.getObj(keyForCacheAccess, this);
                if (n == null || n instanceof EmptyModule || n instanceof SourceModule) {
                    // still not created or not defined as compiled module (as it should be)
                    n = new CompiledModule(name, this, this.getNature());
                    doAddSingleModule(new ModulesKey(n.getName(), null), n);
                    return n;
                }
            }
            if (n instanceof CompiledModule) {
                return n;
            }
        }
    }
    if (foundStartingWithBuiltin) {
        if (builtinsNotConsidered.getObj(name) != null) {
            return null;
        }
        // ok, just add it if it is some module that actually exists
        n = new CompiledModule(name, this, this.getNature());
        TokensList globalTokens = n.getGlobalTokens();
        // the module to see its return values (because that's slow)
        if (globalTokens.size() > 0 && contains(globalTokens, "__file__")) {
            doAddSingleModule(new ModulesKey(name, null), n);
            return n;
        } else {
            builtinsNotConsidered.add(name, name);
            return null;
        }
    }
    return null;
}

Example 5 with PyAstFactory

use of org.python.pydev.parser.jython.ast.factory.PyAstFactory in project Pydev by fabioz.

the class PyCodeCompletion method createOverrideCodeCompletions.

private void createOverrideCodeCompletions(CompletionRequest request, ArrayList<ICompletionProposalHandle> ret, PySelection ps) throws BadLocationException {
    IImageCache imageCache = SharedCorePlugin.getImageCache();
    IImageHandle imageOverride = imageCache != null ? imageCache.get(UIConstants.METHOD_ICON) : null;
    String lineContentsToCursor = ps.getLineContentsToCursor();
    LineStartingScope scopeStart = ps.getPreviousLineThatStartsScope(PySelection.CLASS_TOKEN, false, PySelection.getFirstCharPosition(lineContentsToCursor));
    String className = null;
    if (scopeStart != null) {
        className = PySelection.getClassNameInLine(scopeStart.lineStartingScope);
        if (className != null && className.length() > 0) {
            Tuple<List<String>, Integer> insideParensBaseClasses = ps.getInsideParentesisToks(true, scopeStart.iLineStartingScope);
            if (insideParensBaseClasses != null) {
                // representation -> token and base class
                OrderedMap<String, ImmutableTuple<IToken, String>> map = new OrderedMap<String, ImmutableTuple<IToken, String>>();
                for (String baseClass : insideParensBaseClasses.o1) {
                    try {
                        ICompletionState state = new CompletionState(-1, -1, null, request.nature, baseClass);
                        state.setActivationToken(baseClass);
                        state.setIsInCalltip(false);
                        IPythonNature pythonNature = request.nature;
                        checkPythonNature(pythonNature);
                        ICodeCompletionASTManager astManager = pythonNature.getAstManager();
                        if (astManager == null) {
                            // we're probably still loading it.
                            return;
                        }
                        // Ok, looking for a token in globals.
                        IModule module = request.getModule();
                        if (module == null) {
                            continue;
                        }
                        TokensList comps = astManager.getCompletionsForModule(module, state, true, true);
                        for (IterTokenEntry entry : comps) {
                            IToken iToken = entry.getToken();
                            String representation = iToken.getRepresentation();
                            ImmutableTuple<IToken, String> curr = map.get(representation);
                            if (curr != null && curr.o1 instanceof SourceToken) {
                                // source tokens are never reset!
                                continue;
                            }
                            int type = iToken.getType();
                            if (iToken instanceof SourceToken && ((SourceToken) iToken).getAst() instanceof FunctionDef) {
                                map.put(representation, new ImmutableTuple<IToken, String>(iToken, baseClass));
                            } else if (type == IToken.TYPE_FUNCTION || type == IToken.TYPE_UNKNOWN || type == IToken.TYPE_BUILTIN) {
                                map.put(representation, new ImmutableTuple<IToken, String>(iToken, baseClass));
                            }
                        }
                    } catch (Exception e) {
                        Log.log(e);
                    }
                }
                for (ImmutableTuple<IToken, String> tokenAndBaseClass : map.values()) {
                    FunctionDef functionDef = null;
                    // No checkings needed for type (we already did that above).
                    if (tokenAndBaseClass.o1 instanceof SourceToken) {
                        SourceToken sourceToken = (SourceToken) tokenAndBaseClass.o1;
                        SimpleNode ast = sourceToken.getAst();
                        if (ast instanceof FunctionDef) {
                            functionDef = (FunctionDef) ast;
                        } else {
                            functionDef = sourceToken.getAliased().createCopy();
                            NameTok t = (NameTok) functionDef.name;
                            t.id = sourceToken.getRepresentation();
                        }
                    } else {
                        // unfortunately, for builtins we usually cannot trust the parameters.
                        String representation = tokenAndBaseClass.o1.getRepresentation();
                        PyAstFactory factory = new PyAstFactory(new AdapterPrefs(ps.getEndLineDelim(), request.nature));
                        functionDef = factory.createFunctionDef(representation);
                        functionDef.args = factory.createArguments(true);
                        functionDef.args.vararg = new NameTok("args", NameTok.VarArg);
                        functionDef.args.kwarg = new NameTok("kwargs", NameTok.KwArg);
                        if (!representation.equals("__init__")) {
                            // signal that the return should be added
                            functionDef.body = new stmtType[] { new Return(null) };
                        }
                    }
                    if (functionDef != null) {
                        ret.add(CompletionProposalFactory.get().createOverrideMethodCompletionProposal(request, ps, ps.getAbsoluteCursorOffset(), 0, 0, imageOverride, functionDef, tokenAndBaseClass.o2, className));
                    }
                }
            }
        }
    }
}