Examples with SourceToken org.python.pydev.ast.codecompletion.revisited.modules.SourceToken used on opensource projects

Example 16 with SourceToken

use of org.python.pydev.ast.codecompletion.revisited.modules.SourceToken in project Pydev by fabioz.

the class AbstractASTManager method getCompletionsUnpackingAST.

private TokensList getCompletionsUnpackingAST(SimpleNode ast, final IModule module, ICompletionState state, UnpackInfo unpackPos) throws CompletionRecursionException {
    if (ast instanceof FunctionDef) {
        // let's try to find as an annotation first
        ITypeInfo type = NodeUtils.getReturnTypeFromFuncDefAST(ast);
        if (type != null) {
            TokensList completionsUnpackingType = getCompletionsUnpackingType(module, state, unpackPos, type);
            if (completionsUnpackingType != null && completionsUnpackingType.size() > 0) {
                return completionsUnpackingType;
            }
        }
        TokensList tokens = getCompletionsUnpackingDocstring(module, state, unpackPos, NodeUtils.getNodeDocString(ast));
        if (tokens != null && tokens.size() > 0) {
            return tokens;
        }
        List<Yield> findYields = YieldVisitor.findYields((FunctionDef) ast);
        for (Yield yield : findYields) {
            // what we should complete on.
            if (yield.value != null) {
                String rep = NodeUtils.getFullRepresentationString(yield.value);
                if (rep != null) {
                    ICompletionState copyWithActTok = state.getCopyWithActTok(rep);
                    copyWithActTok.setLookingFor(ICompletionState.LookingFor.LOOKING_FOR_INSTANCED_VARIABLE);
                    TokensList completionsForModule = getCompletionsForModule(module, copyWithActTok);
                    if (completionsForModule.size() > 0) {
                        return completionsForModule;
                    }
                }
            }
        }
        List<Return> findReturns = ReturnVisitor.findReturns((FunctionDef) ast);
        for (Return return1 : findReturns) {
            // Return types have to be unpacked...
            if (return1.value != null) {
                exprType[] elts = NodeUtils.getEltsFromCompoundObject(return1.value);
                if (elts != null) {
                    TokensList ret = getCompletionsFromUnpackedCompoundObject(module, state, elts, unpackPos);
                    if (ret != null && ret.size() > 0) {
                        return ret;
                    }
                } else {
                    String rep = NodeUtils.getFullRepresentationString(return1.value);
                    if (rep != null) {
                        TokensList completionsUnpackingObject = getCompletionsUnpackingObject(module, state.getCopyWithActTok(rep), null, unpackPos);
                        if (completionsUnpackingObject != null && completionsUnpackingObject.size() > 0) {
                            return completionsUnpackingObject;
                        }
                    }
                }
            }
        }
    } else if (ast instanceof ClassDef) {
        String rep = NodeUtils.getFullRepresentationString(ast);
        if (rep != null) {
            TokensList completionsForModule = this.getCompletionsForModule(module, state.getCopyWithActTok(rep));
            IToken getItemToken = null;
            for (IterTokenEntry entry : completionsForModule) {
                IToken iToken = entry.getToken();
                switch(iToken.getRepresentation()) {
                    case "__getitem__":
                        getItemToken = iToken;
                        break;
                    case "__iter__":
                    case "__next__":
                    case "__enter__":
                        // If we find it we'll try to unpack completions from it.
                        if (iToken instanceof SourceToken) {
                            SourceToken sourceToken = (SourceToken) iToken;
                            IModule useModule = null;
                            if (module.getName().equals(sourceToken.getParentPackage())) {
                                useModule = module;
                            }
                            if (useModule == null) {
                                String parentPackage = sourceToken.getParentPackage();
                                useModule = getModule(parentPackage, state.getNature(), true, state);
                            }
                            TokensList ret = getCompletionsUnpackingAST(sourceToken.getAst(), useModule, state, unpackPos);
                            if (ret != null && ret.size() > 0) {
                                return ret;
                            }
                        }
                        break;
                }
            }
            if (getItemToken instanceof SourceToken) {
                // The __getitem__ is already unpacked (i.e.: __iter__ returns a generator
                // and __getitem__ already returns the value we're iterating through).
                SourceToken sourceToken = (SourceToken) getItemToken;
                IModule useModule = null;
                if (module.getName().equals(sourceToken.getParentPackage())) {
                    useModule = module;
                } else {
                    String parentPackage = getItemToken.getParentPackage();
                    useModule = getModule(parentPackage, state.getNature(), true, state);
                }
                TokensList ret = getCompletionsNotUnpackingToken(sourceToken, useModule, state);
                if (ret != null && ret.size() > 0) {
                    return ret;
                }
            }
        }
    }
    return null;
}

Example 17 with SourceToken

use of org.python.pydev.ast.codecompletion.revisited.modules.SourceToken in project Pydev by fabioz.

the class PyCodeCompletion method getCodeCompletionProposals.

@Override
public TokensOrProposalsList getCodeCompletionProposals(CompletionRequest request) throws CoreException, BadLocationException, IOException, MisconfigurationException, PythonNatureWithoutProjectException {
    if (request.getPySelection().getCursorLineContents().trim().startsWith("#")) {
        // this may happen if the context is still not correctly computed in python
        return new PyStringCodeCompletion().getCodeCompletionProposals(request);
    }
    if (PySelection.isCompletionForLiteralNumber(request.getActivationToken())) {
        // suppress completions that would be invalid
        return new TokensOrProposalsList();
    }
    if (DebugSettings.DEBUG_CODE_COMPLETION) {
        org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Starting getCodeCompletionProposals");
        org.python.pydev.shared_core.log.ToLogFile.addLogLevel();
        org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Request:" + request);
    }
    ArrayList<ICompletionProposalHandle> ret = new ArrayList<ICompletionProposalHandle>();
    // let's see if we should do a code-completion in the current scope...
    // this engine does not work 'correctly' in the default scope on:
    // - class definitions - after 'class' and before '('
    // - method definitions - after 'def' and before '('
    PySelection ps = request.getPySelection();
    int lineCtx = ps.isRightAfterDeclarationInLine();
    if (lineCtx != PySelection.DECLARATION_NONE) {
        if (lineCtx == PySelection.DECLARATION_METHOD) {
            createOverrideCodeCompletions(request, ret, ps);
        }
        request.showTemplates = false;
        return new TokensOrProposalsList(ret);
    }
    try {
        IPythonNature nature = request.nature;
        checkPythonNature(nature);
        ICodeCompletionASTManager astManager = nature.getAstManager();
        if (astManager == null) {
            // we're probably still loading it.
            return new TokensOrProposalsList(ret);
        }
        // list of Object[], IToken or ICompletionProposalHandle
        TokensOrProposalsList tokensList = new TokensOrProposalsList();
        String trimmed = request.activationToken.replace('.', ' ').trim();
        ImportInfo importsTipper = getImportsTipperStr(request);
        int line = request.doc.getLineOfOffset(request.documentOffset);
        IRegion region = request.doc.getLineInformation(line);
        ICompletionState state = new CompletionState(line, request.documentOffset - region.getOffset(), null, request.nature, request.qualifier);
        state.setCancelMonitor(request.getCancelMonitor());
        state.setIsInCalltip(request.isInCalltip);
        Map<String, IterTokenEntry> alreadyChecked = new HashMap<>();
        boolean importsTip = false;
        if (request.activationToken.startsWith("super()")) {
            createSuperCodeCompletions(request, tokensList, ps);
        } else if (importsTipper.importsTipperStr.length() != 0) {
            // code completion in imports
            // if found after (, but in an import, it is not a calltip!
            request.isInCalltip = false;
            // if found after (, but in an import, it is not a calltip!
            request.isInMethodKeywordParam = false;
            importsTip = doImportCompletion(request, astManager, tokensList, importsTipper);
        } else if (trimmed.length() > 0 && request.activationToken.indexOf('.') != -1) {
            // code completion for a token
            if (false) {
                // disabled for now.
                fillTokensWithJediCompletions(request, request.getPySelection(), request.nature, astManager, tokensList);
            } else {
                TokensList tokenCompletions = new TokensList();
                doTokenCompletion(request, astManager, tokenCompletions, trimmed, state);
                tokensList.addAll(tokenCompletions);
            }
            handleKeywordParam(request, line, alreadyChecked);
        } else {
            // go to globals
            if (request.isInCalltip) {
                // # SomeCall(|<- here)
                state.setLookingFor(ICompletionState.LookingFor.LOOKING_FOR_INSTANCED_VARIABLE);
            }
            doGlobalsCompletion(request, astManager, tokensList, state);
            // At this point, after doing the globals completion, we may also need to check if we need to show
            // keyword parameters to the user.
            handleKeywordParam(request, line, alreadyChecked);
        }
        List<Object> analyzedTokens = null;
        if (request.qualifier.length() >= PyCodeCompletionPreferences.getArgumentsDeepAnalysisNChars()) {
            // this can take some time on the analysis, so, let's let the user choose on how many chars does he
            // want to do the analysis...
            state.pushFindResolveImportMemoryCtx();
            try {
                IFilter nameFilter = PyCodeCompletionUtils.getNameFilter(request.useSubstringMatchInCodeCompletion, request.qualifier);
                int i = 0;
                analyzedTokens = new ArrayList<Object>();
                // Note: later on we'll clear the tokensList and re-add the tokens on alreadyChecked.
                for (Iterator<IterEntry> it = tokensList.iterator(); it.hasNext(); ) {
                    i++;
                    if (i > 10000) {
                        // tokensList.size(), request));
                        break;
                    }
                    IterEntry entry = it.next();
                    Object o = entry.object;
                    if (!(o instanceof IToken)) {
                        // Don't check things that are not tokens
                        analyzedTokens.add(o);
                    } else {
                        IToken initialToken = (IToken) o;
                        IToken token = initialToken;
                        String strRep = token.getRepresentation();
                        IterTokenEntry iterTokenEntry = alreadyChecked.get(strRep);
                        IToken prev;
                        if (iterTokenEntry == null) {
                            prev = null;
                        } else {
                            prev = iterTokenEntry.getToken();
                        }
                        if (prev != null) {
                            if (prev.getArgs().length() != 0) {
                                // we already have a version with args... just keep going
                                continue;
                            }
                        }
                        if (!nameFilter.acceptName(strRep)) {
                            // just re-add it if we're going to actually use it (depending on the qualifier)
                            continue;
                        }
                        IModule current = request.getModule();
                        while (token.isImportFrom()) {
                            if (token.getArgs().length() > 0) {
                                // if we already have the args, there's also no reason to do it (that's what we'll do here)
                                break;
                            }
                            ICompletionState s = state.getCopyForResolveImportWithActTok(token.getRepresentation());
                            s.checkFindResolveImportMemory(token);
                            ImmutableTuple<IModule, IToken> modTok = astManager.resolveImport(s, token, current);
                            IToken token2 = modTok.o2;
                            current = modTok.o1;
                            if (token2 != null && initialToken != token2) {
                                String args = token2.getArgs();
                                if (args.length() > 0) {
                                    // put it into the map (may override previous if it didn't have args)
                                    initialToken.setArgs(args);
                                    initialToken.setDocStr(token2.getDocStr());
                                    if (initialToken instanceof SourceToken && token2 instanceof SourceToken) {
                                        SourceToken initialSourceToken = (SourceToken) initialToken;
                                        SourceToken token2SourceToken = (SourceToken) token2;
                                        initialSourceToken.setAst(token2SourceToken.getAst());
                                    }
                                    break;
                                }
                                if (token2 == null || (token2.equals(token) && token2.getArgs().equals(token.getArgs()) && token2.getParentPackage().equals(token.getParentPackage()))) {
                                    break;
                                }
                                token = token2;
                            } else {
                                break;
                            }
                        }
                        // Completions of B should have B with the 'a' argument.
                        if (token instanceof SourceToken) {
                            SourceToken sourceToken = (SourceToken) token;
                            if (sourceToken.getAst() instanceof ClassDef && token.getArgs().length() == 0) {
                                ClassDef classDef = (ClassDef) sourceToken.getAst();
                                if (classDef.bases != null && classDef.bases.length > 0) {
                                    String parentPackage = sourceToken.getParentPackage();
                                    if (parentPackage != null) {
                                        IModule module;
                                        if (parentPackage.equals(current.getName())) {
                                            module = current;
                                        } else {
                                            module = astManager.getModule(parentPackage, nature, true, state);
                                        }
                                        if (module != null) {
                                            if (module instanceof SourceModule) {
                                                SourceModule sourceModule = (SourceModule) module;
                                                OUT: for (int j = 0; j < classDef.bases.length; j++) {
                                                    try {
                                                        LookingFor lookingFor = state.getLookingFor();
                                                        TokensList completions;
                                                        try {
                                                            ClassDefTokensExtractor classTokensExtractor = new ClassDefTokensExtractor(classDef, sourceModule, state);
                                                            completions = classTokensExtractor.getCompletionsForBase(astManager, classDef.bases[j]);
                                                        } finally {
                                                            // Completions at this point shouldn't change the state of what we were looking for.
                                                            state.setLookingFor(lookingFor, true);
                                                        }
                                                        if (completions != null && completions.size() > 0) {
                                                            for (IterTokenEntry entry1 : completions) {
                                                                IToken comp = entry1.getToken();
                                                                if ("__init__".equals(comp.getRepresentation())) {
                                                                    if (comp.getArgs().length() > 0) {
                                                                        initialToken.setArgs(comp.getArgs());
                                                                    }
                                                                    break OUT;
                                                                }
                                                            }
                                                        }
                                                    } catch (CompletionRecursionException e) {
                                                    // Ignore (just don't get the args).
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                        alreadyChecked.put(strRep, new IterTokenEntry(initialToken, entry.lookingFor));
                    }
                }
            } finally {
                state.popFindResolveImportMemoryCtx();
            }
        }
        if (analyzedTokens == null) {
            tokensList.addAll(new TokensListMixedLookingFor(alreadyChecked.values()));
        } else {
            tokensList = new TokensOrProposalsList(analyzedTokens.toArray(new Object[0]));
            tokensList.addAll(new TokensListMixedLookingFor(alreadyChecked.values()));
        }
        changeItokenToCompletionPropostal(request, ret, tokensList, importsTip, state);
    } catch (CompletionRecursionException e) {
        if (onCompletionRecursionException != null) {
            onCompletionRecursionException.call(e);
        }
        if (DebugSettings.DEBUG_CODE_COMPLETION) {
            org.python.pydev.shared_core.log.ToLogFile.toLogFile(e);
        }
    // PydevPlugin.log(e);
    // ret.add(new CompletionProposal("",request.documentOffset,0,0,null,e.getMessage(), null,null));
    }
    if (DebugSettings.DEBUG_CODE_COMPLETION) {
        org.python.pydev.shared_core.log.ToLogFile.remLogLevel();
        org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Finished completion. Returned:" + ret.size() + " completions.\r\n");
    }
    return new TokensOrProposalsList(ret);
}

Example 18 with SourceToken

use of org.python.pydev.ast.codecompletion.revisited.modules.SourceToken in project Pydev by fabioz.

the class PyCodeCompletion method createOverrideCodeCompletions.

private void createOverrideCodeCompletions(CompletionRequest request, ArrayList<ICompletionProposalHandle> ret, PySelection ps) throws BadLocationException {
    IImageCache imageCache = SharedCorePlugin.getImageCache();
    IImageHandle imageOverride = imageCache != null ? imageCache.get(UIConstants.METHOD_ICON) : null;
    String lineContentsToCursor = ps.getLineContentsToCursor();
    LineStartingScope scopeStart = ps.getPreviousLineThatStartsScope(PySelection.CLASS_TOKEN, false, PySelection.getFirstCharPosition(lineContentsToCursor));
    String className = null;
    if (scopeStart != null) {
        className = PySelection.getClassNameInLine(scopeStart.lineStartingScope);
        if (className != null && className.length() > 0) {
            Tuple<List<String>, Integer> insideParensBaseClasses = ps.getInsideParentesisToks(true, scopeStart.iLineStartingScope);
            if (insideParensBaseClasses != null) {
                // representation -> token and base class
                OrderedMap<String, ImmutableTuple<IToken, String>> map = new OrderedMap<String, ImmutableTuple<IToken, String>>();
                for (String baseClass : insideParensBaseClasses.o1) {
                    try {
                        ICompletionState state = new CompletionState(-1, -1, null, request.nature, baseClass);
                        state.setActivationToken(baseClass);
                        state.setIsInCalltip(false);
                        IPythonNature pythonNature = request.nature;
                        checkPythonNature(pythonNature);
                        ICodeCompletionASTManager astManager = pythonNature.getAstManager();
                        if (astManager == null) {
                            // we're probably still loading it.
                            return;
                        }
                        // Ok, looking for a token in globals.
                        IModule module = request.getModule();
                        if (module == null) {
                            continue;
                        }
                        TokensList comps = astManager.getCompletionsForModule(module, state, true, true);
                        for (IterTokenEntry entry : comps) {
                            IToken iToken = entry.getToken();
                            String representation = iToken.getRepresentation();
                            ImmutableTuple<IToken, String> curr = map.get(representation);
                            if (curr != null && curr.o1 instanceof SourceToken) {
                                // source tokens are never reset!
                                continue;
                            }
                            int type = iToken.getType();
                            if (iToken instanceof SourceToken && ((SourceToken) iToken).getAst() instanceof FunctionDef) {
                                map.put(representation, new ImmutableTuple<IToken, String>(iToken, baseClass));
                            } else if (type == IToken.TYPE_FUNCTION || type == IToken.TYPE_UNKNOWN || type == IToken.TYPE_BUILTIN) {
                                map.put(representation, new ImmutableTuple<IToken, String>(iToken, baseClass));
                            }
                        }
                    } catch (Exception e) {
                        Log.log(e);
                    }
                }
                for (ImmutableTuple<IToken, String> tokenAndBaseClass : map.values()) {
                    FunctionDef functionDef = null;
                    // No checkings needed for type (we already did that above).
                    if (tokenAndBaseClass.o1 instanceof SourceToken) {
                        SourceToken sourceToken = (SourceToken) tokenAndBaseClass.o1;
                        SimpleNode ast = sourceToken.getAst();
                        if (ast instanceof FunctionDef) {
                            functionDef = (FunctionDef) ast;
                        } else {
                            functionDef = sourceToken.getAliased().createCopy();
                            NameTok t = (NameTok) functionDef.name;
                            t.id = sourceToken.getRepresentation();
                        }
                    } else {
                        // unfortunately, for builtins we usually cannot trust the parameters.
                        String representation = tokenAndBaseClass.o1.getRepresentation();
                        PyAstFactory factory = new PyAstFactory(new AdapterPrefs(ps.getEndLineDelim(), request.nature));
                        functionDef = factory.createFunctionDef(representation);
                        functionDef.args = factory.createArguments(true);
                        functionDef.args.vararg = new NameTok("args", NameTok.VarArg);
                        functionDef.args.kwarg = new NameTok("kwargs", NameTok.KwArg);
                        if (!representation.equals("__init__")) {
                            // signal that the return should be added
                            functionDef.body = new stmtType[] { new Return(null) };
                        }
                    }
                    if (functionDef != null) {
                        ret.add(CompletionProposalFactory.get().createOverrideMethodCompletionProposal(request, ps, ps.getAbsoluteCursorOffset(), 0, 0, imageOverride, functionDef, tokenAndBaseClass.o2, className));
                    }
                }
            }
        }
    }
}

Example 19 with SourceToken

use of org.python.pydev.ast.codecompletion.revisited.modules.SourceToken in project Pydev by fabioz.

the class AbstractVisitor method makeImportToken.

/**
 * The same as above
 */
private static List<IToken> makeImportToken(SimpleNode node, List<IToken> tokens, aliasType[] names, String module, String initialImportName, IPythonNature nature) {
    if (tokens == null) {
        tokens = new ArrayList<IToken>();
    }
    if (initialImportName.length() > 0) {
        initialImportName = initialImportName + ".";
    }
    for (int i = 0; i < names.length; i++) {
        aliasType aliasType = names[i];
        String name = null;
        String original = ((NameTok) aliasType.name).id;
        if (aliasType.asname != null) {
            name = ((NameTok) aliasType.asname).id;
        }
        if (name == null) {
            FullRepIterable iterator = new FullRepIterable(original);
            Iterator<String> it = iterator.iterator();
            while (it.hasNext()) {
                String rep = it.next();
                SourceToken sourceToken;
                if (it.hasNext()) {
                    sourceToken = new ImportPartSourceToken(node, rep, "", "", module, initialImportName + rep, true, nature);
                } else {
                    sourceToken = new SourceToken(node, rep, "", "", module, initialImportName + rep, true, nature);
                }
                tokens.add(sourceToken);
            }
        } else {
            SourceToken sourceToken = new SourceToken(node, name, "", "", module, initialImportName + original, false, nature);
            tokens.add(sourceToken);
        }
    }
    return tokens;
}

Example 20 with SourceToken

use of org.python.pydev.ast.codecompletion.revisited.modules.SourceToken in project Pydev by fabioz.

the class AbstractVisitor method addToken.

/**
 * Adds a token with a docstring.
 *
 * @param node
 */
protected SourceToken addToken(SimpleNode node) {
    // add the token
    SourceToken t = makeToken(node, moduleName, nature);
    this.tokens.add(t);
    return t;
}