Examples with IToken org.python.pydev.core.IToken used on opensource projects

Example 26 with IToken

use of org.python.pydev.core.IToken in project Pydev by fabioz.

the class AbstractASTManager method filterForWildImport.

/**
 * Filters the tokens according to the wild import rules:
 * - the tokens starting with '_' are removed
 * - if __all__ is available, only the tokens contained in __all__ are returned)
 */
private TokensList filterForWildImport(IModule module, boolean handleAsWildImport, TokensList completionsForModule) {
    if (module != null && handleAsWildImport) {
        ArrayList<IToken> ret = new ArrayList<IToken>();
        for (IterTokenEntry entry : completionsForModule) {
            IToken token = entry.getToken();
            // on wild imports we don't get names that start with '_'
            if (!token.getRepresentation().startsWith("_")) {
                ret.add(token);
            }
        }
        if (module instanceof SourceModule) {
            // Support for __all__: filter things if __all__ is available.
            SourceModule sourceModule = (SourceModule) module;
            GlobalModelVisitor globalModelVisitorCache = sourceModule.getGlobalModelVisitorCache();
            if (globalModelVisitorCache != null) {
                globalModelVisitorCache.filterAll(ret);
            }
        }
        IToken[] array = ret.toArray(new IToken[ret.size()]);
        TokensList retTokensList = new TokensList(array);
        retTokensList.setLookingFor(completionsForModule.getLookingFor());
        return retTokensList;
    } else {
        return completionsForModule;
    }
}

Example 27 with IToken

use of org.python.pydev.core.IToken in project Pydev by fabioz.

the class AbstractModule method internalGenerateCachedTokens.

/**
 * Generates the cached tokens in the needed structure for a 'fast' search given a token representation
 * (creates a map with the name of the token --> token).
 */
private Map<String, IToken> internalGenerateCachedTokens(IPythonNature nature, ICompletionCache completionCache, String activationToken, boolean searchSameLevelMods) throws CompletionRecursionException {
    Map<String, IToken> cachedTokens = new HashMap<String, IToken>();
    // if still not found, we have to get all the tokens, including regular and wild imports
    ICompletionState state = CompletionStateFactory.getEmptyCompletionState(nature, completionCache);
    ICodeCompletionASTManager astManager = nature.getAstManager();
    state.setActivationToken(activationToken);
    // we don't want to gather builtins in this case.
    state.setBuiltinsGotten(true);
    TokensList globalTokens = astManager.getCompletionsForModule(this, state, searchSameLevelMods);
    for (IterTokenEntry entry : globalTokens) {
        IToken token = entry.getToken();
        String rep = token.getRepresentation();
        IToken t = cachedTokens.get(rep);
        if (t != null) {
            // only override tokens if it's a getattr that's not defined in the builtin module
            if (rep.equals("__getattribute__") || rep.equals("__getattr__")) {
                if (!isTokenFromBuiltins(token)) {
                    cachedTokens.put(rep, token);
                }
            }
        } else {
            cachedTokens.put(rep, token);
        }
    }
    return cachedTokens;
}

Example 28 with IToken

use of org.python.pydev.core.IToken in project Pydev by fabioz.

the class ClassDefTokensExtractor method getTokens.

/**
 * @param manager
 * @return TokensList
 */
public TokensList getTokens(ICodeCompletionASTManager manager) {
    List<IToken> globalModelVisitorTokens = GlobalModelVisitor.getTokens(classDef, GlobalModelVisitor.INNER_DEFS, this.module.getName(), initialState, false, this.module.getNature());
    TokensList classTokens = new TokensList(globalModelVisitorTokens);
    try {
        for (int j = 0; j < classDef.bases.length; j++) {
            TokensList completions = getCompletionsForBase(manager, classDef.bases[j]);
            classTokens.addAll(completions);
        }
        TokensList assignmentClassTokens = getAssignmentTokens();
        classTokens.addAll(assignmentClassTokens);
    } catch (CompletionRecursionException e) {
    // let's return what we have so far...
    }
    classTokens.setLookingFor(initialState.getLookingFor());
    return classTokens;
}

Example 29 with IToken

use of org.python.pydev.core.IToken in project Pydev by fabioz.

the class ClassDefTokensExtractor method extractTokensFromBody.

private TokensList extractTokensFromBody(stmtType[] body) {
    TokensList ret = new TokensList();
    List<IToken> tokensList = new ArrayList<IToken>();
    for (stmtType node : body) {
        if (node instanceof Assign) {
            Assign assign = (Assign) node;
            Optional<IToken> optionalToken = extractTokenFromAssign(assign);
            if (optionalToken.isPresent()) {
                IToken token = optionalToken.get();
                tokensList.add(token);
            }
        }
    }
    ret.addAll(new TokensList(tokensList));
    return ret;
}

Example 30 with IToken

use of org.python.pydev.core.IToken in project Pydev by fabioz.

the class CompiledModule method getCached.

/**
 * Gets cached information for the given name. Could be a dotted or non-dotted name.
 */
private static Tuple<File, IToken[]> getCached(String name, IModulesManager manager) {
    ISystemModulesManager systemModulesManager = manager.getSystemModulesManager();
    File f = getCacheFile(name, systemModulesManager);
    if (f != null && f.exists()) {
        try {
            IToken[] toks = null;
            File file = null;
            try (FileInputStream fin = new FileInputStream(f)) {
                try (InputStream in = new BufferedInputStream(new GZIPInputStream(fin))) {
                    try (ObjectInputStream stream = new ObjectInputStream(in)) {
                        ObjectsInternPool.ObjectsPoolMap map = new ObjectsInternPool.ObjectsPoolMap();
                        @SuppressWarnings("unused") Object // we already have the name set (so, it's only there for completeness).
                        _name = stream.readObject();
                        file = (File) stream.readObject();
                        int size = stream.readInt();
                        toks = new IToken[size];
                        IPythonNature nature = systemModulesManager.getNature();
                        for (int i = 0; i < size; i++) {
                            // Note intern (we probably have many empty strings -- or the same for parentPackage)
                            String rep = ObjectsInternPool.internLocal(map, (String) stream.readObject());
                            int type = stream.readInt();
                            String args = ObjectsInternPool.internLocal(map, (String) stream.readObject());
                            String parentPackage = ObjectsInternPool.internLocal(map, (String) stream.readObject());
                            toks[i] = new CompiledToken(rep, "", args, parentPackage, type, nature);
                        }
                        for (int i = 0; i < size; i++) {
                            toks[i].setDocStr(ObjectsInternPool.internLocal(map, (String) stream.readObject()));
                        }
                    }
                }
            }
            return new Tuple<File, IToken[]>(file, toks);
        } catch (Exception e) {
            // Unable to read: just log it
            Log.log("Unable to read contents from: " + f, e);
        }
    }
    return null;
}