Examples with NameTok org.python.pydev.parser.jython.ast.NameTok used on opensource projects

Example 1 with NameTok

use of org.python.pydev.parser.jython.ast.NameTok in project Pydev by fabioz.

the class IOUtils method addAstInfo.

/**
 * Adds ast info information for a module.
 *
 * @param m the module we want to add to the info
 */
public List<IInfo> addAstInfo(SimpleNode node, ModulesKey key, boolean generateDelta) {
    List<IInfo> createdInfos = new ArrayList<IInfo>();
    if (node == null || key.name == null) {
        return createdInfos;
    }
    try {
        Tuple<DefinitionsASTIteratorVisitor, Iterator<ASTEntry>> tup = getInnerEntriesForAST(node);
        if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
            org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Adding ast info to: " + key.name);
        }
        try {
            Iterator<ASTEntry> entries = tup.o2;
            FastStack<SimpleNode> tempStack = new FastStack<SimpleNode>(10);
            synchronized (this.lock) {
                synchronized (ObjectsInternPool.lock) {
                    final String file = key.file != null ? ObjectsInternPool.internUnsynched(key.file.toString()) : null;
                    key.name = ObjectsInternPool.internUnsynched(key.name);
                    while (entries.hasNext()) {
                        ASTEntry entry = entries.next();
                        IInfo infoCreated = null;
                        if (entry.parent == null) {
                            // we only want those that are in the global scope
                            if (entry.node instanceof ClassDef) {
                                // no intern construct (locked in this loop)
                                NameTok name = (NameTok) ((ClassDef) entry.node).name;
                                ClassInfo info = new ClassInfo(ObjectsInternPool.internUnsynched(name.id), key.name, null, false, getNature(), file, name.beginLine, name.beginColumn);
                                add(info, TOP_LEVEL);
                                infoCreated = info;
                            } else if (entry.node instanceof FunctionDef) {
                                // no intern construct (locked in this loop)
                                NameTok name = (NameTok) ((FunctionDef) entry.node).name;
                                FuncInfo info2 = new FuncInfo(ObjectsInternPool.internUnsynched(name.id), key.name, null, false, getNature(), file, name.beginLine, name.beginColumn);
                                add(info2, TOP_LEVEL);
                                infoCreated = info2;
                            } else {
                                // it is an assign
                                infoCreated = this.addAssignTargets(entry, key.name, TOP_LEVEL, null, false, file);
                            }
                        } else {
                            if (entry.node instanceof ClassDef || entry.node instanceof FunctionDef) {
                                // ok, it has a parent, so, let's check to see if the path we got only has class definitions
                                // as the parent (and get that path)
                                Tuple<String, Boolean> pathToRoot = this.getPathToRoot(entry, false, false, tempStack);
                                if (pathToRoot != null && pathToRoot.o1 != null && pathToRoot.o1.length() > 0) {
                                    if (entry.node instanceof ClassDef) {
                                        NameTok name = ((NameTok) ((ClassDef) entry.node).name);
                                        ClassInfo info = new ClassInfo(ObjectsInternPool.internUnsynched(name.id), key.name, ObjectsInternPool.internUnsynched(pathToRoot.o1), false, getNature(), file, name.beginLine, name.beginColumn);
                                        add(info, INNER);
                                        infoCreated = info;
                                    } else {
                                        // FunctionDef
                                        NameTok name = ((NameTok) ((FunctionDef) entry.node).name);
                                        FuncInfo info2 = new FuncInfo(ObjectsInternPool.internUnsynched(name.id), key.name, ObjectsInternPool.internUnsynched(pathToRoot.o1), false, getNature(), file, name.beginLine, name.beginColumn);
                                        add(info2, INNER);
                                        infoCreated = info2;
                                    }
                                }
                            } else {
                                // it is an assign
                                Tuple<String, Boolean> pathToRoot = this.getPathToRoot(entry, true, false, tempStack);
                                if (pathToRoot != null && pathToRoot.o1 != null && pathToRoot.o1.length() > 0) {
                                    infoCreated = this.addAssignTargets(entry, key.name, INNER, pathToRoot.o1, pathToRoot.o2, file);
                                }
                            }
                        }
                        if (infoCreated != null) {
                            createdInfos.add(infoCreated);
                        }
                    }
                // end while
                }
            // end lock ObjectsPool.lock
            }
        // end this.lock
        } catch (Exception e) {
            Log.log(e);
        }
    } catch (Exception e) {
        Log.log(e);
    }
    return createdInfos;
}

Example 2 with NameTok

use of org.python.pydev.parser.jython.ast.NameTok in project Pydev by fabioz.

the class MessagesManager method addUnusedMessage.

/**
 * adds a message for something that was not used
 *
 * @param node the node representing the scope being closed when adding the
 *             unused message
 */
public void addUnusedMessage(SimpleNode node, Found f) {
    List<GenAndTok> all = f.getAll();
    int len = all.size();
    for (int i = 0; i < len; i++) {
        GenAndTok g = all.get(i);
        if (g.generator instanceof SourceToken) {
            SimpleNode ast = ((SourceToken) g.generator).getAst();
            // it can be an unused import
            boolean isFromImport = ast instanceof ImportFrom;
            if (isFromImport || ast instanceof Import) {
                if (isFromImport && AbstractVisitor.isWildImport((ImportFrom) ast)) {
                    addMessage(IAnalysisPreferences.TYPE_UNUSED_WILD_IMPORT, g.generator, g.tok);
                } else if (!(g.generator instanceof ImportPartSourceToken)) {
                    addMessage(IAnalysisPreferences.TYPE_UNUSED_IMPORT, g.generator, g.tok);
                }
                // finish it...
                continue;
            }
        }
        // we have to check if this is a name we should ignore
        if (startsWithNamesToIgnore(g)) {
            int type = IAnalysisPreferences.TYPE_UNUSED_VARIABLE;
            if (g.tok instanceof SourceToken) {
                SourceToken t = (SourceToken) g.tok;
                SimpleNode ast = t.getAst();
                if (ast instanceof NameTok) {
                    NameTok n = (NameTok) ast;
                    if (n.ctx == NameTok.KwArg || n.ctx == NameTok.VarArg || n.ctx == NameTok.KeywordName) {
                        type = IAnalysisPreferences.TYPE_UNUSED_PARAMETER;
                    }
                } else if (ast instanceof Name) {
                    Name n = (Name) ast;
                    if (n.ctx == Name.Param || n.ctx == Name.KwOnlyParam) {
                        type = IAnalysisPreferences.TYPE_UNUSED_PARAMETER;
                    }
                }
            }
            boolean addMessage = true;
            if (type == IAnalysisPreferences.TYPE_UNUSED_PARAMETER) {
                if (node instanceof FunctionDef) {
                    addMessage = false;
                    FunctionDef def = (FunctionDef) node;
                    for (stmtType b : def.body) {
                        if (b instanceof Pass) {
                            continue;
                        }
                        if (b instanceof Expr) {
                            Expr expr = (Expr) b;
                            if (expr.value instanceof Str) {
                                continue;
                            }
                        }
                        addMessage = true;
                        break;
                    }
                }
            }
            if (addMessage) {
                addMessage(type, g.generator, g.tok);
            }
        }
    }
}

Example 3 with NameTok

use of org.python.pydev.parser.jython.ast.NameTok in project Pydev by fabioz.

the class TreeBuilder25 method makeArguments.

private argumentsType makeArguments(int l) throws Exception {
    NameTok kwarg = null;
    NameTok stararg = null;
    if (l > 0 && stack.peekNode().getId() == JJTEXTRAKEYWORDLIST) {
        ExtraArg node = (ExtraArg) stack.popNode();
        kwarg = node.tok;
        l--;
        addSpecialsAndClearOriginal(node, kwarg);
    }
    if (l > 0 && stack.peekNode().getId() == JJTEXTRAARGLIST) {
        ExtraArg node = (ExtraArg) stack.popNode();
        stararg = node.tok;
        l--;
        addSpecialsAndClearOriginal(node, stararg);
    }
    ArrayList<SimpleNode> list = new ArrayList<SimpleNode>();
    for (int i = l - 1; i >= 0; i--) {
        SimpleNode popped = stack.popNode();
        try {
            list.add(popped);
        } catch (ClassCastException e) {
            throw new ParseException("Internal error (ClassCastException):" + e.getMessage() + "\n" + popped, popped);
        }
    }
    // we get them in reverse order in the stack
    Collections.reverse(list);
    return makeArguments(list.toArray(new DefaultArg[0]), stararg, kwarg);
}

Example 4 with NameTok

use of org.python.pydev.parser.jython.ast.NameTok in project Pydev by fabioz.

the class FastDefinitionsParser method extractBody.

/**
 * This is the method that actually extracts things from the passed buffer.
 * @throws SyntaxErrorException
 */
private void extractBody() throws SyntaxErrorException {
    ParsingUtils parsingUtils = ParsingUtils.create(cs, false, length);
    if (currIndex < length) {
        handleNewLine(parsingUtils);
    }
    // in the 1st attempt to handle the 1st line, if it had nothing we could actually go backward 1 char
    if (currIndex < 0) {
        currIndex = 0;
    }
    for (; currIndex < length; currIndex++, col++) {
        char c = cs[currIndex];
        switch(c) {
            case '\'':
            case '"':
                if (DEBUG) {
                    System.out.println("literal");
                }
                // go to the end of the literal
                int initialIndex = currIndex;
                currIndex = parsingUtils.getLiteralEnd(currIndex, c);
                // keep the row count correct
                updateCountRow(initialIndex, currIndex);
                break;
            case '#':
                if (DEBUG) {
                    System.out.println("comment");
                }
                // go to the end of the comment
                while (currIndex < length) {
                    c = cs[currIndex];
                    if (c == '\r' || c == '\n') {
                        currIndex--;
                        break;
                    }
                    currIndex++;
                }
                break;
            case '{':
            case '[':
            case '(':
                // starting some call, dict, list, tuple... those don't count on getting some actual definition
                initialIndex = currIndex;
                currIndex = parsingUtils.eatPar(currIndex, null, c);
                // keep the row count correct
                updateCountRow(initialIndex, currIndex);
                break;
            case '\r':
                if (currIndex < length - 1 && cs[currIndex + 1] == '\n') {
                    currIndex++;
                }
            /*FALLTHROUGH**/
            case '\n':
                currIndex++;
                handleNewLine(parsingUtils);
                if (currIndex < length) {
                    c = cs[currIndex];
                }
                break;
            case '=':
                if ((currIndex < length - 1 && cs[currIndex + 1] != '=' && currIndex > 0 && cs[currIndex - 1] != '=')) {
                    if (DEBUG) {
                        System.out.println("Found possible attribute:" + lineBuffer + " col:" + firstCharCol);
                    }
                    // if we've an '=', let's get the whole line contents to analyze...
                    // Note: should have stopped just before the new line (so, as we'll do currIndex++ in the
                    // next loop, that's ok).
                    initialIndex = currIndex;
                    currIndex = parsingUtils.getFullFlattenedLine(currIndex, lineBuffer);
                    // keep the row count correct
                    updateCountRow(initialIndex, currIndex);
                    String equalsLine = lineBuffer.toString().trim();
                    if (!PySelection.startsWithIndentToken(equalsLine)) {
                        lineBuffer.clear();
                        final List<String> splitted = StringUtils.split(equalsLine, '=');
                        final int splittedLen = splitted.size();
                        ArrayList<exprType> targets = new ArrayList<exprType>(2);
                        for (int j = 0; j < splittedLen - 1 || (splittedLen == 1 && j == 0); j++) {
                            // we don't want to get the last one.
                            int addCols = 0;
                            if (j > 0) {
                                for (int k = 0; k < j; k++) {
                                    addCols += splitted.get(j).length();
                                    addCols += 1;
                                }
                            }
                            String lineContents = splitted.get(j).trim();
                            if (lineContents.length() == 0) {
                                continue;
                            }
                            int colonIndex = lineContents.indexOf(':');
                            if (colonIndex > 0) {
                                lineContents = lineContents.substring(0, colonIndex);
                            }
                            boolean add = true;
                            int lineContentsLen = lineContents.length();
                            for (int i = 0; i < lineContentsLen; i++) {
                                char lineC = lineContents.charAt(i);
                                // can only be made of valid java chars (no spaces or similar things)
                                if (lineC != '.' && !Character.isJavaIdentifierPart(lineC)) {
                                    add = false;
                                    break;
                                }
                            }
                            if (add) {
                                // only add if it was something valid
                                if (lineContents.indexOf('.') != -1) {
                                    List<String> dotSplit = StringUtils.dotSplit(lineContents);
                                    if (dotSplit.size() == 2 && dotSplit.get(0).equals("self")) {
                                        Name selfName = new Name("self", Name.Load, false);
                                        NameTok attribName = new NameTok(dotSplit.get(1), NameTok.Attrib);
                                        selfName.beginLine = row;
                                        selfName.beginColumn = this.firstCharCol;
                                        attribName.beginLine = row;
                                        attribName.beginColumn = this.firstCharCol;
                                        Attribute attribute = new Attribute(selfName, attribName, Attribute.Load);
                                        attribute.beginLine = row;
                                        attribute.beginColumn = this.firstCharCol;
                                        targets.add(attribute);
                                    }
                                } else {
                                    Name name = new Name(lineContents, Name.Store, false);
                                    name.beginLine = row;
                                    name.beginColumn = this.firstCharCol + addCols;
                                    targets.add(name);
                                }
                            }
                        }
                        if (targets.size() > 0) {
                            Assign assign = new Assign(targets.toArray(new exprType[targets.size()]), null, null);
                            assign.beginColumn = this.firstCharCol;
                            assign.beginLine = this.row;
                            stack.push(new NodeEntry(assign, leadingTabsInLine));
                        }
                    }
                }
        }
        lineBuffer.append(c);
    }
    endScopesInStack(0);
}

Example 5 with NameTok

use of org.python.pydev.parser.jython.ast.NameTok in project Pydev by fabioz.

the class FastParser method createNameTok.

/**
 * @param matcher this is the class that just matched the class or function
 * @param lastReturnedLine the line it has done the match
 * @param type the type of the name token (@see NameTok constants)
 * @param ps the pyselection that has the document
 * @return null if the location is not a valid location for a function or class or a NameTok to
 * be used with the ClassDef / FunctionDef
 */
private NameTok createNameTok(Matcher matcher, int lastReturnedLine, int type, PySelection ps) {
    int col = matcher.start(NAME_GROUP);
    int absoluteCursorOffset = ps.getAbsoluteCursorOffset(lastReturnedLine, col);
    if (!IPythonPartitions.PY_DEFAULT.equals(ParsingUtils.getContentType(ps.getDoc(), absoluteCursorOffset))) {
        return null;
    }
    NameTok nameTok = new NameTok(matcher.group(NAME_GROUP), type);
    nameTok.beginLine = lastReturnedLine + 1;
    nameTok.beginColumn = col + 1;
    return nameTok;
}