Examples with ParsingUtils org.python.pydev.core.docutils.ParsingUtils used on opensource projects

Example 1 with ParsingUtils

use of org.python.pydev.core.docutils.ParsingUtils in project Pydev by fabioz.

the class PyFormatter method formatStr.

/**
 * This method formats a string given some standard.
 *
 * @param str the string to be formatted
 * @param std the standard to be used
 * @param parensLevel the level of the parenthesis available.
 * @return a new (formatted) string
 * @throws SyntaxErrorException
 */
public static FastStringBuffer formatStr(String doc, FormatStd std, int parensLevel, String delimiter, boolean throwSyntaxError) throws SyntaxErrorException {
    final char[] cs = doc.toCharArray();
    FastStringBuffer buf = new FastStringBuffer();
    // Temporary buffer for some operations. Must always be cleared before it's used.
    FastStringBuffer tempBuf = new FastStringBuffer();
    ParsingUtils parsingUtils = ParsingUtils.create(cs, throwSyntaxError);
    char lastChar = '\0';
    final int length = cs.length;
    for (int i = 0; i < length; i++) {
        char c = cs[i];
        switch(c) {
            case '\'':
            case '"':
                // ignore literals and multi-line literals, including comments...
                i = parsingUtils.eatLiterals(buf, i, std.trimMultilineLiterals);
                break;
            case '#':
                i = handleComment(std, cs, buf, tempBuf, parsingUtils, i);
                break;
            case ',':
                i = formatForComma(std, cs, buf, i, tempBuf);
                break;
            case '(':
                i = formatForPar(parsingUtils, cs, i, std, buf, parensLevel + 1, delimiter, throwSyntaxError);
                break;
            // & ^ ~ |
            case '*':
                // for *, we also need to treat when it's used in varargs, kwargs and list expansion
                boolean isOperator = false;
                for (int j = buf.length() - 1; j >= 0; j--) {
                    char localC = buf.charAt(j);
                    if (Character.isWhitespace(localC)) {
                        continue;
                    }
                    if (localC == '(' || localC == ',') {
                    // it's not an operator, but vararg. kwarg or list expansion
                    }
                    if (Character.isJavaIdentifierPart(localC)) {
                        // ok, there's a chance that it can be an operator, but we still have to check
                        // the chance that it's a wild import
                        tempBuf.clear();
                        while (Character.isJavaIdentifierPart(localC)) {
                            tempBuf.append(localC);
                            j--;
                            if (j < 0) {
                                // break while
                                break;
                            }
                            localC = buf.charAt(j);
                        }
                        String reversed = tempBuf.reverse().toString();
                        if (!reversed.equals("import") && !reversed.equals("lambda")) {
                            isOperator = true;
                        }
                    }
                    if (localC == '\'' || localC == ')' || localC == ']') {
                        isOperator = true;
                    }
                    // If it got here (i.e.: not whitespace), get out of the for loop.
                    break;
                }
                if (!isOperator) {
                    buf.append('*');
                    // break switch
                    break;
                }
            case '+':
            case '-':
                if (c == '-' || c == '+') {
                    // could also be *
                    // handle exponentials correctly: e.g.: 1e-6 cannot have a space
                    tempBuf.clear();
                    boolean started = false;
                    for (int j = buf.length() - 1; ; j--) {
                        if (j < 0) {
                            break;
                        }
                        char localC = buf.charAt(j);
                        if (localC == ' ' || localC == '\t') {
                            if (!started) {
                                continue;
                            } else {
                                break;
                            }
                        }
                        started = true;
                        if (Character.isJavaIdentifierPart(localC) || localC == '.') {
                            tempBuf.append(localC);
                        } else {
                            // break for
                            break;
                        }
                    }
                    boolean isExponential = true;
                    String partialNumber = tempBuf.reverse().toString();
                    int partialLen = partialNumber.length();
                    if (partialLen < 2 || !Character.isDigit(partialNumber.charAt(0))) {
                        // at least 2 chars: the number and the 'e'
                        isExponential = false;
                    } else {
                        // first char checked... now, if the last is an 'e', we must leave it together no matter what
                        if (partialNumber.charAt(partialLen - 1) != 'e' && partialNumber.charAt(partialLen - 1) != 'E') {
                            isExponential = false;
                        }
                    }
                    if (isExponential) {
                        buf.rightTrimWhitespacesAndTabs();
                        buf.append(c);
                        // skip the next whitespaces from the buffer
                        int initial = i;
                        do {
                            i++;
                        } while (i < length && (c = cs[i]) == ' ' || c == '\t');
                        if (i > initial) {
                            // backup 1 because we walked 1 too much.
                            i--;
                        }
                        // break switch
                        break;
                    }
                // Otherwise, FALLTHROUGH
                }
            case '/':
            case '%':
            case '<':
            case '>':
            case '!':
            case '&':
            case '^':
            case '~':
            case '|':
            case ':':
                i = handleOperator(std, cs, buf, parsingUtils, i, c);
                c = cs[i];
                break;
            case '@':
                // @ can mean either a decorator or matrix multiplication,
                // If decorator, do nothing, for matrix multiplication, '@' is an operator which
                // may or may not be followed by an '='
                String append = "@";
                if (i < length - 1 && cs[i + 1] == '=') {
                    // @= found
                    i++;
                    append = "@=";
                } else if (buf.getLastWord().trim().isEmpty()) {
                    // decorator
                    buf.append('@');
                    break;
                }
                while (buf.length() > 0 && buf.lastChar() == ' ') {
                    buf.deleteLast();
                }
                if (std.operatorsWithSpace) {
                    buf.append(' ');
                }
                buf.append(append);
                // add space after
                if (std.operatorsWithSpace) {
                    buf.append(' ');
                }
                i = parsingUtils.eatWhitespaces(null, i + 1);
                break;
            // check for = and == (other cases that have an = as the operator should already be treated)
            case '=':
                if (i < length - 1 && cs[i + 1] == '=') {
                    // if == handle as if a regular operator
                    i = handleOperator(std, cs, buf, parsingUtils, i, c);
                    c = cs[i];
                    break;
                }
                while (buf.length() > 0 && buf.lastChar() == ' ') {
                    buf.deleteLast();
                }
                boolean surroundWithSpaces = std.operatorsWithSpace;
                if (parensLevel > 0) {
                    surroundWithSpaces = std.assignWithSpaceInsideParens;
                }
                // add space before
                if (surroundWithSpaces) {
                    buf.append(' ');
                }
                // add the operator and the '='
                buf.append('=');
                // add space after
                if (surroundWithSpaces) {
                    buf.append(' ');
                }
                i = parsingUtils.eatWhitespaces(null, i + 1);
                break;
            case '\r':
            case '\n':
                if (lastChar == ',' && std.spaceAfterComma && buf.lastChar() == ' ') {
                    buf.deleteLast();
                }
                if (std.trimLines) {
                    buf.rightTrimWhitespacesAndTabs();
                }
                buf.append(c);
                break;
            default:
                buf.append(c);
        }
        lastChar = c;
    }
    if (parensLevel == 0 && std.trimLines) {
        buf.rightTrimWhitespacesAndTabs();
    }
    return buf;
}

Example 2 with ParsingUtils

use of org.python.pydev.core.docutils.ParsingUtils in project Pydev by fabioz.

the class FastDefinitionsParser method extractBody.

/**
 * This is the method that actually extracts things from the passed buffer.
 * @throws SyntaxErrorException
 */
private void extractBody() throws SyntaxErrorException {
    ParsingUtils parsingUtils = ParsingUtils.create(cs, false, length);
    if (currIndex < length) {
        handleNewLine(parsingUtils);
    }
    // in the 1st attempt to handle the 1st line, if it had nothing we could actually go backward 1 char
    if (currIndex < 0) {
        currIndex = 0;
    }
    for (; currIndex < length; currIndex++, col++) {
        char c = cs[currIndex];
        switch(c) {
            case '\'':
            case '"':
                if (DEBUG) {
                    System.out.println("literal");
                }
                // go to the end of the literal
                int initialIndex = currIndex;
                currIndex = parsingUtils.getLiteralEnd(currIndex, c);
                // keep the row count correct
                updateCountRow(initialIndex, currIndex);
                break;
            case '#':
                if (DEBUG) {
                    System.out.println("comment");
                }
                // go to the end of the comment
                while (currIndex < length) {
                    c = cs[currIndex];
                    if (c == '\r' || c == '\n') {
                        currIndex--;
                        break;
                    }
                    currIndex++;
                }
                break;
            case '{':
            case '[':
            case '(':
                // starting some call, dict, list, tuple... those don't count on getting some actual definition
                initialIndex = currIndex;
                currIndex = parsingUtils.eatPar(currIndex, null, c);
                // keep the row count correct
                updateCountRow(initialIndex, currIndex);
                break;
            case '\r':
                if (currIndex < length - 1 && cs[currIndex + 1] == '\n') {
                    currIndex++;
                }
            /*FALLTHROUGH**/
            case '\n':
                currIndex++;
                handleNewLine(parsingUtils);
                if (currIndex < length) {
                    c = cs[currIndex];
                }
                break;
            case '=':
                if ((currIndex < length - 1 && cs[currIndex + 1] != '=' && currIndex > 0 && cs[currIndex - 1] != '=')) {
                    if (DEBUG) {
                        System.out.println("Found possible attribute:" + lineBuffer + " col:" + firstCharCol);
                    }
                    // if we've an '=', let's get the whole line contents to analyze...
                    // Note: should have stopped just before the new line (so, as we'll do currIndex++ in the
                    // next loop, that's ok).
                    initialIndex = currIndex;
                    currIndex = parsingUtils.getFullFlattenedLine(currIndex, lineBuffer);
                    // keep the row count correct
                    updateCountRow(initialIndex, currIndex);
                    String equalsLine = lineBuffer.toString().trim();
                    if (!PySelection.startsWithIndentToken(equalsLine)) {
                        lineBuffer.clear();
                        final List<String> splitted = StringUtils.split(equalsLine, '=');
                        final int splittedLen = splitted.size();
                        ArrayList<exprType> targets = new ArrayList<exprType>(2);
                        for (int j = 0; j < splittedLen - 1 || (splittedLen == 1 && j == 0); j++) {
                            // we don't want to get the last one.
                            int addCols = 0;
                            if (j > 0) {
                                for (int k = 0; k < j; k++) {
                                    addCols += splitted.get(j).length();
                                    addCols += 1;
                                }
                            }
                            String lineContents = splitted.get(j).trim();
                            if (lineContents.length() == 0) {
                                continue;
                            }
                            int colonIndex = lineContents.indexOf(':');
                            if (colonIndex > 0) {
                                lineContents = lineContents.substring(0, colonIndex);
                            }
                            boolean add = true;
                            int lineContentsLen = lineContents.length();
                            for (int i = 0; i < lineContentsLen; i++) {
                                char lineC = lineContents.charAt(i);
                                // can only be made of valid java chars (no spaces or similar things)
                                if (lineC != '.' && !Character.isJavaIdentifierPart(lineC)) {
                                    add = false;
                                    break;
                                }
                            }
                            if (add) {
                                // only add if it was something valid
                                if (lineContents.indexOf('.') != -1) {
                                    List<String> dotSplit = StringUtils.dotSplit(lineContents);
                                    if (dotSplit.size() == 2 && dotSplit.get(0).equals("self")) {
                                        Name selfName = new Name("self", Name.Load, false);
                                        NameTok attribName = new NameTok(dotSplit.get(1), NameTok.Attrib);
                                        selfName.beginLine = row;
                                        selfName.beginColumn = this.firstCharCol;
                                        attribName.beginLine = row;
                                        attribName.beginColumn = this.firstCharCol;
                                        Attribute attribute = new Attribute(selfName, attribName, Attribute.Load);
                                        attribute.beginLine = row;
                                        attribute.beginColumn = this.firstCharCol;
                                        targets.add(attribute);
                                    }
                                } else {
                                    Name name = new Name(lineContents, Name.Store, false);
                                    name.beginLine = row;
                                    name.beginColumn = this.firstCharCol + addCols;
                                    targets.add(name);
                                }
                            }
                        }
                        if (targets.size() > 0) {
                            Assign assign = new Assign(targets.toArray(new exprType[targets.size()]), null, null);
                            assign.beginColumn = this.firstCharCol;
                            assign.beginLine = this.row;
                            stack.push(new NodeEntry(assign, leadingTabsInLine));
                        }
                    }
                }
        }
        lineBuffer.append(c);
    }
    endScopesInStack(0);
}

Example 3 with ParsingUtils

use of org.python.pydev.core.docutils.ParsingUtils in project Pydev by fabioz.

the class PyPeerLinker method isLiteralBalanced.

/**
 * @return true if the passed string has balanced ' and "
 */
private boolean isLiteralBalanced(String cursorLineContents) {
    ParsingUtils parsingUtils = ParsingUtils.create(cursorLineContents, true);
    int offset = 0;
    int end = cursorLineContents.length();
    boolean balanced = true;
    while (offset < end) {
        char curr = cursorLineContents.charAt(offset++);
        if (curr == '"' || curr == '\'') {
            int eaten;
            try {
                eaten = parsingUtils.eatLiterals(null, offset - 1) + 1;
            } catch (SyntaxErrorException e) {
                balanced = false;
                break;
            }
            if (eaten > offset) {
                offset = eaten;
            }
        }
    }
    return balanced;
}

Example 4 with ParsingUtils

use of org.python.pydev.core.docutils.ParsingUtils in project Pydev by fabioz.

the class PyContextInformationValidator method getCurrentParameter.

/**
 * @param document the document from where the contents should be gotten.
 * @param start
 * @param end
 * @param increments this is the string that when found will increment the current parameter
 * @param decrements this is the string that when found will decrement the current parameter
 * @param considerNesting
 * @return
 * @throws BadLocationException
 * @throws SyntaxErrorException
 */
public int getCurrentParameter(IDocument document, final int start, final int end, String increments, String decrements, boolean considerNesting) throws BadLocationException, SyntaxErrorException {
    Assert.isTrue((increments.length() != 0 || decrements.length() != 0) && !increments.equals(decrements));
    final int NONE = 0;
    final int BRACKET = 1;
    final int BRACE = 2;
    final int PAREN = 3;
    final int ANGLE = 4;
    int nestingMode = NONE;
    int nestingLevel = 0;
    int charCount = 0;
    int offset = start;
    ParsingUtils parsingUtils = ParsingUtils.create(document);
    while (offset < end) {
        char curr = document.getChar(offset++);
        switch(curr) {
            case '#':
                if (offset < end) {
                    // '#' comment: nothing to do anymore on this line
                    offset = end;
                }
                break;
            case '"':
            case '\'':
                int eaten = parsingUtils.eatLiterals(null, offset - 1) + 1;
                if (eaten > offset) {
                    offset = eaten;
                }
                break;
            case '[':
                if (considerNesting) {
                    if (nestingMode == BRACKET || nestingMode == NONE) {
                        nestingMode = BRACKET;
                        nestingLevel++;
                    }
                    break;
                }
            case ']':
                if (considerNesting) {
                    if (nestingMode == BRACKET)
                        if (--nestingLevel == 0)
                            nestingMode = NONE;
                    break;
                }
            case '(':
                if (considerNesting) {
                    if (nestingMode == ANGLE) {
                        // generics heuristic failed
                        nestingMode = PAREN;
                        nestingLevel = 1;
                    }
                    if (nestingMode == PAREN || nestingMode == NONE) {
                        nestingMode = PAREN;
                        nestingLevel++;
                    }
                    break;
                }
            case ')':
                if (considerNesting) {
                    if (nestingMode == PAREN)
                        if (--nestingLevel == 0)
                            nestingMode = NONE;
                    break;
                }
            case '{':
                if (considerNesting) {
                    if (nestingMode == ANGLE) {
                        // generics heuristic failed
                        nestingMode = BRACE;
                        nestingLevel = 1;
                    }
                    if (nestingMode == BRACE || nestingMode == NONE) {
                        nestingMode = BRACE;
                        nestingLevel++;
                    }
                    break;
                }
            case '}':
                if (considerNesting) {
                    if (nestingMode == BRACE)
                        if (--nestingLevel == 0)
                            nestingMode = NONE;
                    break;
                }
            default:
                if (nestingLevel != 0)
                    continue;
                if (increments.indexOf(curr) >= 0) {
                    ++charCount;
                }
                if (decrements.indexOf(curr) >= 0) {
                    --charCount;
                }
        }
    }
    return charCount;
}

Example 5 with ParsingUtils

use of org.python.pydev.core.docutils.ParsingUtils in project Pydev by fabioz.

the class AssistFString method getProps.

@Override
public List<ICompletionProposalHandle> getProps(PySelection ps, IImageCache imageCache, File f, IPythonNature nature, IPyEdit edit, int offset) throws BadLocationException, MisconfigurationException {
    List<ICompletionProposalHandle> lst = new ArrayList<ICompletionProposalHandle>();
    IDocument doc = ps.getDoc();
    ITypedRegion partition = ((FastPartitioner) PyPartitionScanner.checkPartitionScanner(doc)).getPartition(offset);
    if (!ParsingUtils.isStringContentType(partition.getType())) {
        return lst;
    }
    // check if the string prefix is byte literal or if it is already a f-string
    int partitionOffset = partition.getOffset();
    char firstPrefix = doc.getChar(partitionOffset);
    char penultPrefix = ' ';
    if (firstPrefix != '\'' && firstPrefix == '\"') {
        penultPrefix = doc.getChar(partitionOffset + 1);
    }
    if (firstPrefix == 'b' || penultPrefix == 'b' || firstPrefix == 'f' || penultPrefix == 'f') {
        return lst;
    }
    final String stringPartitionContents = doc.get(partitionOffset, partition.getLength());
    int formatCount = 0;
    // count how many format variables the string have
    formatCount += StringUtils.count(stringPartitionContents, "%s");
    formatCount += StringUtils.count(stringPartitionContents, "%r");
    ParsingUtils parsingUtils = ParsingUtils.create(doc);
    // get string end offset
    int partitionEndOffset = partitionOffset + partition.getLength();
    // we have to check if this is really a valid format statement
    // first of all, let's iterate to search for a %
    int i = partitionEndOffset;
    boolean validStmt = false;
    while (i < parsingUtils.len()) {
        char c = parsingUtils.charAt(i);
        if (Character.isWhitespace(c)) {
            i++;
            continue;
        }
        if (c == '%') {
            if (formatCount == 0) {
                // there is no reason to search for variables to format
                return lst;
            }
            validStmt = true;
            i++;
        }
        break;
    }
    if (validStmt) {
        try {
            // get format end offset and capture all the variables to format
            int formatVariablesLen = -1;
            boolean acceptMultiLines = false;
            List<String> variables = new ArrayList<String>();
            while (i < parsingUtils.len()) {
                char c = parsingUtils.charAt(i);
                if (Character.isWhitespace(c)) {
                    i++;
                    continue;
                }
                if (c == '{' || c == '%') {
                    return lst;
                }
                if (c == '[' || c == '(') {
                    formatVariablesLen = parsingUtils.eatPar(i, null, c) - partitionEndOffset + 1;
                    if (partitionEndOffset + formatVariablesLen > parsingUtils.len()) {
                        return lst;
                    }
                    i++;
                    c = parsingUtils.charAt(i);
                    acceptMultiLines = true;
                }
                int initial = i;
                while (i < parsingUtils.len()) {
                    c = parsingUtils.charAt(i);
                    if (Character.isJavaIdentifierPart(c) || c == ' ' || c == '.' || (acceptMultiLines && Character.isWhitespace(c))) {
                        i++;
                        continue;
                    }
                    if (c == '{' || c == '[' || c == '(' || c == '"' || c == '\'') {
                        i = parsingUtils.eatPar(i, null, c);
                        if (i < parsingUtils.len()) {
                            // checks first if we caught a closed partition
                            i++;
                            continue;
                        }
                        // it is not closed
                        return lst;
                    }
                    if (c == ',') {
                        variables.add(doc.get(initial, i - initial).trim());
                        i++;
                        initial = i;
                        continue;
                    }
                    break;
                }
                // format variables length has not been defined, so it is the end of the iteration
                if (formatVariablesLen == -1) {
                    formatVariablesLen = i - partitionEndOffset;
                }
                // check if format variables ended with comma or if there is a variable left to capture
                if (initial != i) {
                    String lastVarContent = doc.get(initial, i - initial).trim();
                    if (lastVarContent.length() > 0) {
                        variables.add(lastVarContent);
                    }
                }
                break;
            }
            // initialize format output string with the properly size allocation (f + string len + variables len)
            FastStringBuffer strBuf = new FastStringBuffer(1 + partition.getLength() + formatVariablesLen);
            strBuf.append('f').append(stringPartitionContents);
            // iterate through variables and edit the f-string output
            if (formatCount == variables.size()) {
                for (String variable : variables) {
                    String rep = getReplace(strBuf);
                    String with = null;
                    if ("%r".equals(rep)) {
                        with = "{" + variable + "!r}";
                    } else {
                        with = "{" + variable + "}";
                    }
                    strBuf.replaceFirst(rep, with);
                }
            } else if (variables.size() == 1) {
                String variable = variables.get(0);
                for (i = 0; i < formatCount; i++) {
                    String rep = getReplace(strBuf);
                    String with = null;
                    if ("%r".equals(rep)) {
                        with = "{" + variable + "[" + i + "]" + "!r}";
                    } else {
                        with = "{" + variable + "[" + i + "]" + "}";
                    }
                    strBuf.replaceFirst(rep, with);
                }
            } else {
                return lst;
            }
            lst.add(CompletionProposalFactory.get().createPyCompletionProposal(strBuf.toString(), partitionOffset, partition.getLength() + formatVariablesLen, 0, getImage(imageCache, UIConstants.COMPLETION_TEMPLATE), "Convert to f-string", null, null, IPyCompletionProposal.PRIORITY_DEFAULT, null));
            return lst;
        } catch (SyntaxErrorException e) {
        }
    }
    // if we got here, it means that we do not have % after the string
    FastStringBuffer buf = new FastStringBuffer(stringPartitionContents.length() + 1);
    buf.append('f').append(stringPartitionContents);
    lst.add(CompletionProposalFactory.get().createPyCompletionProposal(buf.toString(), partitionOffset, partition.getLength(), 0, getImage(imageCache, UIConstants.COMPLETION_TEMPLATE), "Convert to f-string", null, null, IPyCompletionProposal.PRIORITY_DEFAULT, null));
    return lst;
}