Examples with TokenMgrError org.python.pydev.parser.jython.TokenMgrError used on opensource projects

Example 1 with TokenMgrError

use of org.python.pydev.parser.jython.TokenMgrError in project Pydev by fabioz.

the class PyParser method reparseDocument.

/**
 * Parses the document, generates error annotations
 *
 * @param argsToReparse: will be passed to fireParserError / fireParserChanged so that the IParserObserver2
 * can check it. This is useful when the reparse was done with some specific thing in mind, so that its requestor
 * can pass some specific thing to the parser observers
 *
 * @return a tuple with the SimpleNode root(if parsed) and the error (if any).
 *         if we are able to recover from a reparse, we have both, the root and the error.
 */
@Override
public ParseOutput reparseDocument(Object... argsToReparse) {
    // get the document ast and error in object
    int version;
    AdditionalGrammarVersionsToCheck additionalGrammarsToCheck = null;
    try {
        version = grammarVersionProvider.getGrammarVersion();
        additionalGrammarsToCheck = grammarVersionProvider.getAdditionalGrammarVersions();
    } catch (MisconfigurationException e1) {
        // Ok, we cannot get it... let's put on the default
        version = IGrammarVersionProvider.LATEST_GRAMMAR_PY3_VERSION;
    }
    long documentTime = System.currentTimeMillis();
    ParseOutput obj = reparseDocument(new ParserInfo(document, version, true, additionalGrammarsToCheck));
    IFile original = null;
    IAdaptable adaptable = null;
    if (input == null) {
        return obj;
    }
    original = (input instanceof IAdaptable) ? ((IAdaptable) input).getAdapter(IFile.class) : null;
    if (original != null) {
        adaptable = original;
    } else {
        // probably an external file, may have some location provider mechanism
        // it may be org.eclipse.ui.internal.editors.text.JavaFileEditorInput
        adaptable = (IAdaptable) input;
    }
    // delete the markers
    if (original != null) {
        try {
            deleteErrorMarkers(original);
        } catch (ResourceException e) {
            // so, there is no need to log this failure
            if (original.exists()) {
                Log.log(e);
            }
        } catch (CoreException e) {
            Log.log(e);
        }
    } else if (adaptable == null) {
        // ok, we have nothing... maybe we are in tests...
        if (!PyParser.ACCEPT_NULL_INPUT_EDITOR) {
            throw new RuntimeException("Null input editor received in parser!");
        }
    }
    if (disposed) {
        // if it was disposed in this time, don't fire any notification nor return anything valid.
        return new ParseOutput();
    }
    ErrorParserInfoForObservers errorInfo = null;
    if (obj.error instanceof ParseException || obj.error instanceof TokenMgrError) {
        errorInfo = new ErrorParserInfoForObservers(obj.error, adaptable, document, argsToReparse);
    }
    if (obj.ast != null) {
        // Ok, reparse successful, lets erase the markers that are in the editor we just parsed
        // Note: we may get the ast even if errors happen (and we'll notify in that case too).
        ChangedParserInfoForObservers info = new ChangedParserInfoForObservers(obj.ast, obj.modificationStamp, adaptable, document, documentTime, errorInfo, argsToReparse);
        fireParserChanged(info);
    }
    if (errorInfo != null) {
        fireParserError(errorInfo);
    }
    if (postParserListeners.size() > 0) {
        ArrayList<IPostParserListener> tempList = new ArrayList<>(postParserListeners);
        for (IPostParserListener iParserObserver : tempList) {
            iParserObserver.participantsNotified(argsToReparse);
        }
    }
    return obj;
}

Example 2 with TokenMgrError

use of org.python.pydev.parser.jython.TokenMgrError in project Pydev by fabioz.

the class VisitorFactory method getRootNode.

public static Module getRootNode(IDocument doc, IGrammarVersionProvider versionProvider) throws ParseException, MisconfigurationException {
    ParseOutput objects = PyParser.reparseDocument(new PyParser.ParserInfo(doc, versionProvider));
    Throwable exception = objects.error;
    if (exception != null) {
        /* We try to get rid of the 'Throwable' exception, if possible */
        if (exception instanceof ParseException) {
            throw (ParseException) exception;
        } else if (exception instanceof TokenMgrError) {
            /* Error from Lexer */
            throw new ParseException(exception.toString());
        } else {
            throw new RuntimeException(exception);
        }
    }
    if (objects.error != null) {
        throw new RuntimeException(objects.error);
    }
    return (Module) objects.ast;
}

Example 3 with TokenMgrError

use of org.python.pydev.parser.jython.TokenMgrError in project Pydev by fabioz.

the class RefactoringInfo method getParsedUserSelection.

public ModuleAdapter getParsedUserSelection() {
    ModuleAdapter parsedAdapter = null;
    String source = normalizeSourceSelection(this.userSelection);
    if (this.userSelection != null && source.length() > 0) {
        try {
            parsedAdapter = VisitorFactory.createModuleAdapter(moduleManager, null, new Document(source), nature, this.versionProvider);
        } catch (TokenMgrError e) {
            return null;
        } catch (ParseException e) {
            /* Parse Exception means the current selection is invalid, discard and return null */
            return null;
        } catch (Throwable e) {
            throw new RuntimeException(e);
        }
    }
    return parsedAdapter;
}

Example 4 with TokenMgrError

use of org.python.pydev.parser.jython.TokenMgrError in project Pydev by fabioz.

the class PyParser method createErrorDesc.

/**
 * Creates the error description for a given error in the parse.
 *
 * Must return an error!
 */
public static ErrorDescription createErrorDesc(Throwable error, IDocument doc) {
    try {
        int errorStart = -1;
        int errorEnd = -1;
        int errorLine = -1;
        String message = null;
        int tokenBeginLine = -1;
        if (error instanceof ParseException) {
            ParseException parseErr = (ParseException) error;
            message = parseErr.getMessage();
            // marker for it
            if (parseErr.currentToken == null) {
                try {
                    IRegion endLine = doc.getLineInformationOfOffset(doc.getLength());
                    errorStart = endLine.getOffset();
                    errorEnd = endLine.getOffset() + endLine.getLength();
                } catch (BadLocationException e) {
                // ignore (can have changed in the meanwhile)
                }
            } else {
                Token errorToken = parseErr.currentToken.next != null ? parseErr.currentToken.next : parseErr.currentToken;
                if (errorToken != null) {
                    tokenBeginLine = errorToken.beginLine - 1;
                    try {
                        IRegion startLine = doc.getLineInformation(getDocPosFromAstPos(errorToken.beginLine));
                        IRegion endLine;
                        if (errorToken.endLine == 0) {
                            endLine = startLine;
                        } else {
                            endLine = doc.getLineInformation(getDocPosFromAstPos(errorToken.endLine));
                        }
                        errorStart = startLine.getOffset() + getDocPosFromAstPos(errorToken.beginColumn);
                        errorEnd = endLine.getOffset() + errorToken.endColumn;
                    } catch (BadLocationException e) {
                    // ignore (can have changed in the meanwhile)
                    }
                }
            }
        } else if (error instanceof TokenMgrError) {
            TokenMgrError tokenErr = (TokenMgrError) error;
            message = tokenErr.getMessage();
            tokenBeginLine = tokenErr.errorLine - 1;
            try {
                IRegion startLine = doc.getLineInformation(tokenErr.errorLine - 1);
                errorStart = startLine.getOffset();
                errorEnd = startLine.getOffset() + tokenErr.errorColumn;
            } catch (BadLocationException e) {
            // ignore (can have changed in the meanwhile)
            }
        } else {
            Log.log("Error, expecting ParseException or TokenMgrError. Received: " + error, error);
            return new ErrorDescription("Internal PyDev Error", 0, 0, 0);
        }
        try {
            errorLine = doc.getLineOfOffset(errorStart) + 1;
        } catch (BadLocationException e) {
            errorLine = tokenBeginLine;
        }
        // in task manager
        if (message != null) {
            // prettyprint
            message = StringUtils.replaceNewLines(message, " ");
        }
        return new ErrorDescription(message, errorLine, errorStart, errorEnd);
    } catch (Exception e) {
        Log.log(e);
        return new ErrorDescription("Internal PyDev Error", 0, 0, 0);
    }
}

Example 5 with TokenMgrError

use of org.python.pydev.parser.jython.TokenMgrError in project Pydev by fabioz.

the class PyParser method reparseDocument.

/**
 * @return a tuple with the SimpleNode root(if parsed) and the error (if any).
 *         if we are able to recover from a reparse, we have both, the root and the error.
 */
public static ParseOutput reparseDocument(ParserInfo info) {
    if (info.grammarVersion == IPythonNature.GRAMMAR_PYTHON_VERSION_CYTHON) {
        return createCythonAst(info);
    }
    // create a stream with document's data
    // Note: safer could be locking, but if for some reason we get the modification stamp and the document changes
    // right after that, at least any cache will check against the old stamp to be reconstructed (which is the main
    // reason for this stamp).
    long modifiedTime = ((IDocumentExtension4) info.document).getModificationStamp();
    String startDoc = info.document.get();
    if (startDoc.trim().length() == 0) {
        // If empty, don't bother to parse!
        return new ParseOutput(new Module(new stmtType[0]), null, modifiedTime);
    }
    Set<Integer> parsedVersions = new HashSet<>();
    char[] charArray;
    try {
        charArray = createCharArrayToParse(startDoc);
    } catch (OutOfMemoryError e1) {
        OnExpectedOutOfMemory.clearCacheOnOutOfMemory.call(null);
        // retry now with caches cleared...
        charArray = createCharArrayToParse(startDoc);
    }
    // it can be garbage-collected now.
    startDoc = null;
    Tuple<ISimpleNode, Throwable> returnVar = new Tuple<ISimpleNode, Throwable>(null, null);
    IGrammar grammar = null;
    try {
        parsedVersions.add(info.grammarVersion);
        grammar = createGrammar(info.generateTree, info.grammarVersion, charArray);
        SimpleNode newRoot;
        try {
            newRoot = grammar.file_input();
        } catch (OutOfMemoryError e) {
            OnExpectedOutOfMemory.clearCacheOnOutOfMemory.call(null);
            // retry now with caches cleared...
            newRoot = grammar.file_input();
        }
        returnVar.o1 = newRoot;
        // only notify successful parses
        if (successfulParseListeners.size() > 0) {
            Tuple3<ISimpleNode, Throwable, ParserInfo> param = new Tuple3<ISimpleNode, Throwable, ParserInfo>(returnVar.o1, returnVar.o2, info);
            for (ICallback<Object, Tuple3<ISimpleNode, Throwable, ParserInfo>> callback : successfulParseListeners) {
                callback.call(param);
            }
        }
        returnVar.o2 = grammar.getErrorOnParsing();
        if (returnVar.o2 == null) {
            AdditionalGrammarVersionsToCheck additionalGrammarVersionsToCheck = info.additionalGrammarVersionsToCheck;
            if (additionalGrammarVersionsToCheck != null) {
                for (int grammarVersion : additionalGrammarVersionsToCheck.getGrammarVersions()) {
                    if (parsedVersions.contains(grammarVersion)) {
                        continue;
                    }
                    parsedVersions.add(grammarVersion);
                    grammar = createGrammar(false, grammarVersion, charArray);
                    try {
                        grammar.file_input();
                    } catch (OutOfMemoryError e) {
                        OnExpectedOutOfMemory.clearCacheOnOutOfMemory.call(null);
                        // retry now with caches cleared...
                        grammar.file_input();
                    }
                    returnVar.o2 = grammar.getErrorOnParsing();
                    if (returnVar.o2 != null) {
                        break;
                    }
                }
            }
        }
    } catch (Throwable e) {
        // another parse.
        if (DEBUG_SHOW_PARSE_ERRORS) {
            e.printStackTrace();
        }
        // If the grammar was not created, the problem wasn't in the parsing... so, let's just rethrow the error
        if (grammar == null) {
            throw new RuntimeException(e);
        }
        // We have to change it for the 1st error we got (the one in catch is the last one).
        Throwable errorOnParsing = grammar.getErrorOnParsing();
        if (errorOnParsing != null) {
            e = errorOnParsing;
        } else if (DEBUG_SHOW_PARSE_ERRORS) {
            System.out.println("Unhandled error");
            e.printStackTrace();
        }
        grammar = null;
        if (e instanceof ParseException || e instanceof TokenMgrError) {
            returnVar = new Tuple<ISimpleNode, Throwable>(null, e);
        } else if (e.getClass().getName().indexOf("LookaheadSuccess") != -1) {
        // don't log this kind of error...
        } else {
            Log.log(e);
        }
    }
    if (DEBUG_SHOW_PARSE_ERRORS) {
        if (returnVar.o1 == null) {
            System.out.println("Unable to parse " + info);
        }
    }
    // System.out.println("Output grammar: "+returnVar);
    return new ParseOutput(returnVar, modifiedTime);
}