Examples with BufferedTokenStream org.antlr.v4.runtime.BufferedTokenStream used on opensource projects

Example 1 with BufferedTokenStream

use of org.antlr.v4.runtime.BufferedTokenStream in project titan.EclipsePlug-ins by eclipse.

the class TTCN3ReparseUpdater method parse.

public int parse(final ITTCN3ReparseBase userDefined) {
    if (modificationStartOffset == modificationEndOffset + shift) {
        return 0;
    }
    // double wideparsing = System.nanoTime();
    mErrors = null;
    warnings = null;
    Iterator<TITANMarker> iterator = unsupportedConstructs.iterator();
    while (iterator.hasNext()) {
        TITANMarker marker = iterator.next();
        if ((marker.getOffset() > modificationStartOffset && marker.getOffset() <= modificationEndOffset) || (marker.getEndOffset() > modificationStartOffset && marker.getEndOffset() <= modificationEndOffset)) {
            iterator.remove();
        }
    }
    MarkerHandler.markAllOnTheFlyMarkersForRemoval(file, modificationStartOffset, modificationEndOffset + shift);
    if (code == null) {
        return Integer.MAX_VALUE;
    }
    int line = getLineOfOffset(code, modificationStartOffset);
    String substring;
    if (code.length() <= modificationEndOffset + shift) {
        substring = code.substring(modificationStartOffset);
    } else {
        substring = code.substring(modificationStartOffset, modificationEndOffset + shift);
    }
    Reader reader = new StringReader(substring);
    CharStream charStream = new UnbufferedCharStream(reader);
    Ttcn3Lexer lexer = new Ttcn3Lexer(charStream);
    lexer.setTokenFactory(new CommonTokenFactory(true));
    lexer.initRootInterval(modificationEndOffset - modificationStartOffset + 1);
    // lexer and parser listener
    TitanListener parserListener = new TitanListener();
    // remove ConsoleErrorListener
    lexer.removeErrorListeners();
    lexer.addErrorListener(parserListener);
    // 1. Previously it was UnbufferedTokenStream(lexer), but it was changed to BufferedTokenStream, because UnbufferedTokenStream seems to be unusable. It is an ANTLR 4 bug.
    // Read this: https://groups.google.com/forum/#!topic/antlr-discussion/gsAu-6d3pKU
    // pr_PatternChunk[StringBuilder builder, boolean[] uni]:
    // $builder.append($v.text); <-- exception is thrown here: java.lang.UnsupportedOperationException: interval 85..85 not in token buffer window: 86..341
    // 2. Changed from BufferedTokenStream to CommonTokenStream, otherwise tokens with "-> channel(HIDDEN)" are not filtered out in lexer.
    final CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    Ttcn3Reparser parser = new Ttcn3Reparser(tokenStream);
    ParserUtilities.setBuildParseTree(parser);
    lexer.setActualFile(file);
    parser.setActualFile(file);
    parser.setProject(file.getProject());
    parser.setOffset(modificationStartOffset);
    parser.setLine(line + 1);
    // remove ConsoleErrorListener
    parser.removeErrorListeners();
    parser.addErrorListener(parserListener);
    userDefined.reparse(parser);
    mErrors = parserListener.getErrorsStored();
    warnings = parser.getWarnings();
    unsupportedConstructs.addAll(parser.getUnsupportedConstructs());
    int result = measureIntervallDamage();
    if (!parser.isErrorListEmpty()) {
        ++result;
    }
    return result;
}

Example 2 with BufferedTokenStream

use of org.antlr.v4.runtime.BufferedTokenStream in project titan.EclipsePlug-ins by eclipse.

the class ExtensionAttributeAnalyzer method parse.

public void parse(final AttributeSpecification specification) {
    ExtensionAttributeLexer lexer;
    Location location = specification.getLocation();
    StringReader reader = new StringReader(specification.getSpecification());
    CharStream charStream = new UnbufferedCharStream(reader);
    lexer = new ExtensionAttributeLexer(charStream);
    lexer.setTokenFactory(new CommonTokenFactory(true));
    TitanListener lexerListener = new TitanListener();
    lexer.removeErrorListeners();
    lexer.addErrorListener(lexerListener);
    // 1. Previously it was UnbufferedTokenStream(lexer), but it was changed to BufferedTokenStream, because UnbufferedTokenStream seems to be unusable. It is an ANTLR 4 bug.
    // Read this: https://groups.google.com/forum/#!topic/antlr-discussion/gsAu-6d3pKU
    // pr_PatternChunk[StringBuilder builder, boolean[] uni]:
    // $builder.append($v.text); <-- exception is thrown here: java.lang.UnsupportedOperationException: interval 85..85 not in token buffer window: 86..341
    // 2. Changed from BufferedTokenStream to CommonTokenStream, otherwise tokens with "-> channel(HIDDEN)" are not filtered out in lexer.
    final CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    ExtensionAttributeParser parser = new ExtensionAttributeParser(tokenStream);
    parser.setBuildParseTree(false);
    TitanListener parserListener = new TitanListener();
    parser.removeErrorListeners();
    parser.addErrorListener(parserListener);
    parser.setActualFile((IFile) location.getFile());
    parser.setLine(location.getLine());
    parser.setOffset(location.getOffset() + 1);
    MarkerHandler.markMarkersForRemoval(GeneralConstants.ONTHEFLY_SYNTACTIC_MARKER, location.getFile(), location.getOffset(), location.getEndOffset());
    attributes = null;
    attributes = parser.pr_ExtensionAttributeRoot().list;
    if (!lexerListener.getErrorsStored().isEmpty()) {
        String reportLevel = Platform.getPreferencesService().getString(ProductConstants.PRODUCT_ID_DESIGNER, PreferenceConstants.REPORTERRORSINEXTENSIONSYNTAX, GeneralConstants.WARNING, null);
        int errorLevel;
        if (GeneralConstants.ERROR.equals(reportLevel)) {
            errorLevel = IMarker.SEVERITY_ERROR;
        } else if (GeneralConstants.WARNING.equals(reportLevel)) {
            errorLevel = IMarker.SEVERITY_WARNING;
        } else {
            return;
        }
        for (int i = 0; i < lexerListener.getErrorsStored().size(); i++) {
            Location temp = new Location(location);
            temp.setOffset(temp.getOffset() + 1);
            ParserMarkerSupport.createOnTheFlyMixedMarker((IFile) location.getFile(), lexerListener.getErrorsStored().get(i), errorLevel, temp);
        }
    }
    if (!parserListener.getErrorsStored().isEmpty()) {
        String reportLevel = Platform.getPreferencesService().getString(ProductConstants.PRODUCT_ID_DESIGNER, PreferenceConstants.REPORTERRORSINEXTENSIONSYNTAX, GeneralConstants.WARNING, null);
        int errorLevel;
        if (GeneralConstants.ERROR.equals(reportLevel)) {
            errorLevel = IMarker.SEVERITY_ERROR;
        } else if (GeneralConstants.WARNING.equals(reportLevel)) {
            errorLevel = IMarker.SEVERITY_WARNING;
        } else {
            return;
        }
        for (int i = 0; i < parserListener.getErrorsStored().size(); i++) {
            Location temp = new Location(location);
            temp.setOffset(temp.getOffset() + 1);
            ParserMarkerSupport.createOnTheFlyMixedMarker((IFile) location.getFile(), parserListener.getErrorsStored().get(i), errorLevel, temp);
        }
    }
}

Example 3 with BufferedTokenStream

use of org.antlr.v4.runtime.BufferedTokenStream in project checkstyle by checkstyle.

the class JavadocDetailNodeParser method getMissedHtmlTag.

/**
 * Method to get the missed HTML tag to generate more informative error message for the user.
 * This method doesn't concern itself with
 * <a href="https://www.w3.org/TR/html51/syntax.html#void-elements">void elements</a>
 * since it is forbidden to close them.
 * Missed HTML tags for the following tags will <i>not</i> generate an error message from ANTLR:
 * {@code
 * <p>
 * <li>
 * <tr>
 * <td>
 * <th>
 * <body>
 * <colgroup>
 * <dd>
 * <dt>
 * <head>
 * <html>
 * <option>
 * <tbody>
 * <thead>
 * <tfoot>
 * }
 *
 * @param exception {@code NoViableAltException} object catched while parsing javadoc
 * @return returns appropriate {@link Token} if a HTML close tag is missed;
 *     null otherwise
 */
private static Token getMissedHtmlTag(RecognitionException exception) {
    Token htmlTagNameStart = null;
    final Interval sourceInterval = exception.getCtx().getSourceInterval();
    final List<Token> tokenList = ((BufferedTokenStream) exception.getInputStream()).getTokens(sourceInterval.a, sourceInterval.b);
    final Deque<Token> stack = new ArrayDeque<>();
    int prevTokenType = JavadocTokenTypes.EOF;
    for (final Token token : tokenList) {
        final int tokenType = token.getType();
        if (tokenType == JavadocTokenTypes.HTML_TAG_NAME && prevTokenType == JavadocTokenTypes.START) {
            stack.push(token);
        } else if (tokenType == JavadocTokenTypes.HTML_TAG_NAME && !stack.isEmpty()) {
            if (stack.peek().getText().equals(token.getText())) {
                stack.pop();
            } else {
                htmlTagNameStart = stack.pop();
            }
        }
        prevTokenType = tokenType;
    }
    if (htmlTagNameStart == null) {
        htmlTagNameStart = stack.pop();
    }
    return htmlTagNameStart;
}

Example 4 with BufferedTokenStream

use of org.antlr.v4.runtime.BufferedTokenStream in project oap by oaplatform.

the class JPath method evaluate.

public void evaluate(String expression, JPathOutput output) {
    var jPathParser = new JPathParser(new BufferedTokenStream(new JPathLexer(new ANTLRInputStream(expression))));
    jPathParser.expr().expression.evaluate(variables, output);
}

Example 5 with BufferedTokenStream

use of org.antlr.v4.runtime.BufferedTokenStream in project oap by oaplatform.

the class TemplateEngine method getTemplate.

@SuppressWarnings("unchecked")
public <TIn, TOut, TOutMutable, TA extends TemplateAccumulator<TOut, TOutMutable, TA>> Template<TIn, TOut, TOutMutable, TA> getTemplate(String name, TypeRef<TIn> type, String template, TA acc, Map<String, String> aliases, ErrorStrategy errorStrategy, Consumer<Ast> postProcess) {
    assert template != null;
    assert acc != null;
    var id = name + "_" + getHash(template) + "_" + aliases.hashCode() + "_" + acc.getClass().hashCode() + (postProcess != null ? "_" + postProcess.getClass().hashCode() : "");
    log.trace("id '{}' acc '{}' template '{}' aliases '{}'", id, acc.getClass(), template, aliases);
    try {
        TemplateFunction tFunc = templates.get(id, () -> {
            var lexer = new TemplateLexer(CharStreams.fromString(template));
            var grammar = new TemplateGrammar(new BufferedTokenStream(lexer), builtInFunction, errorStrategy);
            if (errorStrategy == ErrorStrategy.ERROR) {
                lexer.addErrorListener(ThrowingErrorListener.INSTANCE);
                grammar.addErrorListener(ThrowingErrorListener.INSTANCE);
            }
            var ast = grammar.template(new TemplateType(type.type()), aliases).rootAst;
            log.trace("\n" + ast.print());
            if (postProcess != null)
                postProcess.accept(ast);
            var tf = new JavaTemplate<>(name, type, tmpPath, acc, ast);
            return new TemplateFunction(tf, new Exception().getStackTrace());
        });
        return (Template<TIn, TOut, TOutMutable, TA>) tFunc.template;
    } catch (UncheckedExecutionException | ExecutionException e) {
        if (e.getCause() instanceof TemplateException) {
            throw (TemplateException) e.getCause();
        }
        throw new TemplateException(e.getCause());
    }
}