Examples with Lexer org.antlr.v4.runtime.Lexer used on opensource projects

Example 51 with Lexer

use of org.antlr.v4.runtime.Lexer in project titan.EclipsePlug-ins by eclipse.

the class TTCN3ReparseUpdater method endsWithToken.

/**
 * Checks if the last TTCN-3 lexical token in the substring, that covers the possibly changed interval of the document belongs to a given list of
 * expected tokens or not.
 *
 * @param followSet the possible tokens that can prepend the element before the suspected
 *
 * @return true if the first lexical token is part of the followset, false otherwise
 */
public boolean endsWithToken(final List<Integer> followSet) {
    if (followSet == null || followSet.isEmpty()) {
        return false;
    }
    if (code == null) {
        return false;
    }
    int line = getLineOfOffset(code, modificationStartOffset);
    int column = getPositionInLine(code, modificationStartOffset);
    String substring;
    if (code.length() <= modificationEndOffset + shift) {
        substring = code.substring(modificationStartOffset);
    } else {
        substring = code.substring(modificationStartOffset, modificationEndOffset + shift);
    }
    Reader reader = new StringReader(substring);
    CharStream charStream = new UnbufferedCharStream(reader);
    Ttcn3Lexer lexer = new Ttcn3Lexer(charStream);
    lexer.setTokenFactory(new CommonTokenFactory(true));
    lexer.setLine(line + 1);
    lexer.setCharPositionInLine(column);
    lexer.initRootInterval(modificationEndOffset - modificationStartOffset + 1);
    Token token = lexer.nextToken();
    if (token == null) {
        return false;
    }
    return followSet.contains(token.getType());
}

Example 52 with Lexer

use of org.antlr.v4.runtime.Lexer in project titan.EclipsePlug-ins by eclipse.

the class VariantAttributeAnalyzer method parse.

public void parse(final RawAST rawAST, final AttributeSpecification specification, final int lengthMultiplier, final AtomicBoolean raw_found) {
    VariantAttributeLexer lexer;
    Location location = specification.getLocation();
    StringReader reader = new StringReader(specification.getSpecification());
    CharStream charStream = new UnbufferedCharStream(reader);
    lexer = new VariantAttributeLexer(charStream);
    lexer.setTokenFactory(new CommonTokenFactory(true));
    TitanListener lexerListener = new TitanListener();
    lexer.removeErrorListeners();
    lexer.addErrorListener(lexerListener);
    final CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    VariantAttributeParser parser = new VariantAttributeParser(tokenStream);
    parser.setBuildParseTree(false);
    TitanListener parserListener = new TitanListener();
    parser.removeErrorListeners();
    parser.addErrorListener(parserListener);
    parser.setActualFile((IFile) location.getFile());
    parser.setLine(location.getLine());
    parser.setOffset(location.getOffset() + 1);
    MarkerHandler.markMarkersForRemoval(GeneralConstants.ONTHEFLY_SYNTACTIC_MARKER, location.getFile(), location.getOffset(), location.getEndOffset());
    parser.setRawAST(rawAST);
    parser.setLengthMultiplier(lengthMultiplier);
    parser.pr_AttribSpec();
    if (!lexerListener.getErrorsStored().isEmpty()) {
        for (int i = 0; i < lexerListener.getErrorsStored().size(); i++) {
            Location temp = new Location(location);
            temp.setOffset(temp.getOffset() + 1);
            ParserMarkerSupport.createOnTheFlyMixedMarker((IFile) location.getFile(), lexerListener.getErrorsStored().get(i), IMarker.SEVERITY_ERROR, temp);
        }
    }
    if (!parserListener.getErrorsStored().isEmpty()) {
        for (int i = 0; i < parserListener.getErrorsStored().size(); i++) {
            Location temp = new Location(location);
            temp.setOffset(temp.getOffset() + 1);
            ParserMarkerSupport.createOnTheFlyMixedMarker((IFile) location.getFile(), parserListener.getErrorsStored().get(i), IMarker.SEVERITY_ERROR, temp);
        }
    }
    if (!raw_found.get()) {
        raw_found.set(parser.getRawFound());
    }
}

Example 53 with Lexer

use of org.antlr.v4.runtime.Lexer in project titan.EclipsePlug-ins by eclipse.

the class ExtensionAttributeAnalyzer method parse.

public void parse(final AttributeSpecification specification) {
    ExtensionAttributeLexer lexer;
    Location location = specification.getLocation();
    StringReader reader = new StringReader(specification.getSpecification());
    CharStream charStream = new UnbufferedCharStream(reader);
    lexer = new ExtensionAttributeLexer(charStream);
    lexer.setTokenFactory(new CommonTokenFactory(true));
    TitanListener lexerListener = new TitanListener();
    lexer.removeErrorListeners();
    lexer.addErrorListener(lexerListener);
    // 1. Previously it was UnbufferedTokenStream(lexer), but it was changed to BufferedTokenStream, because UnbufferedTokenStream seems to be unusable. It is an ANTLR 4 bug.
    // Read this: https://groups.google.com/forum/#!topic/antlr-discussion/gsAu-6d3pKU
    // pr_PatternChunk[StringBuilder builder, boolean[] uni]:
    // $builder.append($v.text); <-- exception is thrown here: java.lang.UnsupportedOperationException: interval 85..85 not in token buffer window: 86..341
    // 2. Changed from BufferedTokenStream to CommonTokenStream, otherwise tokens with "-> channel(HIDDEN)" are not filtered out in lexer.
    final CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    ExtensionAttributeParser parser = new ExtensionAttributeParser(tokenStream);
    parser.setBuildParseTree(false);
    TitanListener parserListener = new TitanListener();
    parser.removeErrorListeners();
    parser.addErrorListener(parserListener);
    parser.setActualFile((IFile) location.getFile());
    parser.setLine(location.getLine());
    parser.setOffset(location.getOffset() + 1);
    MarkerHandler.markMarkersForRemoval(GeneralConstants.ONTHEFLY_SYNTACTIC_MARKER, location.getFile(), location.getOffset(), location.getEndOffset());
    attributes = null;
    attributes = parser.pr_ExtensionAttributeRoot().list;
    if (!lexerListener.getErrorsStored().isEmpty()) {
        String reportLevel = Platform.getPreferencesService().getString(ProductConstants.PRODUCT_ID_DESIGNER, PreferenceConstants.REPORTERRORSINEXTENSIONSYNTAX, GeneralConstants.WARNING, null);
        int errorLevel;
        if (GeneralConstants.ERROR.equals(reportLevel)) {
            errorLevel = IMarker.SEVERITY_ERROR;
        } else if (GeneralConstants.WARNING.equals(reportLevel)) {
            errorLevel = IMarker.SEVERITY_WARNING;
        } else {
            return;
        }
        for (int i = 0; i < lexerListener.getErrorsStored().size(); i++) {
            Location temp = new Location(location);
            temp.setOffset(temp.getOffset() + 1);
            ParserMarkerSupport.createOnTheFlyMixedMarker((IFile) location.getFile(), lexerListener.getErrorsStored().get(i), errorLevel, temp);
        }
    }
    if (!parserListener.getErrorsStored().isEmpty()) {
        String reportLevel = Platform.getPreferencesService().getString(ProductConstants.PRODUCT_ID_DESIGNER, PreferenceConstants.REPORTERRORSINEXTENSIONSYNTAX, GeneralConstants.WARNING, null);
        int errorLevel;
        if (GeneralConstants.ERROR.equals(reportLevel)) {
            errorLevel = IMarker.SEVERITY_ERROR;
        } else if (GeneralConstants.WARNING.equals(reportLevel)) {
            errorLevel = IMarker.SEVERITY_WARNING;
        } else {
            return;
        }
        for (int i = 0; i < parserListener.getErrorsStored().size(); i++) {
            Location temp = new Location(location);
            temp.setOffset(temp.getOffset() + 1);
            ParserMarkerSupport.createOnTheFlyMixedMarker((IFile) location.getFile(), parserListener.getErrorsStored().get(i), errorLevel, temp);
        }
    }
}

Example 54 with Lexer

use of org.antlr.v4.runtime.Lexer in project LogHub by fbacchella.

the class TestParser method testType.

@Test
public void testType() throws IOException {
    CharStream cs = CharStreams.fromStream(getClass().getClassLoader().getResourceAsStream("types.conf"), CharsetUtil.UTF_8);
    RouteLexer lexer = new RouteLexer(cs);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    RouteParser parser = new RouteParser(tokens);
    parser.removeErrorListeners();
    ConfigErrorListener errListener = new ConfigErrorListener();
    parser.addErrorListener(errListener);
    // begin parsing at init rule
    loghub.RouteParser.ConfigurationContext tree = parser.configuration();
    ConfigListener conf = new ConfigListener();
    ParseTreeWalker walker = new ParseTreeWalker();
    walker.walk(conf, tree);
    Assert.assertEquals("stack not empty :" + conf.stack, 0, conf.stack.size());
    ConfigListener.PipenodesList main = conf.pipelines.get("main");
    ObjectDescription p = (ObjectDescription) main.processors.get(0);
    Assert.assertTrue(((ObjectWrapped) p.beans.get("string")).wrapped instanceof String);
    Assert.assertTrue(((ObjectWrapped) p.beans.get("boolean")).wrapped instanceof Boolean);
    Assert.assertTrue(((ObjectWrapped) p.beans.get("int")).wrapped instanceof Integer);
    Assert.assertTrue(((ObjectWrapped) p.beans.get("double")).wrapped instanceof Double);
    Assert.assertArrayEquals(new Object[] { "0", 1, 1.0, true }, (Object[]) ((ObjectWrapped) p.beans.get("array")).wrapped);
}

Example 55 with Lexer

use of org.antlr.v4.runtime.Lexer in project inmemantlr by julianthome.

the class GenericParser method parse.

/**
 * parse string and create a context
 *
 * @param toParse    string to parseFile
 * @param production production name to parseFile
 * @param cs         case sensitivity
 * @return context
 * @throws IllegalWorkflowException if compilation did not take place
 * @throws ParsingException         if an error occurs while parsing
 */
public ParserRuleContext parse(String toParse, String production, CaseSensitiveType cs) throws IllegalWorkflowException, ParsingException {
    if (!antrlObjectsAvailable()) {
        throw new IllegalWorkflowException("No antlr objects have been compiled or loaded");
    }
    switch(cs) {
        case NONE:
            break;
        case UPPER:
            toParse = toParse.toUpperCase();
            break;
        case LOWER:
            toParse = toParse.toLowerCase();
            break;
    }
    InmemantlrErrorListener el = new InmemantlrErrorListener();
    listener.reset();
    // CodePointCharStream input = CharStreams.fromString(toParse);
    CharStream input = provider.getCharStream(toParse);
    Objects.requireNonNull(input, "char stream must not be null");
    LOGGER.debug("load lexer {}", lexerName);
    Lexer lex = sc.instanciateLexer(input, lexerName, useCached);
    lex.addErrorListener(el);
    Objects.requireNonNull(lex, "lex must not be null");
    CommonTokenStream tokens = new CommonTokenStream(lex);
    tokens.fill();
    LOGGER.debug("load parser {}", parserName);
    Parser parser = sc.instanciateParser(tokens, parserName);
    Objects.requireNonNull(parser, "Parser must not be null");
    // make parser information available to listener
    listener.setParser(parser);
    parser.removeErrorListeners();
    parser.addErrorListener(el);
    parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
    parser.setBuildParseTree(true);
    parser.setTokenStream(tokens);
    String[] rules = parser.getRuleNames();
    String entryPoint;
    if (production == null) {
        entryPoint = rules[0];
    } else {
        if (!Arrays.asList(rules).contains(production)) {
            throw new IllegalArgumentException("Rule " + production + " not found");
        }
        entryPoint = production;
    }
    ParserRuleContext data = null;
    try {
        Class<?> pc = parser.getClass();
        Method m = pc.getDeclaredMethod(entryPoint, (Class<?>[]) null);
        Objects.requireNonNull(m, "method should not be null");
        data = (ParserRuleContext) m.invoke(parser, (Object[]) null);
    } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
        // e.printStackTrace();
        return null;
    }
    Set<String> msgs = el.getLog().entrySet().stream().filter(e -> e.getKey() == InmemantlrErrorListener.Type.SYNTAX_ERROR).map(e -> e.getValue()).collect(Collectors.toSet());
    if (msgs.size() > 0) {
        String result = msgs.stream().collect(Collectors.joining());
        throw new ParsingException(result);
    }
    ParseTreeWalker walker = new ParseTreeWalker();
    walker.walk(listener, data);
    return data;
}