Examples with CommonTokenStream org.antlr.runtime.CommonTokenStream used on opensource projects

Example 46 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project kie-wb-common by kiegroup.

the class JavaParserFactory method newParser.

private static JavaParser newParser(final CharStream charStream, StringBuilder source, final JavaParserBase.ParserMode mode) {
    final JavaLexer lexer = new JavaLexer(charStream);
    final TokenStream tokenStream = new CommonTokenStream(lexer);
    final JavaParser parser = new JavaParser(tokenStream, source, mode);
    return parser;
}

Example 47 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project cuba by cuba-platform.

the class Jpa2GrammarTest method testEscape.

@Test
public void testEscape() throws Exception {
    String query = "c.name like :pattern escape '/'";
    CharStream cs = new AntlrNoCaseStringStream(query);
    JPA2Lexer lexer = new JPA2Lexer(cs);
    TokenStream tstream = new CommonTokenStream(lexer);
    JPA2Parser jpa2Parser = new JPA2Parser(tstream);
    JPA2Parser.like_expression_return aReturn = jpa2Parser.like_expression();
    Assertions.assertNotNull(aReturn);
    query = "c.name like :pattern escape '.'";
    cs = new AntlrNoCaseStringStream(query);
    lexer = new JPA2Lexer(cs);
    tstream = new CommonTokenStream(lexer);
    jpa2Parser = new JPA2Parser(tstream);
    aReturn = jpa2Parser.like_expression();
    Assertions.assertTrue(isValid((CommonTree) aReturn.getTree()));
}

Example 48 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project scheduler by btrplace.

the class ScriptBuilder method build.

/**
 * Internal method to check a script from a stream.
 *
 * @param cs the stream to analyze
 * @return the built script
 * @throws ScriptBuilderException in an error occurred while building the script
 */
// For the UnsupportedOperationException
@SuppressWarnings("squid:S1166")
private Script build(CharStream cs) throws ScriptBuilderException {
    Script v = new Script();
    ANTLRBtrplaceSL2Lexer lexer = new ANTLRBtrplaceSL2Lexer(cs);
    ErrorReporter errorReporter = errBuilder.build(v);
    lexer.setErrorReporter(errorReporter);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    ANTLRBtrplaceSL2Parser parser = new ANTLRBtrplaceSL2Parser(tokens);
    parser.setErrorReporter(errorReporter);
    SymbolsTable t = new SymbolsTable();
    parser.setTreeAdaptor(new BtrPlaceTreeAdaptor(v, model, namingServiceNodes, namingServiceVMs, tpls, errorReporter, t, includes, catalog));
    try {
        BtrPlaceTree tree = (BtrPlaceTree) parser.script_decl().getTree();
        if (tree != null) {
            if (tree.token != null) {
                // Single instruction
                tree.go(tree);
            } else {
                for (int i = 0; i < tree.getChildCount(); i++) {
                    tree.getChild(i).go(tree);
                }
            }
        }
    } catch (RecognitionException e) {
        throw new ScriptBuilderException(e.getMessage(), e);
    } catch (UnsupportedOperationException e) {
        // We only keep the error message
        errorReporter.append(0, 0, e.getMessage());
    }
    if (!errorReporter.getErrors().isEmpty()) {
        throw new ScriptBuilderException(errorReporter);
    }
    return v;
}

Example 49 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project alfresco-remote-api by Alfresco.

the class WhereCompiler method compileSelectClause.

public static CommonTree compileSelectClause(String selectParam) throws RecognitionException {
    // lexer splits input into tokens
    ANTLRStringStream input = new ANTLRStringStream(selectParam);
    TokenStream tokens = new CommonTokenStream(new WhereClauseLexer(input));
    // parser generates abstract syntax tree
    WhereClauseParser parser = new WhereClauseParser(tokens);
    WhereClauseParser.selectClause_return ret = parser.selectClause();
    // acquire parse result
    CommonTree ast = (CommonTree) ret.getTree();
    if (logger.isDebugEnabled())
        print(ast, 0);
    return ast;
}

Example 50 with CommonTokenStream

use of org.antlr.runtime.CommonTokenStream in project pentaho-kettle by pentaho.

the class Edi2Xml method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (Edi2XmlMeta) smi;
    data = (Edi2XmlData) sdi;
    // get row, blocks when needed!
    Object[] r = getRow();
    if (r == null) {
        // no more input to be expected...
        setOutputDone();
        return false;
    }
    String inputValue = "";
    if (first) {
        first = false;
        data.inputRowMeta = getInputRowMeta().clone();
        data.outputRowMeta = getInputRowMeta().clone();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
        String realInputField = environmentSubstitute(meta.getInputField());
        String realOutputField = environmentSubstitute(meta.getOutputField());
        data.inputFieldIndex = getInputRowMeta().indexOfValue(realInputField);
        int numErrors = 0;
        if (data.inputFieldIndex < 0) {
            logError(BaseMessages.getString(PKG, "Edi2Xml.Log.CouldNotFindInputField", realInputField));
            numErrors++;
        }
        if (!data.inputRowMeta.getValueMeta(data.inputFieldIndex).isString()) {
            logError(BaseMessages.getString(PKG, "Edi2Xml.Log.InputFieldIsNotAString", realInputField));
            numErrors++;
        }
        if (numErrors > 0) {
            setErrors(numErrors);
            stopAll();
            return false;
        }
        data.inputMeta = data.inputRowMeta.getValueMeta(data.inputFieldIndex);
        if (Utils.isEmpty(meta.getOutputField())) {
            // same field
            data.outputMeta = data.outputRowMeta.getValueMeta(data.inputFieldIndex);
            data.outputFieldIndex = data.inputFieldIndex;
        } else {
            // new field
            data.outputMeta = data.outputRowMeta.searchValueMeta(realOutputField);
            data.outputFieldIndex = data.outputRowMeta.size() - 1;
        }
        // create instances of lexer/tokenstream/parser
        // treat null values as empty strings for parsing purposes
        inputValue = Const.NVL(data.inputMeta.getString(r[data.inputFieldIndex]), "");
        lexer = new FastSimpleGenericEdifactDirectXMLLexer(new ANTLRStringStream(inputValue));
        tokens = new CommonTokenStream(lexer);
        parser = new FastSimpleGenericEdifactDirectXMLParser(tokens);
    } else {
        // treat null values as empty strings for parsing purposes
        inputValue = Const.NVL(data.inputMeta.getString(r[data.inputFieldIndex]), "");
        lexer.setCharStream(new ANTLRStringStream(inputValue));
        tokens.setTokenSource(lexer);
        parser.setTokenStream(tokens);
    }
    try {
        parser.edifact();
        // make sure the row is big enough
        r = RowDataUtil.resizeArray(r, data.outputRowMeta.size());
        // place parsing result into output field
        r[data.outputFieldIndex] = parser.buf.toString();
        putRow(data.outputRowMeta, r);
    } catch (MismatchedTokenException e) {
        StringBuilder errorMessage = new StringBuilder(180);
        errorMessage.append("error parsing edi on line " + e.line + " position " + e.charPositionInLine);
        errorMessage.append(": expecting " + ((e.expecting > -1) ? parser.getTokenNames()[e.expecting] : "<UNKNOWN>") + " but found ");
        errorMessage.append((e.token.getType() >= 0) ? parser.getTokenNames()[e.token.getType()] : "<EOF>");
        if (getStepMeta().isDoingErrorHandling()) {
            putError(getInputRowMeta(), r, 1L, errorMessage.toString(), environmentSubstitute(meta.getInputField()), "MALFORMED_EDI");
        } else {
            logError(errorMessage.toString());
            // try to determine the error line
            String errorline = "<UNKNOWN>";
            try {
                errorline = inputValue.split("\\r?\\n")[e.line - 1];
            } catch (Exception ee) {
            // Ignore pattern syntax errors
            }
            logError("Problem line: " + errorline);
            logError(StringUtils.leftPad("^", e.charPositionInLine + "Problem line: ".length() + 1));
            throw new KettleException(e);
        }
    } catch (RecognitionException e) {
        StringBuilder errorMessage = new StringBuilder(180);
        errorMessage.append("error parsing edi on line ").append(e.line).append(" position ").append(e.charPositionInLine).append(". ").append(e.toString());
        if (getStepMeta().isDoingErrorHandling()) {
            putError(getInputRowMeta(), r, 1L, errorMessage.toString(), environmentSubstitute(meta.getInputField()), "MALFORMED_EDI");
        } else {
            logError(errorMessage.toString());
            // try to determine the error line
            String errorline = "<UNKNOWN>";
            try {
                errorline = inputValue.split("\\r?\\n")[e.line - 1];
            } catch (Exception ee) {
            // Ignore pattern syntax errors
            }
            logError("Problem line: " + errorline);
            logError(StringUtils.leftPad("^", e.charPositionInLine + "Problem line: ".length() + 1));
            throw new KettleException(e);
        }
    }
    if (checkFeedback(getLinesRead())) {
        logBasic("Linenr " + getLinesRead());
    }
    return true;
}