Examples with RedmatchLexer au.csiro.redmatch.grammar.RedmatchLexer used on opensource projects

Example 1 with RedmatchLexer

use of au.csiro.redmatch.grammar.RedmatchLexer in project redmatch by aehrc.

the class RedmatchCompiler method compile.

/**
 * Compiles a Redmatch document.
 *
 * @param baseFolder The base folder where the input files are contained.
 * @param document The Redmatch document.
 *
 * @return A Document object or null if there is an unrecoverable compilation problem.
 * @throws CompilationException If something goes wrong with the compilation.
 */
public Document compile(File baseFolder, String document) throws CompilationException {
    clear();
    this.baseFolder = baseFolder;
    if (baseFolder != null) {
        log.info("Running compilation from base folder: " + baseFolder.getAbsolutePath());
    } else {
        log.info("Running compilation with no base folder");
    }
    if (document == null || document.isEmpty()) {
        return new Document();
    }
    final Lexer lexer = new RedmatchLexer(CharStreams.fromString(document));
    final RedmatchGrammar parser = new RedmatchGrammar(new CommonTokenStream(lexer));
    lexer.removeErrorListeners();
    lexer.addErrorListener(new DiagnosticErrorListener() {

        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
            addError(offendingSymbol != null ? offendingSymbol.toString() : "", line, charPositionInLine, msg, SRC_LEXER, CODE_LEXER.toString());
        }
    });
    parser.removeErrorListeners();
    parser.addErrorListener(new DiagnosticErrorListener() {

        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
            addError(offendingSymbol != null ? offendingSymbol.toString() : "", line, charPositionInLine, msg, SRC_PARSER, CODE_PARSER.toString());
        }
    });
    if (Thread.interrupted()) {
        throw new CompilationException("The compilation was interrupted");
    }
    final DocumentContext docCtx = parser.document();
    // We need to check if the EOF token was matched. If not, then there is a problem.
    final Token finalToken = lexer.getToken();
    if (finalToken.getType() != Token.EOF) {
        addError(finalToken.getText(), finalToken.getLine(), finalToken.getCharPositionInLine(), "Unexpected token '" + finalToken.getText() + "'.", SRC_COMPILER, CODE_PARSER.toString());
    }
    String tree = docCtx.toStringTree(parser);
    if (log.isDebugEnabled()) {
        printPrettyLispTree(tree);
    }
    try {
        if (Thread.interrupted()) {
            throw new CompilationException("The compilation was interrupted");
        }
        Document doc = (Document) docCtx.accept(this);
        doc.setDiagnostics(new ArrayList<>(this.diagnostics));
        // Validate the resulting FHIR graph if there are no errors
        if (this.diagnostics.stream().noneMatch(d -> d.getSeverity().equals(DiagnosticSeverity.Error))) {
            if (Thread.interrupted()) {
                throw new CompilationException("The compilation was interrupted");
            }
            GraphUtils.Results res = GraphUtils.buildGraph(doc);
            doc.getDiagnostics().addAll(res.getDiagnostics());
        }
        return doc;
    } catch (Throwable t) {
        // Special case: Interrupted exception wrapped in a runtime exception
        if (t.getCause() != null && t.getCause() instanceof InterruptedException) {
            throw new CompilationException("The compilation was interrupted", t.getCause());
        }
        throw t;
    }
}

Example 2 with RedmatchLexer

use of au.csiro.redmatch.grammar.RedmatchLexer in project redmatch by aehrc.

the class SemanticTokeniserTest method printTokens.

protected void printTokens(String rule) {
    System.out.println("TOKENS:");
    final Lexer lexer = new RedmatchLexer(CharStreams.fromString(rule));
    for (Token tok : lexer.getAllTokens()) {
        System.out.println(tok);
    }
}

Example 3 with RedmatchLexer

use of au.csiro.redmatch.grammar.RedmatchLexer in project redmatch by aehrc.

the class SemanticTokeniser method tokenise.

public static SemanticTokens tokenise(String text) {
    // First we need to get the tokens from the lexer
    List<SemanticToken> tokens = new ArrayList<>();
    final Lexer lexer = new RedmatchLexer(CharStreams.fromString(text));
    for (Token tok : lexer.getAllTokens()) {
        // account for 0-index differences
        int line = tok.getLine() - 1;
        int startChar = tok.getCharPositionInLine();
        int length = tok.getText().length();
        int tokenType = mapTokenType(tok.getType());
        if (tokenType != -1) {
            tokens.add(new SemanticToken(line, startChar, length, tokenType));
        }
    }
    // { deltaLine: 2, deltaStartChar: 5, length: 3, tokenType: 0, tokenModifiers: 3 },
    if (!tokens.isEmpty()) {
        tokens.get(0).deltaLine = tokens.get(0).line;
        tokens.get(0).deltaStartChar = tokens.get(0).startChar;
    }
    for (int i = 1; i < tokens.size(); i++) {
        SemanticToken prevToken = tokens.get(i - 1);
        SemanticToken token = tokens.get(i);
        token.deltaLine = token.line - prevToken.line;
        // Now we need to "deltify" the current token
        if (prevToken.line == token.line) {
            // They are on the same line, so we need to make the start token relative
            token.deltaStartChar = token.startChar - prevToken.startChar;
        } else {
            // There was a change of line so the first token delta start char is the same as the start char
            token.deltaStartChar = token.startChar;
        }
    }
    log.info("Encoding " + tokens.size() + " semantic tokens");
    // Finally, we transform them into the format expected by the language server protocol
    List<Integer> encodedTokens = new ArrayList<>();
    for (SemanticToken token : tokens) {
        encodedTokens.add(token.deltaLine);
        encodedTokens.add(token.deltaStartChar);
        encodedTokens.add(token.length);
        encodedTokens.add(token.tokenType);
        // No modifiers for Redmatch
        encodedTokens.add(0);
    }
    return new SemanticTokens(encodedTokens);
}

Example 4 with RedmatchLexer

use of au.csiro.redmatch.grammar.RedmatchLexer in project redmatch by aehrc.

the class CompletionProcessor method getCompletions.

/**
 * Returns a list of possible auto-completions.
 *
 * @param url The document url.
 * @param document The text of the document.
 * @param position The position in the text where the autocompletion was triggered.
 * @return A list of possible auto-completions.
 */
public synchronized List<CompletionItem> getCompletions(String url, String document, Position position) {
    String snippet = document.substring(0, DocumentUtils.getPosition(position, document));
    if (snippet.length() >= 6) {
        log.debug("Processing snippet ending in " + snippet.substring(snippet.length() - 6));
    } else {
        log.debug("Processing snippet " + snippet);
    }
    final Lexer lexer = new RedmatchLexer(CharStreams.fromString(snippet));
    List<? extends Token> tokens = lexer.getAllTokens();
    List<CompletionItem> completionItems = handleOpen(url, tokens);
    if (!completionItems.isEmpty()) {
        return completionItems;
    }
    completionItems = handleResource(url, tokens);
    if (!completionItems.isEmpty()) {
        return completionItems;
    }
    completionItems = handleAttribute(url, tokens);
    return completionItems;
}

Example 5 with RedmatchLexer

use of au.csiro.redmatch.grammar.RedmatchLexer in project redmatch by aehrc.

the class RedmatchCompilerTest method printTokens.

protected void printTokens(String rule) {
    System.out.println("TOKENS:");
    final Lexer lexer = new RedmatchLexer(CharStreams.fromString(rule));
    for (Token tok : lexer.getAllTokens()) {
        System.out.println(tok);
    }
}