Examples with Token org.anarres.cpp.Token used on opensource projects

Example 1 with Token

use of org.anarres.cpp.Token in project Cpp4CIA by thanhminhmr.

the class PreprocessorBuilder method readablePreprocessor.

private static void readablePreprocessor(@Nonnull Preprocessor preprocessor, @Nonnull StringBuilder fileContent) throws IOException, LexerException {
    int emptyLine = 1;
    final StringBuilder emptyLineBuilder = new StringBuilder();
    while (true) {
        final Token tok = preprocessor.token();
        if (tok.getType() == Token.EOF)
            break;
        if (tok.getType() != Token.C_COMMENT && tok.getType() != Token.CPP_COMMENT) {
            final String tokText = tok.getText().replace("\r\n", "\n").replace('\r', '\n');
            if (tok.getType() != Token.WHITESPACE && !tokText.isBlank()) {
                if (tok.getType() != Token.P_LINE && emptyLine > 0) {
                    fileContent.append(emptyLineBuilder);
                }
                fileContent.append(tokText);
                emptyLineBuilder.setLength(0);
                emptyLine = 0;
            } else {
                if (!tokText.contains("\n")) {
                    if (emptyLine == 0) {
                        fileContent.append(' ');
                    } else {
                        emptyLineBuilder.append(tokText);
                    }
                } else if (emptyLine < 2) {
                    fileContent.append('\n');
                    emptyLineBuilder.setLength(0);
                    emptyLine += 1;
                } else {
                    emptyLineBuilder.setLength(0);
                }
            }
        }
    }
    fileContent.append('\n');
}

Example 2 with Token

use of org.anarres.cpp.Token in project Iris by IrisShaders.

the class JcppProcessor method glslPreprocessSource.

// Derived from GlShader from Canvas, licenced under LGPL
public static String glslPreprocessSource(String source) {
    if (source.contains(GlslCollectingListener.VERSION_MARKER) || source.contains(GlslCollectingListener.EXTENSION_MARKER)) {
        throw new RuntimeException("Some shader author is trying to exploit internal Iris implementation details, stop!");
    }
    // Note: This is an absolutely awful hack. But JCPP's lack of extensibility leaves me with no choice...
    // We should write our own preprocessor at some point to avoid this.
    // 
    // Why are we doing this awful hack instead of just using the preprocessor like a normal person? Because it lets
    // us only hoist #extension directives if they're actually used. This is needed for shader packs written on
    // lenient drivers that allow #extension directives to be placed anywhere to work on strict drivers like Mesa
    // that require #extension directives to occur at the top.
    // 
    // TODO: This allows #version to not appear as the first non-comment non-whitespace thing in the file.
    // That's not the behavior we want. If you're reading this, don't rely on this behavior.
    source = source.replace("#version", GlslCollectingListener.VERSION_MARKER);
    source = source.replace("#extension", GlslCollectingListener.EXTENSION_MARKER);
    GlslCollectingListener listener = new GlslCollectingListener();
    @SuppressWarnings("resource") final Preprocessor pp = new Preprocessor();
    pp.setListener(listener);
    pp.addInput(new StringLexerSource(source, true));
    pp.addFeature(Feature.KEEPCOMMENTS);
    final StringBuilder builder = new StringBuilder();
    try {
        for (; ; ) {
            final Token tok = pp.token();
            if (tok == null)
                break;
            if (tok.getType() == Token.EOF)
                break;
            builder.append(tok.getText());
        }
    } catch (final Exception e) {
        throw new RuntimeException("GLSL source pre-processing failed", e);
    }
    builder.append("\n");
    source = listener.collectLines() + builder;
    return source;
}

Example 3 with Token

use of org.anarres.cpp.Token in project Iris by IrisShaders.

the class JcppProcessor method glslPreprocessSource.

// Derived from GlShader from Canvas, licenced under LGPL
public static String glslPreprocessSource(String source, Iterable<StringPair> environmentDefines) {
    if (source.contains(GlslCollectingListener.VERSION_MARKER) || source.contains(GlslCollectingListener.EXTENSION_MARKER)) {
        throw new RuntimeException("Some shader author is trying to exploit internal Iris implementation details, stop!");
    }
    // Note: This is an absolutely awful hack. But JCPP's lack of extensibility leaves me with no choice...
    // We should write our own preprocessor at some point to avoid this.
    // 
    // Why are we doing this awful hack instead of just using the preprocessor like a normal person? Because it lets
    // us only hoist #extension directives if they're actually used. This is needed for shader packs written on
    // lenient drivers that allow #extension directives to be placed anywhere to work on strict drivers like Mesa
    // that require #extension directives to occur at the top.
    // 
    // TODO: This allows #version to not appear as the first non-comment non-whitespace thing in the file.
    // That's not the behavior we want. If you're reading this, don't rely on this behavior.
    source = source.replace("#version", GlslCollectingListener.VERSION_MARKER);
    source = source.replace("#extension", GlslCollectingListener.EXTENSION_MARKER);
    GlslCollectingListener listener = new GlslCollectingListener();
    @SuppressWarnings("resource") final Preprocessor pp = new Preprocessor();
    // in errors...
    try {
        for (StringPair envDefine : environmentDefines) {
            pp.addMacro(envDefine.getKey(), envDefine.getValue());
        }
    } catch (LexerException e) {
        throw new RuntimeException("Unexpected LexerException processing macros", e);
    }
    pp.setListener(listener);
    pp.addInput(new StringLexerSource(source, true));
    pp.addFeature(Feature.KEEPCOMMENTS);
    final StringBuilder builder = new StringBuilder();
    try {
        for (; ; ) {
            final Token tok = pp.token();
            if (tok == null)
                break;
            if (tok.getType() == Token.EOF)
                break;
            builder.append(tok.getText());
        }
    } catch (final Exception e) {
        throw new RuntimeException("GLSL source pre-processing failed", e);
    }
    builder.append("\n");
    source = listener.collectLines() + builder;
    return source;
}

Example 4 with Token

use of org.anarres.cpp.Token in project Iris by IrisShaders.

the class PropertiesPreprocessor method process.

private static String process(Preprocessor preprocessor, String source) {
    preprocessor.setListener(new PropertiesCommentListener());
    PropertyCollectingListener listener = new PropertyCollectingListener();
    preprocessor.setListener(listener);
    // Not super efficient, but this removes trailing whitespace on lines, fixing an issue with whitespace after
    // line continuations (see PreprocessorTest#testWeirdPropertiesLineContinuation)
    // Required for Voyager Shader
    source = Arrays.stream(source.split("\\R")).map(String::trim).map(line -> {
        if (line.startsWith("#")) {
            // marker.
            return line;
        } else {
            // In properties files, we don't substitute #define values except on macro lines.
            return "#warning IRIS_PASSTHROUGH " + line;
        }
    }).collect(Collectors.joining("\n")) + "\n";
    preprocessor.addInput(new StringLexerSource(source, true));
    preprocessor.addFeature(Feature.KEEPCOMMENTS);
    final StringBuilder builder = new StringBuilder();
    try {
        for (; ; ) {
            final Token tok = preprocessor.token();
            if (tok == null)
                break;
            if (tok.getType() == Token.EOF)
                break;
            builder.append(tok.getText());
        }
    } catch (final Exception e) {
        Iris.logger.error("Properties pre-processing failed", e);
    }
    source = builder.toString();
    return listener.collectLines() + source;
}

Example 5 with Token

use of org.anarres.cpp.Token in project Cpp4CIA by thanhminhmr.

the class PreprocessorBuilder method fastPreprocessor.

private static void fastPreprocessor(@Nonnull Preprocessor preprocessor, @Nonnull StringBuilder fileContent) throws IOException, LexerException {
    boolean haveEndSpace = true;
    while (true) {
        final Token token = preprocessor.token();
        switch(token.getType()) {
            case Token.NEW_LINE:
            case Token.WHITESPACE:
            case Token.C_COMMENT:
            case Token.CPP_COMMENT:
            case Token.P_LINE:
                haveEndSpace = true;
                continue;
            case Token.EOF:
                fileContent.append('\n');
                return;
        }
        final String tokenText = token.getText().trim();
        if (tokenText.isBlank()) {
            haveEndSpace = true;
            continue;
        }
        if (haveEndSpace)
            fileContent.append(' ');
        fileContent.append(tokenText);
        haveEndSpace = false;
    }
}