Examples with Token net.morimekta.providence.serializer.pretty.Token used on opensource projects

Example 21 with Token

use of net.morimekta.providence.serializer.pretty.Token in project providence by morimekta.

the class ThriftTokenizerTest method testJavaComments.

@Test
public void testJavaComments() throws IOException {
    try {
        tokenizer("/").next();
        fail("no exception");
    } catch (ParseException e) {
        assertThat(e.asString(), is("Error on line 1, pos 2: Expected java-style comment, got end of file\n" + "/\n" + "-^"));
    }
    try {
        tokenizer("/b").next();
        fail("no exception");
    } catch (ParseException e) {
        assertThat(e.asString(), is("Error on line 1, pos 1: Expected java-style comment, got 'b' after '/'\n" + "/b\n" + "^^"));
    }
    Token token = tokenizer("\n\n// b\n").next();
    assertThat(token, is(notNullValue()));
    assertThat(token.asString(), is("//"));
}

Example 22 with Token

use of net.morimekta.providence.serializer.pretty.Token in project providence by morimekta.

the class ThriftTokenizer method parseValue.

public Token parseValue() throws IOException {
    Stack<Character> enclosures = new Stack<>();
    int startLineNo = 0;
    int startLinePos = 0;
    int offset = -1;
    while (true) {
        Token token = expect("const value");
        if (offset < 0) {
            offset = token.getOffset();
            startLineNo = token.getLineNo();
            startLinePos = token.getLinePos();
        }
        if (token.strEquals(kBlockCommentStart)) {
            // ignore.
            parseDocBlock();
            continue;
        } else if (token.strEquals(kLineCommentStart)) {
            IOUtils.readString(this, Token.kNewLine);
            continue;
        } else if (token.isSymbol(Token.kMessageStart)) {
            enclosures.push(Token.kMessageEnd);
        } else if (token.isSymbol(Token.kListStart)) {
            enclosures.push(Token.kListEnd);
        } else if ((token.isSymbol(Token.kMessageEnd) || token.isSymbol(Token.kListEnd)) && enclosures.peek().equals(token.charAt(0))) {
            enclosures.pop();
        }
        if (enclosures.isEmpty()) {
            return token(offset, (token.getOffset() - offset) + token.length(), startLineNo, startLinePos);
        }
    }
}