Examples with StringType org.apache.jena.riot.tokens.StringType used on opensource projects

Example 1 with StringType

use of org.apache.jena.riot.tokens.StringType in project jena by apache.

the class TokenizerJSON method parseToken.

// ---- Machinary
// ""-string, ''-string, *X, 
// various single characters . , : ; 
// (), [], {}, <>
// Numbers (integer, decimal, double)
// Keys (restricted strings, used as keys in maps)
//  ALPHA (ALPHA,NUMERIC,_,...)
private Token parseToken() {
    token = new Token(getLine(), getColumn());
    int ch = reader.peekChar();
    // Support both "" and '' strings (only "" is legal JSON)
    if (ch == CH_QUOTE1 || ch == CH_QUOTE2) {
        token.setType(TokenType.STRING);
        reader.readChar();
        int ch2 = reader.peekChar();
        if (ch2 == ch) {
            // Maybe """-strings/'''-strings
            // Read potential second quote.
            reader.readChar();
            int ch3 = reader.peekChar();
            if (ch3 == ch) {
                // """-strings/'''-strings
                reader.readChar();
                token.setImage(readLong(ch, false));
                StringType st = (ch == CH_QUOTE1) ? StringType.LONG_STRING1 : StringType.LONG_STRING2;
                token.setStringType(st);
                return token;
            }
            // Two quotes then a non-quote.
            // Must be '' or ""
            token.setImage("");
        } else
            // Single quote character.
            token.setImage(allBetween(ch, ch, true, false));
        // Single quoted string.
        StringType st = (ch == CH_QUOTE1) ? StringType.STRING1 : StringType.STRING2;
        token.setStringType(st);
        return token;
    }
    switch(ch) {
        // DOT can't start a decimal in JSON.  Check for digit.
        case CH_DOT:
            //                reader.readChar() ;
            //                ch = reader.peekChar() ;
            //                if ( range(ch, '0', '9') )
            //                {
            //                    // Not a DOT after all.
            //                    reader.pushbackChar(CH_DOT) ;
            //                    // Drop through to number code.
            //                    break ;
            //                }
            token.setType(TokenType.DOT);
            return token;
        case CH_SEMICOLON:
            reader.readChar();
            token.setType(TokenType.SEMICOLON);
            return token;
        case CH_COMMA:
            reader.readChar();
            token.setType(TokenType.COMMA);
            return token;
        case CH_LBRACE:
            reader.readChar();
            token.setType(TokenType.LBRACE);
            return token;
        case CH_RBRACE:
            reader.readChar();
            token.setType(TokenType.RBRACE);
            return token;
        case CH_LPAREN:
            reader.readChar();
            token.setType(TokenType.LPAREN);
            return token;
        case CH_RPAREN:
            reader.readChar();
            token.setType(TokenType.RPAREN);
            return token;
        case CH_LBRACKET:
            reader.readChar();
            token.setType(TokenType.LBRACKET);
            return token;
        case CH_RBRACKET:
            reader.readChar();
            token.setType(TokenType.RBRACKET);
            return token;
        // Some interesting characters
        case CH_COLON:
            reader.readChar();
            token.setType(TokenType.COLON);
            return token;
        // token.setType(TokenType.UNDERSCORE) ; return token ;
        case CH_LT:
            reader.readChar();
            token.setType(TokenType.LT);
            return token;
        case CH_GT:
            reader.readChar();
            token.setType(TokenType.GT);
            return token;
    }
    if (ch == CH_PLUS || ch == CH_MINUS || range(ch, '0', '9')) {
        readNumber();
        return token;
    }
    // Plain words and prefixes.
    // Can't start with a number due to numeric test above.
    // Can start with a '_' (no blank node test above)
    readKeyWord(token);
    return token;
}