Examples with Token org.apache.jena.riot.tokens.Token used on opensource projects

Example 6 with Token

use of org.apache.jena.riot.tokens.Token in project jena by apache.

the class RiotLib method parse.

/** Parse a string to get one Node (the first token in the string) */
public static Node parse(String string) {
    Tokenizer tokenizer = TokenizerFactory.makeTokenizerString(string);
    if (!tokenizer.hasNext())
        return null;
    Token t = tokenizer.next();
    Node n = profile.create(null, t);
    if (tokenizer.hasNext())
        Log.warn(RiotLib.class, "String has more than one token in it: " + string);
    return n;
}

Example 7 with Token

use of org.apache.jena.riot.tokens.Token in project jena by apache.

the class JSONInputIterator method parseVars.

private void parseVars() {
    if (lookingAt(TokenType.LBRACKET)) {
        nextToken();
        vars.clear();
        do {
            if (lookingAt(TokenType.STRING)) {
                Token t = nextToken();
                String var = t.getImage();
                vars.add(var);
                checkComma(TokenType.RBRACKET);
            } else if (lookingAt(TokenType.RBRACKET)) {
                nextToken();
                return;
            } else {
                exception(peekToken(), "Unexpected Token encountered while parsing the variables list in the head object");
            }
        } while (true);
    } else {
        exception(peekToken(), "Unexpected Token ecountered, expected a [ to start the array of variables in the head object");
    }
}

Example 8 with Token

use of org.apache.jena.riot.tokens.Token in project jena by apache.

the class CmdTokens method tokens.

public static void tokens(final boolean print, final boolean timing, String... args) {
    if (args.length == 0)
        args = new String[] { "-" };
    String arg = args[0];
    if (arg.equals("--help") || arg.equals("-help") || arg.equals("-h") || arg.equals("--h")) {
        System.err.println("Usage: stdin | FILE ...");
        System.exit(1);
    }
    for (String filename : args) {
        InputStream in = IO.openFile(filename);
        Tokenizer tokenize = TokenizerFactory.makeTokenizerUTF8(in);
        Timer timer = new Timer();
        long count = 0;
        timer.startTimer();
        for (; tokenize.hasNext(); ) {
            Token t = tokenize.next();
            if (print)
                System.out.println(t);
            count++;
        }
        tokenize.close();
        long millis = timer.endTimer();
        if (timing) {
            if (millis == 0)
                System.out.printf("Tokens=%,d : Time=0.00s\n", count);
            else {
                double seconds = millis / 1000.0;
                System.out.printf("Tokens=%,d : Time=%,.2fs : Rate=%,.2f\n", count, seconds, count / seconds);
            }
        }
    }
}

Example 9 with Token

use of org.apache.jena.riot.tokens.Token in project jena by apache.

the class LangRDFJSON method tryParsePredicateObjectList.

private void tryParsePredicateObjectList(Node subj) {
    //First must see the { to start the object which we can then discard
    if (!lookingAt(TokenType.LBRACE)) {
        exception(peekToken(), "Expected a { character to start the JSON Object for a Predicate Object List but got a %s", peekToken());
    }
    nextToken();
    //Then we must see a Property Name or the end of the predicate object list
    boolean first = true;
    boolean propertyNameExpected = true;
    while (true) {
        if (isPropertyName()) {
            first = false;
            propertyNameExpected = false;
            Token t = nextToken();
            Node pred = profile.createURI(t.getImage(), t.getLine(), t.getColumn());
            //Must be a : after Property Name
            checkColon();
            //Then we can try and parse the Object List
            tryParseObjectList(subj, pred);
            //i.e. further predicates for this subject
            if (lookingAt(TokenType.COMMA)) {
                nextToken();
                propertyNameExpected = true;
            }
        } else if (!first && lookingAt(TokenType.RBRACE)) {
            //This is the end of the Predicate Object List Object
            if (propertyNameExpected) {
                exception(peekToken(), "Expected a further Property Name to represent a Predicate after a comma in a Predicate Object List but got %s", peekToken());
            }
            nextToken();
            return;
        } else {
            if (propertyNameExpected) {
                exception(peekToken(), "Expected a Property Name to represent a Predicate as part of a Predicate Object List but got %s", peekToken());
            } else {
                exception(peekToken(), "Expected a Property Name or the end of the Predicate Object List but got %s", peekToken());
            }
        }
    }
}

Example 10 with Token

use of org.apache.jena.riot.tokens.Token in project jena by apache.

the class LangRDFJSON method tryParseObject.

private Node tryParseObject() {
    //RDF/JSON defines the Object of a Triple to be encoded as a
    //JSON Object
    //It has mandatory properties 'value' and 'type' plus optional
    //properties 'lang', 'xml:lang' and 'datatype'
    Node obj = null;
    Token value = null, type = null, lang = null, datatype = null;
    //First we expect to see the { character to start the JSON Object
    if (lookingAt(TokenType.LBRACE)) {
        //Discard the {
        nextToken();
        //Then see a stream of tokens which are property value pairs
        //representing the properties of the object
        boolean first = true;
        boolean propertyNameExpected = true;
        while (true) {
            if (isPropertyName()) {
                first = false;
                propertyNameExpected = false;
                Token t = nextToken();
                String name = t.getImage();
                //Must always be a : after a property name
                checkColon();
                //Is this one of our valid properties
                switch(name) {
                    case "value":
                        if (value == null) {
                            value = checkValidForObjectProperty();
                        } else {
                            exception(t, "Encountered the value property on an Object when the value property has already been specified");
                        }
                        break;
                    case "type":
                        if (type == null) {
                            type = checkValidForObjectProperty();
                        } else {
                            exception(t, "Encountered the type property on an Object when the type property has already been specified");
                        }
                        break;
                    case "lang":
                    case "xml:lang":
                        if (lang == null && datatype == null) {
                            lang = checkValidForObjectProperty();
                        } else {
                            exception(t, "Encountered the %s property on an Object when lang/datatype has already been specified", name);
                        }
                        break;
                    case "datatype":
                        if (lang == null && datatype == null) {
                            datatype = checkValidForObjectProperty();
                        } else {
                            exception(t, "Encountered the %s property on an Object when lang/datatype has already been specified", name);
                        }
                        break;
                    default:
                        exception(t, "Unexpected Property Name %s encountered, expected one of value, type, lang or datatype", t.getImage());
                        break;
                }
                //see a comma to indicate further pairs are present
                if (lookingAt(TokenType.COMMA)) {
                    nextToken();
                    propertyNameExpected = true;
                }
            } else if (!first && lookingAt(TokenType.RBRACE)) {
                if (propertyNameExpected) {
                    exception(peekToken(), "Expected a further Property Name to represent a property of the Object of a Triple after a comma but got %s", peekToken());
                }
                break;
            } else {
                exception(peekToken(), "Expected a Property Name to define a property relating to the Object of a Triple but got %s", peekToken());
            }
        }
        //Discard the } which terminated the Object
        nextToken();
        //Next up validate the tokens we've got
        if (type == null)
            exception(peekToken(), "Unable to parse the Object for a Triple from a JSON Object as the required 'type' property is not present");
        if (value == null)
            exception(peekToken(), "Unable to parse the Object for a Triple from a JSON Object as the required 'value' property is not present");
        //Use these to create the Object
        String typeStr = type.getImage();
        switch(typeStr) {
            case "uri":
                obj = profile.createURI(value.getImage(), value.getLine(), value.getColumn());
                break;
            case "bnode":
                obj = profile.createBlankNode(null, value.getImage().substring(2), value.getLine(), value.getColumn());
                break;
            case "literal":
                if (lang != null) {
                    obj = profile.createLangLiteral(value.getImage(), lang.getImage(), value.getLine(), value.getColumn());
                } else if (datatype != null) {
                    obj = profile.createTypedLiteral(value.getImage(), TypeMapper.getInstance().getSafeTypeByName(datatype.getImage()), value.getLine(), value.getColumn());
                } else {
                    obj = profile.createStringLiteral(value.getImage(), value.getLine(), value.getColumn());
                }
                break;
            default:
                exception(type, "Unable to parse the Object for a Triple from a JSON Object as the value %s given for the 'type' property is not one of uri, bnode or literal", typeStr);
                break;
        }
    } else {
        exception(peekToken(), "Expected a { character to start a JSON Object to represent the Object of a Triple but got %s", peekToken());
    }
    return obj;
}