Examples with Tokenizer org.apache.jena.riot.tokens.Tokenizer used on opensource projects

Example 11 with Tokenizer

use of org.apache.jena.riot.tokens.Tokenizer in project jena by apache.

the class SerializationFactoryFinder method tripleSerializationFactory.

public static SerializationFactory<Triple> tripleSerializationFactory() {
    return new SerializationFactory<Triple>() {

        @Override
        public Sink<Triple> createSerializer(OutputStream out) {
            return new SinkTripleOutput(out, null, NodeToLabel.createBNodeByLabelEncoded());
        }

        @Override
        public Iterator<Triple> createDeserializer(InputStream in) {
            Tokenizer tokenizer = TokenizerFactory.makeTokenizerASCII(in);
            ParserProfile profile = RiotLib.createParserProfile(RiotLib.factoryRDF(LabelToNode.createUseLabelEncoded()), ErrorHandlerFactory.errorHandlerNoWarnings, IRIResolver.createNoResolve(), false);
            LangNTriples parser = new LangNTriples(tokenizer, profile, null);
            return parser;
        }

        @Override
        public long getEstimatedMemorySize(Triple item) {
            // TODO
            return 0;
        }
    };
}

Example 12 with Tokenizer

use of org.apache.jena.riot.tokens.Tokenizer in project jena by apache.

the class TestLangRdfJson method rdfjson_invalid_tokenizer.

@Test(expected = IllegalArgumentException.class)
public void rdfjson_invalid_tokenizer() {
    byte[] b = StrUtils.asUTF8bytes("");
    ByteArrayInputStream in = new ByteArrayInputStream(b);
    Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in);
    StreamRDFCounting sink = StreamRDFLib.count();
    LangRDFJSON parser = RiotParsers.createParserRdfJson(tokenizer, sink, RiotLib.dftProfile());
}

Example 13 with Tokenizer

use of org.apache.jena.riot.tokens.Tokenizer in project jena by apache.

the class TestParserFactory method ntriples_01.

@Test
public void ntriples_01() {
    {
        String s = "<x> <p> <q> .";
        CatchParserOutput sink = parseCapture(s, Lang.NT);
        assertEquals(1, sink.startCalled);
        assertEquals(1, sink.finishCalled);
        assertEquals(1, sink.triples.size());
        assertEquals(0, sink.quads.size());
        Triple t = SSE.parseTriple("(<x> <p> <q>)");
        assertEquals(t, last(sink.triples));
    }
    // Old style, direct to LangRIOT -- very deprecated.
    // NQ version tests that relative URIs remain relative. 
    Tokenizer tokenizer = TokenizerFactory.makeTokenizerString("<x> <p> <q> .");
    CatchParserOutput sink = new CatchParserOutput();
    ParserProfile profile = makeParserProfile(IRIResolver.createNoResolve(), null, false);
    LangRIOT parser = RiotParsers.createParserNTriples(tokenizer, sink, profile);
    parser.parse();
    assertEquals(1, sink.startCalled);
    assertEquals(1, sink.finishCalled);
    assertEquals(1, sink.triples.size());
    assertEquals(0, sink.quads.size());
    assertEquals(SSE.parseTriple("(<x> <p> <q>)"), last(sink.triples));
}

Example 14 with Tokenizer

use of org.apache.jena.riot.tokens.Tokenizer in project jena by apache.

the class NodeFactoryExtra method parseNode.

/**
     * Parse a string into a node.
     * <p>
     * Allows surrounding white space.
     * </p>
     * 
     * @param nodeString Node string to parse
     * @param pmap Prefix Map, null to use no prefix mappings
     * @return Parsed Node
     * @throws RiotException Thrown if a valid node cannot be parsed
     */
public static Node parseNode(String nodeString, PrefixMap pmap) {
    Tokenizer tokenizer = TokenizerFactory.makeTokenizerString(nodeString);
    if (!tokenizer.hasNext())
        throw new RiotException("Empty RDF term");
    Token token = tokenizer.next();
    Node node = token.asNode(pmap);
    if (node == null)
        throw new RiotException("Bad RDF Term: " + nodeString);
    if (tokenizer.hasNext())
        throw new RiotException("Trailing characters in string: " + nodeString);
    if (node.isURI()) {
        // Lightly test for bad URIs.
        String x = node.getURI();
        if (x.indexOf(' ') >= 0)
            throw new RiotException("Space(s) in  IRI: " + nodeString);
    }
    return node;
}

Example 15 with Tokenizer

use of org.apache.jena.riot.tokens.Tokenizer in project jena by apache.

the class TestLangNTuples method tokenizer.

protected static Tokenizer tokenizer(String string) {
    // UTF-8
    byte[] b = StrUtils.asUTF8bytes(string);
    ByteArrayInputStream in = new ByteArrayInputStream(b);
    Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in);
    return tokenizer;
}