Examples with RawDataSchema com.ociweb.pronghorn.pipe.RawDataSchema used on opensource projects

Example 1 with RawDataSchema

use of com.ociweb.pronghorn.pipe.RawDataSchema in project PronghornPipes by oci-pronghorn.

the class JSONParseTest method parseJSONLoad.

private void parseJSONLoad(int i, String sourceData, JSONExtractorCompleted extractor) {
    PipeConfig<RawDataSchema> targetDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> targetData = new Pipe<RawDataSchema>(targetDataConfig);
    targetData.initBuffers();
    TrieParserReader reader = new TrieParserReader(5, true);
    JSONStreamParser parser = new JSONStreamParser();
    PipeConfig<RawDataSchema> testInputDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> testInputData = new Pipe<RawDataSchema>(testInputDataConfig);
    testInputData.initBuffers();
    // ///////////////
    // ///////////////
    // ///////////////
    JSONStreamVisitorToChannel visitor = extractor.newJSONVisitor();
    byte[] sourceBytes = sourceData.getBytes();
    while (--i >= 0) {
        // ///////////
        // write JSON data
        // ///////////
        assertTrue("content size " + Pipe.contentRemaining(testInputData), Pipe.contentRemaining(testInputData) == 0);
        int size = Pipe.addMsgIdx(testInputData, 0);
        Pipe.addByteArray(sourceBytes, testInputData);
        Pipe.confirmLowLevelWrite(testInputData, size);
        Pipe.publishWrites(testInputData);
        // //
        // ////////
        // start consuming the data from the pipe
        // call the parser
        // /////
        int msgIdx = Pipe.takeMsgIdx(testInputData);
        TrieParserReader.parseSetup(reader, testInputData);
        // parse data data
        parser.parse(reader, extractor.trieParser(), visitor);
        Pipe.confirmLowLevelRead(testInputData, Pipe.sizeOf(testInputData, msgIdx));
        Pipe.releaseReadLock(testInputData);
        // ///write the captured data into the pipe
        int writeSize = Pipe.addMsgIdx(targetData, 0);
        DataOutputBlobWriter<RawDataSchema> stream = Pipe.openOutputStream(targetData);
        visitor.export(stream, null);
        stream.closeLowLevelField();
        Pipe.confirmLowLevelWrite(targetData, writeSize);
        Pipe.publishWrites(targetData);
        // ///////////////////
        // read the parsed data
        // //////////////////
        RawDataSchema.consume(targetData);
    }
}

Example 2 with RawDataSchema

use of com.ociweb.pronghorn.pipe.RawDataSchema in project PronghornPipes by oci-pronghorn.

the class JSONParseTest method simpleParseTest.

@Test
public void simpleParseTest() {
    Pipe<RawDataSchema> targetData = parseJSON(simpleExample, simpleExtractor);
    // confirm data on the pipe is good...
    Pipe.takeMsgIdx(targetData);
    ChannelReader dataStream = (ChannelReader) Pipe.openInputStream(targetData);
    long header = dataStream.readPackedLong();
    assertEquals(0, header);
    String valueB = dataStream.readUTFOfLength(dataStream.readPackedInt());
    assertEquals("hello", valueB);
    long valueA = dataStream.readPackedLong();
    assertEquals(123, valueA);
}

Example 3 with RawDataSchema

use of com.ociweb.pronghorn.pipe.RawDataSchema in project PronghornPipes by oci-pronghorn.

the class JSONParseTest method parseJSON.

private Pipe<RawDataSchema> parseJSON(String sourceData, JSONExtractorCompleted extractor) {
    // ///////////////
    // source test data.
    PipeConfig<RawDataSchema> testInputDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> testInputData = new Pipe<RawDataSchema>(testInputDataConfig);
    testInputData.initBuffers();
    int size = Pipe.addMsgIdx(testInputData, 0);
    Pipe.addUTF8(sourceData, testInputData);
    Pipe.confirmLowLevelWrite(testInputData, size);
    Pipe.publishWrites(testInputData);
    // //
    TrieParserReader reader = new TrieParserReader(5, true);
    // start consuming the data from the pipe
    int msgIdx = Pipe.takeMsgIdx(testInputData);
    TrieParserReader.parseSetup(reader, testInputData);
    // export data to this pipe
    PipeConfig<RawDataSchema> targetDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> targetData = new Pipe<RawDataSchema>(targetDataConfig);
    targetData.initBuffers();
    Pipe.confirmLowLevelRead(testInputData, Pipe.sizeOf(testInputData, msgIdx));
    Pipe.releaseReadLock(testInputData);
    // parse data data
    JSONStreamParser parser = new JSONStreamParser();
    JSONStreamVisitorToChannel visitor = extractor.newJSONVisitor();
    do {
        parser.parse(reader, extractor.trieParser(), visitor);
        // ///write the captured data into the pipe
        Pipe.presumeRoomForWrite(targetData);
        int writeSize = Pipe.addMsgIdx(targetData, 0);
        DataOutputBlobWriter<RawDataSchema> stream = Pipe.openOutputStream(targetData);
        visitor.export(stream, null);
        stream.closeLowLevelField();
        Pipe.confirmLowLevelWrite(targetData, writeSize);
        Pipe.publishWrites(targetData);
    } while (visitor.isReady() && TrieParserReader.parseHasContent(reader));
    return targetData;
}

Example 4 with RawDataSchema

use of com.ociweb.pronghorn.pipe.RawDataSchema in project PronghornPipes by oci-pronghorn.

the class JSONParseTest method simpleMissingATest.

@Test
public void simpleMissingATest() {
    Pipe<RawDataSchema> targetData = parseJSON(missingAExample, simpleExtractor);
    // confirm data on the pipe is good...
    Pipe.takeMsgIdx(targetData);
    ChannelReader dataStream = (ChannelReader) Pipe.openInputStream(targetData);
    long header = dataStream.readPackedLong();
    // the second field is null so the bit pattern is 10
    assertEquals(2, header);
    String valueB = dataStream.readUTFOfLength(dataStream.readPackedInt());
    assertEquals("hello", valueB);
    // a is missing by design
    // this MUST be zero but it is not not sure why??
    assertEquals(0, dataStream.available());
}

Example 5 with RawDataSchema

use of com.ociweb.pronghorn.pipe.RawDataSchema in project PronghornPipes by oci-pronghorn.

the class JSONParseTest method simpleArrayParseTest.

@Test
public void simpleArrayParseTest() {
    try {
        Pipe<RawDataSchema> targetData = parseJSON(simpleArrayExample, simpleArrayExtractor);
        // confirm data on the pipe is good...
        Pipe.takeMsgIdx(targetData);
        ChannelReader dataStream = (ChannelReader) Pipe.openInputStream(targetData);
        long header = dataStream.readPackedLong();
        assertEquals(0, header);
        assertEquals(5, dataStream.readPackedInt());
        // these are 19 bytes plus 10, 29
        assertEquals("one", dataStream.readUTFOfLength(dataStream.readPackedInt()));
        assertEquals("two", dataStream.readUTFOfLength(dataStream.readPackedInt()));
        assertEquals("three", dataStream.readUTFOfLength(dataStream.readPackedInt()));
        assertEquals("four", dataStream.readUTFOfLength(dataStream.readPackedInt()));
        assertEquals("five", dataStream.readUTFOfLength(dataStream.readPackedInt()));
        assertEquals(5, dataStream.readPackedInt());
        assertEquals(1, dataStream.readPackedLong());
        assertEquals(2, dataStream.readPackedLong());
        assertEquals(3, dataStream.readPackedLong());
        assertEquals(4, dataStream.readPackedLong());
        assertEquals(5, dataStream.readPackedLong());
        assertEquals(0, dataStream.available());
    } catch (Throwable t) {
        t.printStackTrace();
        throw new AssertionError("rethrow", t);
    }
}