Examples with Pipe com.ociweb.pronghorn.pipe.Pipe used on opensource projects

Example 16 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class StreamingConsumerTest method matchingTestNegative.

@Test
public void matchingTestNegative() {
    Pipe<MessageSchemaDynamic> ring1 = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    Pipe<MessageSchemaDynamic> ring2 = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), primaryRingSizeInBits, byteRingSizeInBits));
    ring1.initBuffers();
    ring2.initBuffers();
    int commonSeed = 300;
    StreamingWriteVisitorGenerator swvg1 = new StreamingWriteVisitorGenerator(FROM, new Random(commonSeed), 30, 30);
    StreamingVisitorWriter svw1 = new StreamingVisitorWriter(ring1, swvg1);
    StreamingWriteVisitorGenerator swvg2 = new StreamingWriteVisitorGenerator(FROM, new Random(commonSeed + 1), 30, 30);
    StreamingVisitorWriter svw2 = new StreamingVisitorWriter(ring2, swvg2);
    svw1.startup();
    svw2.startup();
    svw1.run();
    svw2.run();
    svw1.run();
    svw2.run();
    StreamingReadVisitorMatcher srvm = new StreamingReadVisitorMatcher(ring1);
    StreamingVisitorReader svr = new StreamingVisitorReader(ring2, srvm);
    svr.startup();
    try {
        svr.run();
        fail("expected exception");
    } catch (Throwable t) {
    // success
    // t.printStackTrace();
    }
    svr.shutdown();
    svw1.shutdown();
    svw2.shutdown();
}

Example 17 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class PipeSingleTemplateASCIITest method simpleBytesWriteReadThreaded.

@Test
public void simpleBytesWriteReadThreaded() {
    // this ring is 2^7 eg 128
    final byte primaryRingSizeInBits = 7;
    final byte byteRingSizeInBits = 16;
    final Pipe<RawDataSchema> ring = new Pipe<RawDataSchema>(new PipeConfig(RawDataSchema.instance, 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring.initBuffers();
    final int messageSize = RawDataSchema.FROM.fragDataSize[RawDataSchema.MSG_CHUNKEDSTREAM_1];
    final int varDataMax = (ring.blobMask / (ring.slabMask >> 1)) / messageSize;
    final int testSize = (1 << primaryRingSizeInBits) / messageSize;
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            populateRingBufferWithASCII(ring, varDataMax, testSize);
        }
    });
    t.start();
    // now read the data back
    StringBuilder target = new StringBuilder();
    char[] target2 = new char[varDataMax];
    int k = testSize;
    while (k > 1) {
        // System.err.println("content "+ring.contentRemaining(ring));
        if (PipeReader.tryReadFragment(ring)) {
            // this method releases old messages as needed and moves pointer up to the next fragment
            // count down all the expected messages so we stop this test at the right time
            k--;
            target.setLength(0);
            // would use this method rarely to determine if fragment starts new message
            assertTrue(PipeReader.isNewMessage(ring));
            // when we only have 1 message type this would not normally be called
            assertEquals(0, PipeReader.getMsgIdx(ring));
            int expectedLength = (varDataMax * k) / testSize;
            String testString = buildTestString(expectedLength);
            if (0 == (k & 2)) {
                int actualLength = ((StringBuilder) PipeReader.readASCII(ring, RawDataSchema.MSG_CHUNKEDSTREAM_1_FIELD_BYTEARRAY_2, target)).length();
                assertEquals(expectedLength, actualLength);
                assertEquals(testString, target.toString());
            } else {
                int actualLength = PipeReader.readASCII(ring, RawDataSchema.MSG_CHUNKEDSTREAM_1_FIELD_BYTEARRAY_2, target2, 0);
                assertEquals(expectedLength, actualLength);
                assertTrue(testString + " vs " + new String(target2, 0, actualLength), Arrays.equals(testString.toCharArray(), Arrays.copyOfRange(target2, 0, actualLength)));
            }
        } else {
            // unable to read so at this point
            // we can do other work and try again soon
            Thread.yield();
        }
    }
}

Example 18 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class MessageConsumer method storeValue.

private void storeValue(ChannelReader reader, int type, FieldConsumer consumer) {
    // NB: the order of these are from most common to least common
    if (type < 0x0B) {
        if (type < 0x0A) {
            // integers
            consumer.store(reader.readPackedLong());
        } else {
            Pipe backingPipe = DataInputBlobReader.getBackingPipe((DataInputBlobReader) reader);
            short length = reader.readShort();
            consumer.store(backingPipe.blobRing, DataInputBlobReader.absolutePosition((DataInputBlobReader) reader), length, backingPipe.blobMask);
            if (length > 0) {
                reader.skipBytes(length);
            }
        }
    } else {
        // >=0x0B
        if (type == TypeMask.ByteVector) {
            // bytes
            Pipe backingPipe = DataInputBlobReader.getBackingPipe((DataInputBlobReader) reader);
            short length = reader.readShort();
            consumer.store(backingPipe.blobRing, DataInputBlobReader.absolutePosition((DataInputBlobReader) reader), length, backingPipe.blobMask);
            if (length > 0) {
                reader.skipBytes(length);
            }
        } else {
            if (type == TypeMask.Decimal) {
                // decimal
                consumer.store(reader.readByte(), reader.readPackedLong());
            } else {
                // rational
                consumer.store(reader.readPackedLong(), reader.readPackedLong());
            }
        }
    }
}

Example 19 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class JSONParseTest method parseJSONLoad.

private void parseJSONLoad(int i, String sourceData, JSONExtractorCompleted extractor) {
    PipeConfig<RawDataSchema> targetDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> targetData = new Pipe<RawDataSchema>(targetDataConfig);
    targetData.initBuffers();
    TrieParserReader reader = new TrieParserReader(5, true);
    JSONStreamParser parser = new JSONStreamParser();
    PipeConfig<RawDataSchema> testInputDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> testInputData = new Pipe<RawDataSchema>(testInputDataConfig);
    testInputData.initBuffers();
    // ///////////////
    // ///////////////
    // ///////////////
    JSONStreamVisitorToChannel visitor = extractor.newJSONVisitor();
    byte[] sourceBytes = sourceData.getBytes();
    while (--i >= 0) {
        // ///////////
        // write JSON data
        // ///////////
        assertTrue("content size " + Pipe.contentRemaining(testInputData), Pipe.contentRemaining(testInputData) == 0);
        int size = Pipe.addMsgIdx(testInputData, 0);
        Pipe.addByteArray(sourceBytes, testInputData);
        Pipe.confirmLowLevelWrite(testInputData, size);
        Pipe.publishWrites(testInputData);
        // //
        // ////////
        // start consuming the data from the pipe
        // call the parser
        // /////
        int msgIdx = Pipe.takeMsgIdx(testInputData);
        TrieParserReader.parseSetup(reader, testInputData);
        // parse data data
        parser.parse(reader, extractor.trieParser(), visitor);
        Pipe.confirmLowLevelRead(testInputData, Pipe.sizeOf(testInputData, msgIdx));
        Pipe.releaseReadLock(testInputData);
        // ///write the captured data into the pipe
        int writeSize = Pipe.addMsgIdx(targetData, 0);
        DataOutputBlobWriter<RawDataSchema> stream = Pipe.openOutputStream(targetData);
        visitor.export(stream, null);
        stream.closeLowLevelField();
        Pipe.confirmLowLevelWrite(targetData, writeSize);
        Pipe.publishWrites(targetData);
        // ///////////////////
        // read the parsed data
        // //////////////////
        RawDataSchema.consume(targetData);
    }
}

Example 20 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class JSONParseTest method parseJSON.

private Pipe<RawDataSchema> parseJSON(String sourceData, JSONExtractorCompleted extractor) {
    // ///////////////
    // source test data.
    PipeConfig<RawDataSchema> testInputDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> testInputData = new Pipe<RawDataSchema>(testInputDataConfig);
    testInputData.initBuffers();
    int size = Pipe.addMsgIdx(testInputData, 0);
    Pipe.addUTF8(sourceData, testInputData);
    Pipe.confirmLowLevelWrite(testInputData, size);
    Pipe.publishWrites(testInputData);
    // //
    TrieParserReader reader = new TrieParserReader(5, true);
    // start consuming the data from the pipe
    int msgIdx = Pipe.takeMsgIdx(testInputData);
    TrieParserReader.parseSetup(reader, testInputData);
    // export data to this pipe
    PipeConfig<RawDataSchema> targetDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> targetData = new Pipe<RawDataSchema>(targetDataConfig);
    targetData.initBuffers();
    Pipe.confirmLowLevelRead(testInputData, Pipe.sizeOf(testInputData, msgIdx));
    Pipe.releaseReadLock(testInputData);
    // parse data data
    JSONStreamParser parser = new JSONStreamParser();
    JSONStreamVisitorToChannel visitor = extractor.newJSONVisitor();
    do {
        parser.parse(reader, extractor.trieParser(), visitor);
        // ///write the captured data into the pipe
        Pipe.presumeRoomForWrite(targetData);
        int writeSize = Pipe.addMsgIdx(targetData, 0);
        DataOutputBlobWriter<RawDataSchema> stream = Pipe.openOutputStream(targetData);
        visitor.export(stream, null);
        stream.closeLowLevelField();
        Pipe.confirmLowLevelWrite(targetData, writeSize);
        Pipe.publishWrites(targetData);
    } while (visitor.isReady() && TrieParserReader.parseHasContent(reader));
    return targetData;
}