Examples with Pipe com.ociweb.pronghorn.pipe.Pipe used on opensource projects

Example 11 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class PipeSingleTemplateDecimalTest method simpleWriteReadThreaded.

@Test
public void simpleWriteReadThreaded() {
    // this ring is 2^7 eg 128
    final byte primaryRingSizeInBits = 8;
    final byte byteRingSizeInBits = 16;
    final Pipe ring = new Pipe(new PipeConfig(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring.initBuffers();
    final int messageSize = FROM.fragDataSize[FRAG_LOC];
    final int varDataMax = (ring.blobMask / (ring.slabMask >> 1)) / messageSize;
    final int testSize = (1 << primaryRingSizeInBits) / messageSize;
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            writeTestValue(ring, varDataMax, testSize);
        }
    });
    t.start();
    // now read the data back
    int FIELD_LOC = FieldReferenceOffsetManager.lookupFieldLocator(SINGLE_MESSAGE_NAMES[0], FRAG_LOC, FROM);
    int k = testSize;
    while (k > 0) {
        // System.err.println("content "+ring.contentRemaining(ring));
        if (PipeReader.tryReadFragment(ring)) {
            // this method releases old messages as needed and moves pointer up to the next fragment
            // count down all the expected messages so we stop this test at the right time
            k--;
            assertTrue(PipeReader.isNewMessage(ring));
            int messageIdx = PipeReader.getMsgIdx(ring);
            if (messageIdx < 0) {
                break;
            }
            readTestValue(ring, varDataMax, testSize, FIELD_LOC, k, messageIdx);
        } else {
            // unable to read so at this point
            // we can do other work and try again soon
            Thread.yield();
        }
    }
}

Example 12 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class PipeSingleTemplateTest method simpleBytesWriteReadThreaded.

@Test
public void simpleBytesWriteReadThreaded() {
    // this ring is 2^7 eg 128
    final byte primaryRingSizeInBits = 7;
    final byte byteRingSizeInBits = 16;
    final Pipe ring = new Pipe(new PipeConfig(RawDataSchema.instance, 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring.initBuffers();
    final int messageSize = FROM.fragDataSize[FRAG_LOC];
    final int varDataMax = (ring.blobMask / (ring.slabMask >> 1)) / messageSize;
    final int testSize = (1 << primaryRingSizeInBits) / messageSize;
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            populateRingBufferWithBytes(ring, varDataMax, testSize);
        }
    });
    t.start();
    // now read the data back
    byte[] target = new byte[varDataMax];
    int BYTE_LOC = FieldReferenceOffsetManager.lookupFieldLocator("ByteArray", FRAG_LOC, FROM);
    int k = testSize;
    while (k > 1) {
        // System.err.println("content "+ring.contentRemaining(ring));
        if (PipeReader.tryReadFragment(ring)) {
            // this method releases old messages as needed and moves pointer up to the next fragment
            // count down all the expected messages so we stop this test at the right time
            k--;
            // would use this method rarely to determine if fragment starts new message
            assertTrue(PipeReader.isNewMessage(ring));
            // when we only have 1 message type this would not normally be called
            assertEquals(0, PipeReader.getMsgIdx(ring));
            int expectedLength = (varDataMax * k) / testSize;
            // read bytes as normal code would do
            int actualLength = PipeReader.readBytes(ring, BYTE_LOC, target, 0);
            assertEquals(expectedLength, actualLength);
        } else {
            // unable to read so at this point
            // we can do other work and try again soon
            Thread.yield();
        }
    }
}

Example 13 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class PipeSingleTemplateTest method simpleBytesWriteRead.

@Test
public void simpleBytesWriteRead() {
    // this ring is 2^7 eg 128
    byte primaryRingSizeInBits = 7;
    byte byteRingSizeInBits = 16;
    Pipe ring = new Pipe(new PipeConfig(RawDataSchema.instance, 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring.initBuffers();
    int messageSize = FROM.fragDataSize[FRAG_LOC];
    int varDataMax = (ring.blobMask / (ring.slabMask >> 1)) / messageSize;
    int testSize = (1 << primaryRingSizeInBits) / messageSize;
    populateRingBufferWithBytes(ring, varDataMax, testSize);
    // now read the data back
    int BYTE_LOC = FieldReferenceOffsetManager.lookupFieldLocator("ByteArray", FRAG_LOC, FROM);
    byte[] target = new byte[varDataMax];
    int k = testSize;
    while (PipeReader.tryReadFragment(ring)) {
        if (PipeReader.isNewMessage(ring)) {
            assertEquals(0, PipeReader.getMsgIdx(ring));
            int expectedLength = (varDataMax * (--k)) / testSize;
            // read bytes as normal code would do
            int actualLength = PipeReader.readBytes(ring, BYTE_LOC, target, 0);
            assertEquals(expectedLength, actualLength);
        }
    }
}

Example 14 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class StreamingConsumerTest method matchingTestPositive.

@Test
public void matchingTestPositive() {
    Pipe<MessageSchemaDynamic> ring1 = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    Pipe<MessageSchemaDynamic> ring2 = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring1.initBuffers();
    ring2.initBuffers();
    int commonSeed = 100;
    StreamingWriteVisitorGenerator swvg1 = new StreamingWriteVisitorGenerator(FROM, new Random(commonSeed), 30, 30);
    StreamingVisitorWriter svw1 = new StreamingVisitorWriter(ring1, swvg1);
    StreamingWriteVisitorGenerator swvg2 = new StreamingWriteVisitorGenerator(FROM, new Random(commonSeed), 30, 30);
    StreamingVisitorWriter svw2 = new StreamingVisitorWriter(ring2, swvg2);
    svw1.startup();
    svw2.startup();
    svw1.run();
    svw2.run();
    svw1.run();
    svw2.run();
    // confirm that both rings contain the exact same thing
    assertTrue(Arrays.equals(Pipe.primaryBuffer(ring1), Pipe.primaryBuffer(ring2)));
    assertTrue(Arrays.equals(Pipe.byteBuffer(ring1), Pipe.byteBuffer(ring2)));
    // now use matcher to confirm the same.
    StreamingReadVisitorMatcher srvm = new StreamingReadVisitorMatcher(ring1);
    // new StreamingReadVisitorDebugDelegate(srvm) );
    StreamingVisitorReader svr = new StreamingVisitorReader(ring2, srvm);
    svr.startup();
    try {
        svr.run();
    } catch (Throwable t) {
        t.printStackTrace();
        fail(t.getMessage());
    }
    svr.shutdown();
    svw1.shutdown();
    svw2.shutdown();
}

Example 15 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class StreamingConsumerTest method sequenceFragmentWriteRead.

@Test
public void sequenceFragmentWriteRead() {
    Pipe<MessageSchemaDynamic> ring = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring.initBuffers();
    int testSize = 5;
    // in this method we write two sequence members but only record the count after writing the members
    populateRingBufferWithSequence(ring, testSize);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintStream ps = new PrintStream(baos);
    StreamingReadVisitor visitor = new StreamingReadVisitorToJSON(ps);
    // new StreamingReadVisitorDebugDelegate(visitor) );
    StreamingVisitorReader reader = new StreamingVisitorReader(ring, visitor);
    // ring is fully populated so we should not need to call this run again
    while (Pipe.contentRemaining(ring) > 0) {
        reader.run();
    }
    ps.close();
    String results = new String(baos.toByteArray());
    // spot check the produced JSON
    assertTrue(results, results.indexOf("\"TruckId\":10") > 0);
    assertTrue(results, results.indexOf("{\"AThing\":7}") > 0);
    assertTrue(results, results.indexOf("{\"JustOneMoreQuestion\":42}") > 0);
}