Examples with Pipe com.ociweb.pronghorn.pipe.Pipe used on opensource projects

Example 36 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project GreenLightning by oci-pronghorn.

the class Grouper method add.

public void add(Pipe[] pipes) {
    if (0 == count) {
        first = pipes;
    }
    count++;
    int i = pipes.length;
    while (--i >= 0) {
        Pipe p = pipes[i];
        int j = inputPipes.length;
        while (--j >= 0) {
            if (Pipe.isForSameSchema(inputPipes[j], p)) {
                Pipe[] targetArray = groupedPipes[j];
                Pipe[] newArray = new Pipe[targetArray.length + 1];
                System.arraycopy(targetArray, 0, newArray, 0, targetArray.length);
                newArray[targetArray.length] = p;
                groupedPipes[j] = newArray;
                return;
            }
        }
    }
}

Example 37 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class PipeSingleTemplateDecimalTest method simpleWriteRead.

@Test
public void simpleWriteRead() {
    byte primaryRingSizeInBits = 9;
    byte byteRingSizeInBits = 17;
    Pipe ring = new Pipe(new PipeConfig(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring.initBuffers();
    String emptyToString = ring.toString();
    assertTrue(emptyToString, emptyToString.contains("slabHeadPos 0"));
    assertTrue(emptyToString, emptyToString.contains("slabTailPos 0"));
    assertTrue(emptyToString, emptyToString.contains("RingId"));
    int messageSize = FROM.fragDataSize[FRAG_LOC];
    int varDataMax = (ring.blobMask / (ring.slabMask >> 1)) / messageSize;
    // reduce by one so we have room for the ending EOM value
    int testSize = ((1 << primaryRingSizeInBits) / messageSize) - 1;
    writeTestValue(ring, varDataMax, testSize);
    // now read the data back
    int FIELD_LOC = FieldReferenceOffsetManager.lookupFieldLocator(SINGLE_MESSAGE_NAMES[0], FRAG_LOC, FROM);
    int k = testSize;
    while (PipeReader.tryReadFragment(ring)) {
        --k;
        assertTrue(PipeReader.isNewMessage(ring));
        int messageIdx = PipeReader.getMsgIdx(ring);
        if (messageIdx < 0) {
            break;
        }
        readTestValue(ring, varDataMax, testSize, FIELD_LOC, k, messageIdx);
    }
}

Example 38 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class StreamingConsumerTest method generatorTest.

@Test
public void generatorTest() {
    final int seed = 2;
    // hard coded value that comes from this seed 2
    final long aLongValue = 2945688134060370505l;
    // hard coded value that comes from this seed 2
    final int aNegIntValue = -29;
    Pipe<MessageSchemaDynamic> ring = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), 50, 30));
    ring.initBuffers();
    StreamingWriteVisitorGenerator swvg = new StreamingWriteVisitorGenerator(FROM, new Random(seed), 30, 30);
    StreamingVisitorWriter svw = new StreamingVisitorWriter(ring, swvg);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintStream ps = new PrintStream(baos);
    // PrintStream ps = System.out;
    StreamingReadVisitor visitor = new StreamingReadVisitorToJSON(ps);
    // , new StreamingReadVisitorDebugDelegate(visitor) );
    StreamingVisitorReader reader = new StreamingVisitorReader(ring, visitor);
    svw.startup();
    reader.startup();
    do {
        svw.run();
    } while (!svw.isAtBreakPoint());
    reader.run();
    svw.shutdown();
    reader.shutdown();
    byte[] byteArray = baos.toByteArray();
    assertTrue("No JSON was produced", byteArray.length > 0);
    String results = new String(byteArray);
    // spot check the produced JSON
    assertTrue(results, results.indexOf("\"Trucks\":") > 0);
    assertTrue(results, results.indexOf("{\"Squad\":") > 0);
    assertTrue(results, results.indexOf(Long.toString(aLongValue)) > 0);
    assertTrue(results, results.indexOf(Integer.toString(aNegIntValue)) > 0);
}

Example 39 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class PipeSingleTemplateFloatTest method simpleWriteReadThreaded.

@Test
public void simpleWriteReadThreaded() {
    // this ring is 2^7 eg 128
    final byte primaryRingSizeInBits = 7;
    final byte byteRingSizeInBits = 16;
    final Pipe ring = new Pipe(new PipeConfig(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring.initBuffers();
    final int messageSize = FROM.fragDataSize[FRAG_LOC];
    final int varDataMax = (ring.blobMask / (ring.slabMask >> 1)) / messageSize;
    final int testSize = (1 << primaryRingSizeInBits) / messageSize;
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            writeTestValue(ring, varDataMax, testSize);
        }
    });
    t.start();
    // now read the data back
    int FIELD_LOC = FieldReferenceOffsetManager.lookupFieldLocator(SINGLE_MESSAGE_NAMES[0], FRAG_LOC, FROM);
    int k = testSize;
    while (k > 0) {
        // System.err.println("content "+ring.contentRemaining(ring));
        if (PipeReader.tryReadFragment(ring)) {
            // this method releases old messages as needed and moves pointer up to the next fragment
            // count down all the expected messages so we stop this test at the right time
            k--;
            assertTrue(PipeReader.isNewMessage(ring));
            int messageIdx = PipeReader.getMsgIdx(ring);
            if (messageIdx < 0) {
                return;
            }
            testReadValue(ring, varDataMax, testSize, FIELD_LOC, k, messageIdx);
        } else {
            // unable to read so at this point
            // we can do other work and try again soon
            Thread.yield();
        }
    }
}

Example 40 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project PronghornPipes by oci-pronghorn.

the class RingStreamsTest method testRingToRingInputStream.

// TODO: RingStreams needs to be deleted and no longer used.
@Ignore
public void testRingToRingInputStream() {
    Pipe testRing = new Pipe(new PipeConfig(RawDataSchema.instance, 1 << 5, 1 << 13));
    testRing.initBuffers();
    int blockSize = testRing.maxVarLen;
    RingInputStream ringInputStream = new RingInputStream(testRing);
    Pipe targetRing = new Pipe(new PipeConfig(RawDataSchema.instance, 1 << 5, 1 << 13));
    targetRing.initBuffers();
    int testSize = 3000;
    byte[] testData = new byte[testSize];
    int testIdx = 0;
    while (testIdx < testSize) {
        assertEquals(0, Pipe.contentRemaining(targetRing));
        // Write data into the the ring buffer
        RingStreams.writeBytesToRing(testData, 0, testIdx, testRing, blockSize);
        RingStreams.writeEOF(testRing);
        // Here we are reading from one ring and writing to another ring going through an InputStream
        try {
            RingStreams.readFromInputStream(ringInputStream, targetRing);
            RingStreams.writeEOF(targetRing);
        } catch (IOException e) {
            e.printStackTrace();
            fail();
        }
        // Now read the data off the target ring to confirm it matches
        ByteArrayOutputStream baost = new ByteArrayOutputStream();
        try {
            RingStreams.writeToOutputStream(targetRing, baost);
        } catch (IOException e) {
            e.printStackTrace();
            fail();
        }
        assertTrue("len:" + testIdx, Arrays.equals(Arrays.copyOfRange(testData, 0, testIdx), baost.toByteArray()));
        testData[testIdx] = (byte) (testIdx & 0xFF);
        testIdx++;
    }
}