Examples with PipeConfig com.ociweb.pronghorn.pipe.PipeConfig used on opensource projects

Example 6 with PipeConfig

use of com.ociweb.pronghorn.pipe.PipeConfig in project PronghornPipes by oci-pronghorn.

the class RingStreamsTest method testRingToRingInputStreamToggleMethods.

// TODO: RingStreams needs to be deleted and no longer used.
@Ignore
public void testRingToRingInputStreamToggleMethods() {
    Pipe testRing = new Pipe(new PipeConfig(RawDataSchema.instance, 1 << 4, 1 << 12));
    testRing.initBuffers();
    int blockSize = testRing.maxVarLen;
    RingInputStream ringInputStream = new RingInputStream(testRing);
    int testSize = 2048;
    byte[] testData = new byte[testSize];
    int testIdx = 0;
    while (testIdx < testSize) {
        assertEquals(0, Pipe.contentRemaining(testRing));
        int j = 10;
        while (--j >= 0) {
            // Write data into the the ring buffer
            RingStreams.writeBytesToRing(testData, 0, testIdx, testRing, blockSize);
            RingStreams.writeEOF(testRing);
            ByteArrayOutputStream baost = new ByteArrayOutputStream();
            int value;
            try {
                int buf = 7;
                byte[] tempBuf = new byte[buf];
                // must span these to calls.
                while ((value = ringInputStream.read(tempBuf)) >= 0) {
                    // using array read
                    baost.write(tempBuf, 0, value);
                    if ((value = ringInputStream.read()) >= 0) {
                        // using single byte read
                        baost.write(value);
                    } else {
                        break;
                    }
                }
                ringInputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
                fail();
            }
            assertTrue("len:" + testIdx + " vs " + baost.toByteArray().length, Arrays.equals(Arrays.copyOfRange(testData, 0, testIdx), baost.toByteArray()));
        }
        testData[testIdx] = (byte) (testIdx & 0xFF);
        testIdx++;
    }
}

Example 7 with PipeConfig

use of com.ociweb.pronghorn.pipe.PipeConfig in project PronghornPipes by oci-pronghorn.

the class RingStreamsTest method testRingToRingOutputStreamByte.

// TODO: RingStreams needs to be deleted and no longer used.
@Ignore
public void testRingToRingOutputStreamByte() {
    Pipe targetRing = new Pipe(new PipeConfig(RawDataSchema.instance, 1 << 10, 1 << 15));
    targetRing.initBuffers();
    targetRing.reset((1 << 10) - 3, 1 << 14);
    RingOutputStream ringOutputStream = new RingOutputStream(targetRing);
    int testBits = 8;
    int testSize = 1 << testBits;
    int testMask = testSize - 1;
    byte[] testData = new byte[testSize];
    int j = testSize;
    while (--j >= 0) {
        testData[j] = (byte) (j & 0xFF);
    }
    int testIdx = 0;
    int testTotal = testSize * 40;
    while (testIdx < testTotal) {
        int datLen = testIdx & testMask;
        assertEquals(0, Pipe.contentRemaining(targetRing));
        int i = 0;
        while (i < datLen) {
            ringOutputStream.write(testData[i++]);
        }
        ringOutputStream.close();
        // Now read the data off the target ring to confirm it matches
        ByteArrayOutputStream baost = new ByteArrayOutputStream();
        try {
            RingStreams.writeToOutputStream(targetRing, baost);
        } catch (Throwable e) {
            e.printStackTrace();
            fail();
        }
        byte[] byteArray = baost.toByteArray();
        // Arrays.equals(Arrays.copyOfRange(testData,0,datLen), byteArray));
        assertEquals("test:" + testIdx + " expected len:" + datLen + " data len:" + byteArray.length, datLen, byteArray.length);
        assertTrue(Arrays.toString(Arrays.copyOfRange(testData, 0, datLen)) + " vs " + Arrays.toString(byteArray), Arrays.equals(Arrays.copyOfRange(testData, 0, datLen), byteArray));
        testIdx++;
    }
}

Example 8 with PipeConfig

use of com.ociweb.pronghorn.pipe.PipeConfig in project PronghornPipes by oci-pronghorn.

the class GenerativeTest method mostImporantCoverageTest.

@Test
public void mostImporantCoverageTest() {
    StringBuilder schema = new StringBuilder();
    generateCoveringTestSchema(schema);
    FieldReferenceOffsetManager from = loadFrom(schema.toString());
    PipeConfig rbConfig = new PipeConfig(new MessageSchemaDynamic(from), messages, varLength);
    int commonSeed = 300;
    int iterations = 2;
    Pipe ring1 = buildPopulatedRing(from, rbConfig, commonSeed, iterations);
    Pipe ring2 = buildPopulatedRing(from, rbConfig, commonSeed, iterations);
    // confirm that both rings contain the exact same thing
    assertTrue(Arrays.equals(Pipe.primaryBuffer(ring1), Pipe.primaryBuffer(ring2)));
    assertTrue(Arrays.equals(Pipe.byteBuffer(ring1), Pipe.byteBuffer(ring2)));
    // ////////////////////////
    // now starts the real test, we need to read/write these values, and check them against the original
    // ///////////////////////
    // while try read from token look up the right read type.
    // reads all the messsages until the ring is empty
    Pipe ring1byReadWrite = new Pipe(rbConfig);
    Pipe ring1byCopy = new Pipe(rbConfig);
    ring1byReadWrite.initBuffers();
    // first and only messsage;
    int messageIdx = 0;
    while (PipeReader.tryReadFragment(ring1) && PipeWriter.tryWriteFragment(ring1byReadWrite, messageIdx)) {
        int msgId = PipeReader.getMsgIdx(ring1);
        int scriptSize = from.fragScriptSize[msgId];
        int s = 0;
        int fieldId = 1000;
        while (++s < scriptSize) {
            int idx = msgId + s;
            fieldId++;
            long fromFieldId = from.fieldIdScript[idx];
            if (0 != fromFieldId && fromFieldId != fieldId) {
                fail("Did not find expected field id " + fieldId + " != " + fromFieldId);
            }
            // this is a slow linear search repeated for each message, TODO: B, replace this with an array of pre-build values
            int fieldLOC = 0 == fromFieldId ? 0 : FieldReferenceOffsetManager.lookupFieldLocator(fromFieldId, msgId, from);
            int token = from.tokens[idx];
            int type = TokenBuilder.extractType(token);
            switch(type) {
                case TypeMask.IntegerUnsigned:
                case TypeMask.IntegerUnsignedOptional:
                case TypeMask.IntegerSigned:
                case TypeMask.IntegerSignedOptional:
                    int intValue = PipeReader.readInt(ring1, fieldLOC);
                    float floatValue = PipeReader.readIntBitsToFloat(ring1, fieldLOC);
                    int expectedInt = Float.floatToRawIntBits(floatValue);
                    if (intValue != expectedInt) {
                        fail();
                    }
                    PipeWriter.writeInt(ring1byReadWrite, fieldLOC, intValue);
                    break;
                case TypeMask.LongUnsigned:
                case TypeMask.LongUnsignedOptional:
                case TypeMask.LongSigned:
                case TypeMask.LongSignedOptional:
                    long longValue = PipeReader.readLong(ring1, fieldLOC);
                    double doubleValue = PipeReader.readLongBitsToDouble(ring1, fieldLOC);
                    long expectedLong = Double.doubleToRawLongBits(doubleValue);
                    if (longValue != expectedLong) {
                        fail();
                    }
                    break;
                case TypeMask.TextASCII:
                case TypeMask.TextASCIIOptional:
                    PipeReader.readASCII(ring1, fieldLOC, new StringBuilder());
                    break;
                case TypeMask.TextUTF8:
                case TypeMask.TextUTF8Optional:
                    // RingReader.readUTF8(ring1, fieldLOC, new StringBuilder());
                    break;
                case TypeMask.Decimal:
                case TypeMask.DecimalOptional:
                    PipeReader.readDecimalExponent(ring1, fieldLOC);
                    PipeReader.readDecimalMantissa(ring1, fieldLOC);
                    break;
                case TypeMask.ByteVector:
                case TypeMask.ByteVectorOptional:
                    // RingReader.readBytes(ring1, fieldLOC, ByteBuffer.allocate(70));
                    break;
            }
            if (TypeMask.Decimal == type || TypeMask.DecimalOptional == type) {
                // extra slot for the long
                s++;
            }
        }
    }
    // once per message.
    PipeReader.releaseReadLock(ring1);
    System.err.println(ring1);
// load schema and generate test data
// generate test data off generated template files
// test read and write of all data
// use visitor API
// use high level RingReader/RingWalker
}

Example 9 with PipeConfig

use of com.ociweb.pronghorn.pipe.PipeConfig in project PronghornPipes by oci-pronghorn.

the class PipeSingleTemplateDecimalTest method simpleWriteReadThreaded.

@Test
public void simpleWriteReadThreaded() {
    // this ring is 2^7 eg 128
    final byte primaryRingSizeInBits = 8;
    final byte byteRingSizeInBits = 16;
    final Pipe ring = new Pipe(new PipeConfig(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring.initBuffers();
    final int messageSize = FROM.fragDataSize[FRAG_LOC];
    final int varDataMax = (ring.blobMask / (ring.slabMask >> 1)) / messageSize;
    final int testSize = (1 << primaryRingSizeInBits) / messageSize;
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            writeTestValue(ring, varDataMax, testSize);
        }
    });
    t.start();
    // now read the data back
    int FIELD_LOC = FieldReferenceOffsetManager.lookupFieldLocator(SINGLE_MESSAGE_NAMES[0], FRAG_LOC, FROM);
    int k = testSize;
    while (k > 0) {
        // System.err.println("content "+ring.contentRemaining(ring));
        if (PipeReader.tryReadFragment(ring)) {
            // this method releases old messages as needed and moves pointer up to the next fragment
            // count down all the expected messages so we stop this test at the right time
            k--;
            assertTrue(PipeReader.isNewMessage(ring));
            int messageIdx = PipeReader.getMsgIdx(ring);
            if (messageIdx < 0) {
                break;
            }
            readTestValue(ring, varDataMax, testSize, FIELD_LOC, k, messageIdx);
        } else {
            // unable to read so at this point
            // we can do other work and try again soon
            Thread.yield();
        }
    }
}

Example 10 with PipeConfig

use of com.ociweb.pronghorn.pipe.PipeConfig in project PronghornPipes by oci-pronghorn.

the class PipeSingleTemplateTest method simpleBytesWriteReadThreaded.

@Test
public void simpleBytesWriteReadThreaded() {
    // this ring is 2^7 eg 128
    final byte primaryRingSizeInBits = 7;
    final byte byteRingSizeInBits = 16;
    final Pipe ring = new Pipe(new PipeConfig(RawDataSchema.instance, 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring.initBuffers();
    final int messageSize = FROM.fragDataSize[FRAG_LOC];
    final int varDataMax = (ring.blobMask / (ring.slabMask >> 1)) / messageSize;
    final int testSize = (1 << primaryRingSizeInBits) / messageSize;
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            populateRingBufferWithBytes(ring, varDataMax, testSize);
        }
    });
    t.start();
    // now read the data back
    byte[] target = new byte[varDataMax];
    int BYTE_LOC = FieldReferenceOffsetManager.lookupFieldLocator("ByteArray", FRAG_LOC, FROM);
    int k = testSize;
    while (k > 1) {
        // System.err.println("content "+ring.contentRemaining(ring));
        if (PipeReader.tryReadFragment(ring)) {
            // this method releases old messages as needed and moves pointer up to the next fragment
            // count down all the expected messages so we stop this test at the right time
            k--;
            // would use this method rarely to determine if fragment starts new message
            assertTrue(PipeReader.isNewMessage(ring));
            // when we only have 1 message type this would not normally be called
            assertEquals(0, PipeReader.getMsgIdx(ring));
            int expectedLength = (varDataMax * k) / testSize;
            // read bytes as normal code would do
            int actualLength = PipeReader.readBytes(ring, BYTE_LOC, target, 0);
            assertEquals(expectedLength, actualLength);
        } else {
            // unable to read so at this point
            // we can do other work and try again soon
            Thread.yield();
        }
    }
}