Examples with MessageSchemaDynamic com.ociweb.pronghorn.pipe.MessageSchemaDynamic used on opensource projects

Example 1 with MessageSchemaDynamic

use of com.ociweb.pronghorn.pipe.MessageSchemaDynamic in project PronghornPipes by oci-pronghorn.

the class TemplateProcessGeneratorTest method testGenerateLowLevelReaderCleanCompile.

@Test
public void testGenerateLowLevelReaderCleanCompile() {
    if ("arm".equals(System.getProperty("os.arch"))) {
        assertTrue(true);
    } else {
        try {
            FieldReferenceOffsetManager from = TemplateHandler.loadFrom("/template/smallExample.xml");
            MessageSchema schema = new MessageSchemaDynamic(from);
            String className = "LowLevelReader";
            StringBuilder target = new StringBuilder();
            TemplateProcessGeneratorLowLevelReader simple = new TemplateProcessGeneratorLowLevelReader(schema, target);
            simple.processSchema();
            // System.out.println(target);
            validateCleanCompile(className, target);
        } catch (ParserConfigurationException e) {
            e.printStackTrace();
            fail();
        } catch (SAXException e) {
            e.printStackTrace();
            fail();
        } catch (IOException e) {
            e.printStackTrace();
            fail();
        }
    }
}

Example 2 with MessageSchemaDynamic

use of com.ociweb.pronghorn.pipe.MessageSchemaDynamic in project PronghornPipes by oci-pronghorn.

the class GenerativeTest method mostImporantCoverageTest.

@Test
public void mostImporantCoverageTest() {
    StringBuilder schema = new StringBuilder();
    generateCoveringTestSchema(schema);
    FieldReferenceOffsetManager from = loadFrom(schema.toString());
    PipeConfig rbConfig = new PipeConfig(new MessageSchemaDynamic(from), messages, varLength);
    int commonSeed = 300;
    int iterations = 2;
    Pipe ring1 = buildPopulatedRing(from, rbConfig, commonSeed, iterations);
    Pipe ring2 = buildPopulatedRing(from, rbConfig, commonSeed, iterations);
    // confirm that both rings contain the exact same thing
    assertTrue(Arrays.equals(Pipe.primaryBuffer(ring1), Pipe.primaryBuffer(ring2)));
    assertTrue(Arrays.equals(Pipe.byteBuffer(ring1), Pipe.byteBuffer(ring2)));
    // ////////////////////////
    // now starts the real test, we need to read/write these values, and check them against the original
    // ///////////////////////
    // while try read from token look up the right read type.
    // reads all the messsages until the ring is empty
    Pipe ring1byReadWrite = new Pipe(rbConfig);
    Pipe ring1byCopy = new Pipe(rbConfig);
    ring1byReadWrite.initBuffers();
    // first and only messsage;
    int messageIdx = 0;
    while (PipeReader.tryReadFragment(ring1) && PipeWriter.tryWriteFragment(ring1byReadWrite, messageIdx)) {
        int msgId = PipeReader.getMsgIdx(ring1);
        int scriptSize = from.fragScriptSize[msgId];
        int s = 0;
        int fieldId = 1000;
        while (++s < scriptSize) {
            int idx = msgId + s;
            fieldId++;
            long fromFieldId = from.fieldIdScript[idx];
            if (0 != fromFieldId && fromFieldId != fieldId) {
                fail("Did not find expected field id " + fieldId + " != " + fromFieldId);
            }
            // this is a slow linear search repeated for each message, TODO: B, replace this with an array of pre-build values
            int fieldLOC = 0 == fromFieldId ? 0 : FieldReferenceOffsetManager.lookupFieldLocator(fromFieldId, msgId, from);
            int token = from.tokens[idx];
            int type = TokenBuilder.extractType(token);
            switch(type) {
                case TypeMask.IntegerUnsigned:
                case TypeMask.IntegerUnsignedOptional:
                case TypeMask.IntegerSigned:
                case TypeMask.IntegerSignedOptional:
                    int intValue = PipeReader.readInt(ring1, fieldLOC);
                    float floatValue = PipeReader.readIntBitsToFloat(ring1, fieldLOC);
                    int expectedInt = Float.floatToRawIntBits(floatValue);
                    if (intValue != expectedInt) {
                        fail();
                    }
                    PipeWriter.writeInt(ring1byReadWrite, fieldLOC, intValue);
                    break;
                case TypeMask.LongUnsigned:
                case TypeMask.LongUnsignedOptional:
                case TypeMask.LongSigned:
                case TypeMask.LongSignedOptional:
                    long longValue = PipeReader.readLong(ring1, fieldLOC);
                    double doubleValue = PipeReader.readLongBitsToDouble(ring1, fieldLOC);
                    long expectedLong = Double.doubleToRawLongBits(doubleValue);
                    if (longValue != expectedLong) {
                        fail();
                    }
                    break;
                case TypeMask.TextASCII:
                case TypeMask.TextASCIIOptional:
                    PipeReader.readASCII(ring1, fieldLOC, new StringBuilder());
                    break;
                case TypeMask.TextUTF8:
                case TypeMask.TextUTF8Optional:
                    // RingReader.readUTF8(ring1, fieldLOC, new StringBuilder());
                    break;
                case TypeMask.Decimal:
                case TypeMask.DecimalOptional:
                    PipeReader.readDecimalExponent(ring1, fieldLOC);
                    PipeReader.readDecimalMantissa(ring1, fieldLOC);
                    break;
                case TypeMask.ByteVector:
                case TypeMask.ByteVectorOptional:
                    // RingReader.readBytes(ring1, fieldLOC, ByteBuffer.allocate(70));
                    break;
            }
            if (TypeMask.Decimal == type || TypeMask.DecimalOptional == type) {
                // extra slot for the long
                s++;
            }
        }
    }
    // once per message.
    PipeReader.releaseReadLock(ring1);
    System.err.println(ring1);
// load schema and generate test data
// generate test data off generated template files
// test read and write of all data
// use visitor API
// use high level RingReader/RingWalker
}

Example 3 with MessageSchemaDynamic

use of com.ociweb.pronghorn.pipe.MessageSchemaDynamic in project PronghornPipes by oci-pronghorn.

the class StreamingConsumerTest method matchingTestPositive.

@Test
public void matchingTestPositive() {
    Pipe<MessageSchemaDynamic> ring1 = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    Pipe<MessageSchemaDynamic> ring2 = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring1.initBuffers();
    ring2.initBuffers();
    int commonSeed = 100;
    StreamingWriteVisitorGenerator swvg1 = new StreamingWriteVisitorGenerator(FROM, new Random(commonSeed), 30, 30);
    StreamingVisitorWriter svw1 = new StreamingVisitorWriter(ring1, swvg1);
    StreamingWriteVisitorGenerator swvg2 = new StreamingWriteVisitorGenerator(FROM, new Random(commonSeed), 30, 30);
    StreamingVisitorWriter svw2 = new StreamingVisitorWriter(ring2, swvg2);
    svw1.startup();
    svw2.startup();
    svw1.run();
    svw2.run();
    svw1.run();
    svw2.run();
    // confirm that both rings contain the exact same thing
    assertTrue(Arrays.equals(Pipe.primaryBuffer(ring1), Pipe.primaryBuffer(ring2)));
    assertTrue(Arrays.equals(Pipe.byteBuffer(ring1), Pipe.byteBuffer(ring2)));
    // now use matcher to confirm the same.
    StreamingReadVisitorMatcher srvm = new StreamingReadVisitorMatcher(ring1);
    // new StreamingReadVisitorDebugDelegate(srvm) );
    StreamingVisitorReader svr = new StreamingVisitorReader(ring2, srvm);
    svr.startup();
    try {
        svr.run();
    } catch (Throwable t) {
        t.printStackTrace();
        fail(t.getMessage());
    }
    svr.shutdown();
    svw1.shutdown();
    svw2.shutdown();
}

Example 4 with MessageSchemaDynamic

use of com.ociweb.pronghorn.pipe.MessageSchemaDynamic in project PronghornPipes by oci-pronghorn.

the class StreamingConsumerTest method sequenceFragmentWriteRead.

@Test
public void sequenceFragmentWriteRead() {
    Pipe<MessageSchemaDynamic> ring = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    ring.initBuffers();
    int testSize = 5;
    // in this method we write two sequence members but only record the count after writing the members
    populateRingBufferWithSequence(ring, testSize);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintStream ps = new PrintStream(baos);
    StreamingReadVisitor visitor = new StreamingReadVisitorToJSON(ps);
    // new StreamingReadVisitorDebugDelegate(visitor) );
    StreamingVisitorReader reader = new StreamingVisitorReader(ring, visitor);
    // ring is fully populated so we should not need to call this run again
    while (Pipe.contentRemaining(ring) > 0) {
        reader.run();
    }
    ps.close();
    String results = new String(baos.toByteArray());
    // spot check the produced JSON
    assertTrue(results, results.indexOf("\"TruckId\":10") > 0);
    assertTrue(results, results.indexOf("{\"AThing\":7}") > 0);
    assertTrue(results, results.indexOf("{\"JustOneMoreQuestion\":42}") > 0);
}

Example 5 with MessageSchemaDynamic

use of com.ociweb.pronghorn.pipe.MessageSchemaDynamic in project PronghornPipes by oci-pronghorn.

the class StreamingConsumerTest method matchingTestNegative.

@Test
public void matchingTestNegative() {
    Pipe<MessageSchemaDynamic> ring1 = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), 1 << primaryRingSizeInBits, 1 << byteRingSizeInBits));
    Pipe<MessageSchemaDynamic> ring2 = new Pipe<MessageSchemaDynamic>(new PipeConfig<MessageSchemaDynamic>(new MessageSchemaDynamic(FROM), primaryRingSizeInBits, byteRingSizeInBits));
    ring1.initBuffers();
    ring2.initBuffers();
    int commonSeed = 300;
    StreamingWriteVisitorGenerator swvg1 = new StreamingWriteVisitorGenerator(FROM, new Random(commonSeed), 30, 30);
    StreamingVisitorWriter svw1 = new StreamingVisitorWriter(ring1, swvg1);
    StreamingWriteVisitorGenerator swvg2 = new StreamingWriteVisitorGenerator(FROM, new Random(commonSeed + 1), 30, 30);
    StreamingVisitorWriter svw2 = new StreamingVisitorWriter(ring2, swvg2);
    svw1.startup();
    svw2.startup();
    svw1.run();
    svw2.run();
    svw1.run();
    svw2.run();
    StreamingReadVisitorMatcher srvm = new StreamingReadVisitorMatcher(ring1);
    StreamingVisitorReader svr = new StreamingVisitorReader(ring2, srvm);
    svr.startup();
    try {
        svr.run();
        fail("expected exception");
    } catch (Throwable t) {
    // success
    // t.printStackTrace();
    }
    svr.shutdown();
    svw1.shutdown();
    svw2.shutdown();
}