Examples with TrieParserReader com.ociweb.pronghorn.util.TrieParserReader used on opensource projects

Example 1 with TrieParserReader

use of com.ociweb.pronghorn.util.TrieParserReader in project GreenLightning by oci-pronghorn.

the class MessagePubSubStage method startup.

@Override
public void startup() {
    super.startup();
    tempSubject.initBuffers();
    int incomingPipeCount = incomingSubsAndPubsPipe.length;
    // for each pipe we must keep track of the consumed marks before sending the ack back
    int outgoingPipeCount = outgoingMessagePipes.length;
    consumedMarks = new long[incomingPipeCount][outgoingPipeCount];
    pendingAck = new boolean[incomingPipeCount];
    requiredConsumes = new int[incomingPipeCount];
    // maximum count of outgoing pipes * 2 for extra hash room
    deDupeTable = new IntHashTable(IntHashTable.computeBits(outgoingPipeCount * 2));
    this.subscriberLists = new int[initialSubscriptions * subscriberListSize];
    Arrays.fill(this.subscriberLists, (short) -1);
    // must support extraction for wild cards.
    this.localSubscriptionTrie = new TrieParser(initialSubscriptions * estimatedAvgTopicLength, 1, false, true);
    // this reader is set up for complete text only, all topics are sent in complete.
    this.localSubscriptionTrieReader = new TrieParserReader(2, true);
    this.pendingPublish = new int[subscriberListSize];
    processStartupSubscriptions(hardware.consumeStartupSubscriptions());
}

Example 2 with TrieParserReader

use of com.ociweb.pronghorn.util.TrieParserReader in project PronghornPipes by oci-pronghorn.

the class JSONParseTest method parseJSONLoad.

private void parseJSONLoad(int i, String sourceData, JSONExtractorCompleted extractor) {
    PipeConfig<RawDataSchema> targetDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> targetData = new Pipe<RawDataSchema>(targetDataConfig);
    targetData.initBuffers();
    TrieParserReader reader = new TrieParserReader(5, true);
    JSONStreamParser parser = new JSONStreamParser();
    PipeConfig<RawDataSchema> testInputDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> testInputData = new Pipe<RawDataSchema>(testInputDataConfig);
    testInputData.initBuffers();
    // ///////////////
    // ///////////////
    // ///////////////
    JSONStreamVisitorToChannel visitor = extractor.newJSONVisitor();
    byte[] sourceBytes = sourceData.getBytes();
    while (--i >= 0) {
        // ///////////
        // write JSON data
        // ///////////
        assertTrue("content size " + Pipe.contentRemaining(testInputData), Pipe.contentRemaining(testInputData) == 0);
        int size = Pipe.addMsgIdx(testInputData, 0);
        Pipe.addByteArray(sourceBytes, testInputData);
        Pipe.confirmLowLevelWrite(testInputData, size);
        Pipe.publishWrites(testInputData);
        // //
        // ////////
        // start consuming the data from the pipe
        // call the parser
        // /////
        int msgIdx = Pipe.takeMsgIdx(testInputData);
        TrieParserReader.parseSetup(reader, testInputData);
        // parse data data
        parser.parse(reader, extractor.trieParser(), visitor);
        Pipe.confirmLowLevelRead(testInputData, Pipe.sizeOf(testInputData, msgIdx));
        Pipe.releaseReadLock(testInputData);
        // ///write the captured data into the pipe
        int writeSize = Pipe.addMsgIdx(targetData, 0);
        DataOutputBlobWriter<RawDataSchema> stream = Pipe.openOutputStream(targetData);
        visitor.export(stream, null);
        stream.closeLowLevelField();
        Pipe.confirmLowLevelWrite(targetData, writeSize);
        Pipe.publishWrites(targetData);
        // ///////////////////
        // read the parsed data
        // //////////////////
        RawDataSchema.consume(targetData);
    }
}

Example 3 with TrieParserReader

use of com.ociweb.pronghorn.util.TrieParserReader in project PronghornPipes by oci-pronghorn.

the class JSONParseTest method parseJSON.

private Pipe<RawDataSchema> parseJSON(String sourceData, JSONExtractorCompleted extractor) {
    // ///////////////
    // source test data.
    PipeConfig<RawDataSchema> testInputDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> testInputData = new Pipe<RawDataSchema>(testInputDataConfig);
    testInputData.initBuffers();
    int size = Pipe.addMsgIdx(testInputData, 0);
    Pipe.addUTF8(sourceData, testInputData);
    Pipe.confirmLowLevelWrite(testInputData, size);
    Pipe.publishWrites(testInputData);
    // //
    TrieParserReader reader = new TrieParserReader(5, true);
    // start consuming the data from the pipe
    int msgIdx = Pipe.takeMsgIdx(testInputData);
    TrieParserReader.parseSetup(reader, testInputData);
    // export data to this pipe
    PipeConfig<RawDataSchema> targetDataConfig = RawDataSchema.instance.newPipeConfig(4, 512);
    Pipe<RawDataSchema> targetData = new Pipe<RawDataSchema>(targetDataConfig);
    targetData.initBuffers();
    Pipe.confirmLowLevelRead(testInputData, Pipe.sizeOf(testInputData, msgIdx));
    Pipe.releaseReadLock(testInputData);
    // parse data data
    JSONStreamParser parser = new JSONStreamParser();
    JSONStreamVisitorToChannel visitor = extractor.newJSONVisitor();
    do {
        parser.parse(reader, extractor.trieParser(), visitor);
        // ///write the captured data into the pipe
        Pipe.presumeRoomForWrite(targetData);
        int writeSize = Pipe.addMsgIdx(targetData, 0);
        DataOutputBlobWriter<RawDataSchema> stream = Pipe.openOutputStream(targetData);
        visitor.export(stream, null);
        stream.closeLowLevelField();
        Pipe.confirmLowLevelWrite(targetData, writeSize);
        Pipe.publishWrites(targetData);
    } while (visitor.isReady() && TrieParserReader.parseHasContent(reader));
    return targetData;
}

Example 4 with TrieParserReader

use of com.ociweb.pronghorn.util.TrieParserReader in project FogLight-Examples by oci-pronghorn.

the class ValveDataParserStage method startup.

@Override
public void startup() {
    trie = buildParser();
    reader = new TrieParserReader(1);
// logger.info("started up");
}

Example 5 with TrieParserReader

use of com.ociweb.pronghorn.util.TrieParserReader in project FogLight-Examples by oci-pronghorn.

the class UARTDataParserTest method coreParserTest.

@Test
public void coreParserTest() {
    TrieParser trie = ValveDataParserStage.buildParser();
    // System.out.println(trie);
    String example = "[st1sn100100pn\"NX-DCV-SM-BLU-1-1-VO-L1-SO-OO\"lr-100cc184587lf0pf\"L\"vf0sp80]";
    byte[] exampleBytes = example.getBytes();
    TrieParserReader reader = new TrieParserReader(10);
    TrieParserReader.parseSetup(reader, exampleBytes, 0, exampleBytes.length, Integer.MAX_VALUE);
    int actualStartId = (int) TrieParserReader.parseNext(reader, trie);
    assertEquals(DATA_START, actualStartId);
    assertEquals(1, TrieParserReader.capturedLongField(reader, 0));
    assertEquals(ValveSchema.MSG_VALVESERIALNUMBER_311, (int) TrieParserReader.parseNext(reader, trie));
    assertEquals(100100, TrieParserReader.capturedLongField(reader, 0));
    assertEquals(ValveSchema.MSG_PARTNUMBER_330, (int) TrieParserReader.parseNext(reader, trie));
    StringBuilder target = new StringBuilder();
    TrieParserReader.capturedFieldBytesAsUTF8(reader, 0, target);
    assertEquals("NX-DCV-SM-BLU-1-1-VO-L1-SO-OO", target.toString());
    assertEquals(ValveSchema.MSG_RESIDUALOFDYNAMICANALYSIS_318, (int) TrieParserReader.parseNext(reader, trie));
    assertEquals(-100, TrieParserReader.capturedLongField(reader, 0));
    assertEquals(ValveSchema.MSG_LIFECYCLECOUNT_312, (int) TrieParserReader.parseNext(reader, trie));
    assertEquals(184587, TrieParserReader.capturedLongField(reader, 0));
    assertEquals(ValveSchema.MSG_LEAKFAULT_360, (int) TrieParserReader.parseNext(reader, trie));
    assertEquals(ValveSchema.MSG_PRESSUREFAULT_350, (int) TrieParserReader.parseNext(reader, trie));
    assertEquals(ValveSchema.MSG_VALVEFAULT_340, (int) TrieParserReader.parseNext(reader, trie));
    assertEquals(ValveSchema.MSG_SUPPLYPRESSURE_313, (int) TrieParserReader.parseNext(reader, trie));
    assertEquals(80, TrieParserReader.capturedLongField(reader, 0));
    assertEquals(DATA_END, (int) TrieParserReader.parseNext(reader, trie));
    assertEquals(-1, (int) TrieParserReader.parseNext(reader, trie));
}