Examples with RawDataSchema com.ociweb.pronghorn.pipe.RawDataSchema used on opensource projects

Example 16 with RawDataSchema

use of com.ociweb.pronghorn.pipe.RawDataSchema in project PronghornPipes by oci-pronghorn.

the class TrieParserTest method testparseGather.

@Test
public void testparseGather() {
    TrieParserReader reader = new TrieParserReader(3);
    TrieParser map = new TrieParser(16);
    map.setUTF8Value("12%b12", 33);
    CharSequence test = "12abcd12";
    TrieParserReader.parseSetup(reader, "12abcd12".getBytes(), 0, 8, 7);
    long val = reader.query(map, test);
    Pipe<RawDataSchema> pipe = RawDataSchema.instance.newPipe(2, 64);
    pipe.initBuffers();
    int size = Pipe.addMsgIdx(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1);
    DataOutputBlobWriter x = Pipe.outputStream(pipe);
    DataOutputBlobWriter.openField(x);
    // so will give 5. 5 bytes
    reader.parseGather(reader, x, (byte) 'd');
    // until d is hit. if i put
    // 'c' will return length of
    // 4.
    x.closeLowLevelField();
    Pipe.confirmLowLevelWrite(pipe, size);
    Pipe.publishWrites(pipe);
    // /////////
    int msg = Pipe.takeMsgIdx(pipe);
    assertEquals(RawDataSchema.MSG_CHUNKEDSTREAM_1, msg);
    DataInputBlobReader y = Pipe.inputStream(pipe);
    y.openLowLevelAPIField();
    StringBuilder str = new StringBuilder();
    y.readUTFOfLength(y.available(), str);
    // dont understand what this parseGather() is supposed to do.
    reader.parseGather(reader, (byte) 'd');
    // just changes sourcePos? but this shouldne be 22 if my sixe of source
    // array is only like 8
    assertEquals(str.length(), 5);
    assertEquals(str.toString(), "12abc");
    assertEquals(22, reader.sourcePos);
}

Example 17 with RawDataSchema

use of com.ociweb.pronghorn.pipe.RawDataSchema in project PronghornPipes by oci-pronghorn.

the class TrieParserTest method testwriteCapturedValuesToDataOutput.

@Test
public void testwriteCapturedValuesToDataOutput() throws IOException {
    TrieParserReader reader = new TrieParserReader(3);
    TrieParser map = new TrieParser(16);
    // map.setValue(wrapping(dataBytesExtractStart,4), 0,
    // dataBytesExtractStart.length, 15, value2);
    map.setUTF8Value("%b1234", 33);
    CharSequence test1 = "abcd1234";
    reader.query(map, test1);
    Pipe<RawDataSchema> pipe = RawDataSchema.instance.newPipe(2, 64);
    pipe.initBuffers();
    int size = Pipe.addMsgIdx(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1);
    DataOutputBlobWriter x = Pipe.outputStream(pipe);
    DataOutputBlobWriter.openField(x);
    // write something
    // ->
    int numCapturedBytes = TrieParserReader.writeCapturedValuesToDataOutput(reader, x);
    // should
    // equal
    // 4
    // from
    // above
    // ex.
    x.closeLowLevelField();
    Pipe.confirmLowLevelWrite(pipe, size);
    Pipe.publishWrites(pipe);
    // /////////
    int msg = Pipe.takeMsgIdx(pipe);
    assertEquals(RawDataSchema.MSG_CHUNKEDSTREAM_1, msg);
    // String value = Pipe.takeUTF8(pipe); //something with UTF in method
    // name
    DataInputBlobReader y = Pipe.inputStream(pipe);
    y.openLowLevelAPIField();
    StringBuilder str = new StringBuilder();
    y.readUTFOfLength(y.available(), str);
    // assert("safdsfasdf", value);
    Pipe.confirmLowLevelRead(pipe, size);
    Pipe.releaseReadLock(pipe);
    assertEquals(numCapturedBytes, 4);
    // test to cover unsigned decimal value.
    reader = new TrieParserReader(3);
    map = new TrieParser(16);
    map.setUTF8Value("%i%.", 33);
    String test2 = "3.75";
    reader.query(map, test2);
    pipe = RawDataSchema.instance.newPipe(2, 64);
    pipe.initBuffers();
    size = Pipe.addMsgIdx(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1);
    x = Pipe.outputStream(pipe);
    DataOutputBlobWriter.openField(x);
    // write something
    // ->
    numCapturedBytes = TrieParserReader.writeCapturedValuesToDataOutput(reader, x);
    // should
    // equal
    // 4
    // from
    // above
    // ex.
    x.closeLowLevelField();
    Pipe.confirmLowLevelWrite(pipe, size);
    Pipe.publishWrites(pipe);
    // /////////
    msg = Pipe.takeMsgIdx(pipe);
    assertEquals(RawDataSchema.MSG_CHUNKEDSTREAM_1, msg);
    y = Pipe.inputStream(pipe);
    y.openLowLevelAPIField();
    // this will return 375 for "3.75
    long l1 = y.readPackedLong();
    // this will give us a -2, to tell us where the
    int b1 = y.read();
    // decimal should go.
    Pipe.confirmLowLevelRead(pipe, size);
    Pipe.releaseReadLock(pipe);
    assertEquals(l1, 375);
    assertEquals(b1, -2);
}

Example 18 with RawDataSchema

use of com.ociweb.pronghorn.pipe.RawDataSchema in project PronghornPipes by oci-pronghorn.

the class TrieParserTest method testparseCopy.

@Test
public void testparseCopy() {
    TrieParserReader reader = new TrieParserReader(3);
    TrieParser map = new TrieParser(16);
    map.setUTF8Value("12%b12", 33);
    CharSequence test = "12abcd12";
    // TrieParserReader that, byte[] source, int offset, int length, int
    // mask
    TrieParserReader.parseSetup(reader, "12%b12".getBytes(), 0, 6, 7);
    assertEquals(6, reader.sourceLen);
    Pipe<RawDataSchema> pipe = RawDataSchema.instance.newPipe(2, 64);
    pipe.initBuffers();
    int size = Pipe.addMsgIdx(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1);
    DataOutputBlobWriter x = Pipe.outputStream(pipe);
    DataOutputBlobWriter.openField(x);
    // will copy the
    int val = TrieParserReader.parseCopy(reader, 6, x);
    // value in mapping
    // to
    // dataoutputblobwriter
    x.closeLowLevelField();
    Pipe.confirmLowLevelWrite(pipe, size);
    Pipe.publishWrites(pipe);
    // /////////
    int msg = Pipe.takeMsgIdx(pipe);
    assertEquals(RawDataSchema.MSG_CHUNKEDSTREAM_1, msg);
    DataInputBlobReader y = Pipe.inputStream(pipe);
    y.openLowLevelAPIField();
    StringBuilder str = new StringBuilder();
    y.readUTFOfLength(y.available(), str);
    assertEquals(str.toString(), "12%b12");
    // asserting length returned by parseCopy is
    assertEquals(val, 6);
// length given. would not be the case if
// sourcelength was greater.
}

Example 19 with RawDataSchema

use of com.ociweb.pronghorn.pipe.RawDataSchema in project PronghornPipes by oci-pronghorn.

the class TrieParserTest method testwriteCapturedShort.

@Test
public void testwriteCapturedShort() {
    TrieParserReader reader = new TrieParserReader(3);
    TrieParser map = new TrieParser(16);
    map.setUTF8Value("12%iabcd", 33);
    CharSequence test = "128abcd";
    // query holds most recent thing
    reader.query(map, test);
    Pipe<RawDataSchema> pipe = RawDataSchema.instance.newPipe(2, 64);
    pipe.initBuffers();
    int size = Pipe.addMsgIdx(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1);
    DataOutputBlobWriter x = Pipe.outputStream(pipe);
    DataOutputBlobWriter.openField(x);
    TrieParserReader.writeCapturedShort(reader, 0, x);
    x.closeLowLevelField();
    Pipe.confirmLowLevelWrite(pipe, size);
    Pipe.publishWrites(pipe);
    // /////////
    int msg = Pipe.takeMsgIdx(pipe);
    assertEquals(RawDataSchema.MSG_CHUNKEDSTREAM_1, msg);
    DataInputBlobReader y = Pipe.inputStream(pipe);
    y.openLowLevelAPIField();
    // 2nd value has length of captured field.
    byte[] vals = new byte[2];
    y.read(vals);
    assertEquals(vals[1], 8);
}