Examples with DataOutputBlobWriter com.ociweb.pronghorn.pipe.DataOutputBlobWriter used on opensource projects

Example 6 with DataOutputBlobWriter

use of com.ociweb.pronghorn.pipe.DataOutputBlobWriter in project GreenLightning by oci-pronghorn.

the class HTTPClientRequestTrafficStage method processMessagesForPipe.

@Override
protected void processMessagesForPipe(int activePipe) {
    Pipe<ClientHTTPRequestSchema> requestPipe = input[activePipe];
    while (PipeReader.hasContentToRead(requestPipe) && hasReleaseCountRemaining(activePipe) && isChannelUnBlocked(activePipe) && hasOpenConnection(requestPipe, output, ccm, activePipe) && PipeReader.tryReadFragment(requestPipe)) {
        int msgIdx = PipeReader.getMsgIdx(requestPipe);
        switch(msgIdx) {
            case ClientHTTPRequestSchema.MSG_FASTHTTPGET_200:
                {
                    final byte[] hostBack = Pipe.blob(requestPipe);
                    final int hostPos = PipeReader.readBytesPosition(requestPipe, ClientHTTPRequestSchema.MSG_FASTHTTPGET_200_FIELD_HOST_2);
                    final int hostLen = PipeReader.readBytesLength(requestPipe, ClientHTTPRequestSchema.MSG_FASTHTTPGET_200_FIELD_HOST_2);
                    final int hostMask = Pipe.blobMask(requestPipe);
                    int routeId = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_FASTHTTPGET_200_FIELD_DESTINATION_11);
                    int port = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_FASTHTTPGET_200_FIELD_PORT_1);
                    int userId = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_FASTHTTPGET_200_FIELD_SESSION_10);
                    long connectionId = PipeReader.readLong(requestPipe, ClientHTTPRequestSchema.MSG_FASTHTTPGET_200_FIELD_CONNECTIONID_20);
                    ClientConnection clientConnection;
                    if (-1 != connectionId && null != (clientConnection = (ClientConnection) ccm.connectionForSessionId(connectionId))) {
                        assert (clientConnection.singleUsage(stageId)) : "Only a single Stage may update the clientConnection.";
                        assert (routeId >= 0);
                        clientConnection.recordDestinationRouteId(routeId);
                        publishGet(requestPipe, hostBack, hostPos, hostLen, connectionId, clientConnection, ClientHTTPRequestSchema.MSG_FASTHTTPGET_200_FIELD_PATH_3, ClientHTTPRequestSchema.MSG_FASTHTTPGET_200_FIELD_HEADERS_7);
                    }
                }
                break;
            case ClientHTTPRequestSchema.MSG_HTTPGET_100:
                {
                    final byte[] hostBack = Pipe.blob(requestPipe);
                    final int hostPos = PipeReader.readBytesPosition(requestPipe, ClientHTTPRequestSchema.MSG_HTTPGET_100_FIELD_HOST_2);
                    final int hostLen = PipeReader.readBytesLength(requestPipe, ClientHTTPRequestSchema.MSG_HTTPGET_100_FIELD_HOST_2);
                    final int hostMask = Pipe.blobMask(requestPipe);
                    int routeId = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_HTTPGET_100_FIELD_DESTINATION_11);
                    int port = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_HTTPGET_100_FIELD_PORT_1);
                    int userId = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_HTTPGET_100_FIELD_SESSION_10);
                    long connectionId = ccm.lookup(ccm.lookupHostId(hostBack, hostPos, hostLen, hostMask), port, userId);
                    ClientConnection clientConnection;
                    if (-1 != connectionId && null != (clientConnection = (ClientConnection) ccm.connectionForSessionId(connectionId))) {
                        assert (clientConnection.singleUsage(stageId)) : "Only a single Stage may update the clientConnection.";
                        assert (routeId >= 0);
                        clientConnection.recordDestinationRouteId(routeId);
                        publishGet(requestPipe, hostBack, hostPos, hostLen, connectionId, clientConnection, ClientHTTPRequestSchema.MSG_HTTPGET_100_FIELD_PATH_3, ClientHTTPRequestSchema.MSG_HTTPGET_100_FIELD_HEADERS_7);
                    }
                }
                break;
            case ClientHTTPRequestSchema.MSG_HTTPPOST_101:
                {
                    final byte[] hostBack = Pipe.blob(requestPipe);
                    final int hostPos = PipeReader.readBytesPosition(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_HOST_2);
                    final int hostLen = PipeReader.readBytesLength(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_HOST_2);
                    final int hostMask = Pipe.blobMask(requestPipe);
                    int routeId = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_DESTINATION_11);
                    int userId = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_SESSION_10);
                    int port = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_PORT_1);
                    long connectionId = ccm.lookup(ccm.lookupHostId(hostBack, hostPos, hostLen, hostMask), port, userId);
                    // openConnection(activeHost, port, userId, outIdx);
                    ClientConnection clientConnection;
                    if ((-1 != connectionId) && (null != (clientConnection = (ClientConnection) ccm.connectionForSessionId(connectionId)))) {
                        assert (routeId >= 0);
                        clientConnection.recordDestinationRouteId(routeId);
                        int outIdx = clientConnection.requestPipeLineIdx();
                        // count of messages can only be done here.
                        clientConnection.incRequestsSent();
                        Pipe<NetPayloadSchema> outputPipe = output[outIdx];
                        PipeWriter.presumeWriteFragment(outputPipe, NetPayloadSchema.MSG_PLAIN_210);
                        PipeWriter.writeLong(outputPipe, NetPayloadSchema.MSG_PLAIN_210_FIELD_CONNECTIONID_201, connectionId);
                        DataOutputBlobWriter<NetPayloadSchema> activeWriter = PipeWriter.outputStream(outputPipe);
                        DataOutputBlobWriter.openField(activeWriter);
                        DataOutputBlobWriter.encodeAsUTF8(activeWriter, "POST");
                        int len = PipeReader.readBytesLength(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_PATH_3);
                        int first = PipeReader.readBytesPosition(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_PATH_3);
                        boolean prePendSlash = (0 == len) || ('/' != PipeReader.readBytesBackingArray(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_PATH_3)[first & Pipe.blobMask(requestPipe)]);
                        if (prePendSlash) {
                            // NOTE: these can be pre-coverted to bytes so we need not convert on each write. may want to improve.
                            DataOutputBlobWriter.encodeAsUTF8(activeWriter, " /");
                        } else {
                            DataOutputBlobWriter.encodeAsUTF8(activeWriter, " ");
                        }
                        // Reading from UTF8 field and writing to UTF8 encoded field so we are doing a direct copy here.
                        PipeReader.readBytes(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_PATH_3, activeWriter);
                        HeaderUtil.writeHeaderBeginning(hostBack, hostPos, hostLen, Pipe.blobMask(requestPipe), activeWriter);
                        HeaderUtil.writeHeaderMiddle(activeWriter, implementationVersion);
                        PipeReader.readBytes(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_HEADERS_7, activeWriter);
                        long postLength = PipeReader.readBytesLength(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_PAYLOAD_5);
                        HeaderUtil.writeHeaderEnding(activeWriter, true, postLength);
                        // TODO: How is chunking supported, that code is not here yet but length must be -1 I think.
                        PipeReader.readBytes(requestPipe, ClientHTTPRequestSchema.MSG_HTTPPOST_101_FIELD_PAYLOAD_5, activeWriter);
                        DataOutputBlobWriter.closeHighLevelField(activeWriter, NetPayloadSchema.MSG_PLAIN_210_FIELD_PAYLOAD_204);
                        PipeWriter.publishWrites(outputPipe);
                    }
                }
                break;
            case ClientHTTPRequestSchema.MSG_CLOSE_104:
                final byte[] hostBack = Pipe.blob(requestPipe);
                final int hostPos = PipeReader.readBytesPosition(requestPipe, ClientHTTPRequestSchema.MSG_CLOSE_104_FIELD_HOST_2);
                final int hostLen = PipeReader.readBytesLength(requestPipe, ClientHTTPRequestSchema.MSG_CLOSE_104_FIELD_HOST_2);
                final int hostMask = Pipe.blobMask(requestPipe);
                final int port = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_CLOSE_104_FIELD_PORT_1);
                final int userId = PipeReader.readInt(requestPipe, ClientHTTPRequestSchema.MSG_CLOSE_104_FIELD_SESSION_10);
                long connectionId = ccm.lookup(ccm.lookupHostId(hostBack, hostPos, hostLen, hostMask), port, userId);
                // only close if we find a live connection
                if ((-1 != connectionId)) {
                    ClientConnection connectionToKill = (ClientConnection) ccm.connectionForSessionId(connectionId);
                    if (null != connectionToKill) {
                        Pipe<NetPayloadSchema> outputPipe = output[connectionToKill.requestPipeLineIdx()];
                        // do not close that will be done by last stage
                        // must be done first before we send the message
                        connectionToKill.beginDisconnect();
                        PipeWriter.presumeWriteFragment(outputPipe, NetPayloadSchema.MSG_DISCONNECT_203);
                        PipeWriter.writeLong(outputPipe, NetPayloadSchema.MSG_DISCONNECT_203_FIELD_CONNECTIONID_201, connectionId);
                        PipeWriter.publishWrites(outputPipe);
                    }
                }
                break;
            default:
                logger.info("not yet supporting message {}", msgIdx);
        }
        PipeReader.releaseReadLock(requestPipe);
        // only do now after we know its not blocked and was completed
        decReleaseCount(activePipe);
    }
}

Example 7 with DataOutputBlobWriter

use of com.ociweb.pronghorn.pipe.DataOutputBlobWriter in project GreenLightning by oci-pronghorn.

the class GreenParserTest method simpleReaderTest.

@Test
public void simpleReaderTest() {
    GreenReader gr = new GreenTokenMap().add(1, "moe").add(2, "larry").add(3, "curly").add(0, "shemp").newReader();
    ChannelReader testToRead = BlobReaderFactory.generateExtractionDataToTest(new MyConsumer<DataOutputBlobWriter<?>>() {

        @Override
        public void accept(DataOutputBlobWriter<?> dataOutputBlobWriter) {
            larryShempStreamAppend(dataOutputBlobWriter);
        }
    });
    // /////
    // example consumer code starts here
    // /////
    boolean foundLarry = false;
    boolean foundShemp = false;
    gr.beginRead(testToRead);
    while (gr.hasMore()) {
        long token = gr.readToken();
        switch((int) token) {
            case // this is a token id
            0:
                foundShemp = true;
                break;
            case // this is a token id
            2:
                foundLarry = true;
                break;
            case -1:
                // unknown
                gr.skipByte();
                break;
            case 1:
            case 3:
                // ignore
                break;
        }
    }
    assertTrue("can not find larry", foundLarry);
    assertTrue("can not find shemp", foundShemp);
}

Example 8 with DataOutputBlobWriter

use of com.ociweb.pronghorn.pipe.DataOutputBlobWriter in project PronghornPipes by oci-pronghorn.

the class TrieParserTest method testparseGather.

@Test
public void testparseGather() {
    TrieParserReader reader = new TrieParserReader(3);
    TrieParser map = new TrieParser(16);
    map.setUTF8Value("12%b12", 33);
    CharSequence test = "12abcd12";
    TrieParserReader.parseSetup(reader, "12abcd12".getBytes(), 0, 8, 7);
    long val = reader.query(map, test);
    Pipe<RawDataSchema> pipe = RawDataSchema.instance.newPipe(2, 64);
    pipe.initBuffers();
    int size = Pipe.addMsgIdx(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1);
    DataOutputBlobWriter x = Pipe.outputStream(pipe);
    DataOutputBlobWriter.openField(x);
    // so will give 5. 5 bytes
    reader.parseGather(reader, x, (byte) 'd');
    // until d is hit. if i put
    // 'c' will return length of
    // 4.
    x.closeLowLevelField();
    Pipe.confirmLowLevelWrite(pipe, size);
    Pipe.publishWrites(pipe);
    // /////////
    int msg = Pipe.takeMsgIdx(pipe);
    assertEquals(RawDataSchema.MSG_CHUNKEDSTREAM_1, msg);
    DataInputBlobReader y = Pipe.inputStream(pipe);
    y.openLowLevelAPIField();
    StringBuilder str = new StringBuilder();
    y.readUTFOfLength(y.available(), str);
    // dont understand what this parseGather() is supposed to do.
    reader.parseGather(reader, (byte) 'd');
    // just changes sourcePos? but this shouldne be 22 if my sixe of source
    // array is only like 8
    assertEquals(str.length(), 5);
    assertEquals(str.toString(), "12abc");
    assertEquals(22, reader.sourcePos);
}

Example 9 with DataOutputBlobWriter

use of com.ociweb.pronghorn.pipe.DataOutputBlobWriter in project PronghornPipes by oci-pronghorn.

the class TrieParserTest method testwriteCapturedValuesToDataOutput.

@Test
public void testwriteCapturedValuesToDataOutput() throws IOException {
    TrieParserReader reader = new TrieParserReader(3);
    TrieParser map = new TrieParser(16);
    // map.setValue(wrapping(dataBytesExtractStart,4), 0,
    // dataBytesExtractStart.length, 15, value2);
    map.setUTF8Value("%b1234", 33);
    CharSequence test1 = "abcd1234";
    reader.query(map, test1);
    Pipe<RawDataSchema> pipe = RawDataSchema.instance.newPipe(2, 64);
    pipe.initBuffers();
    int size = Pipe.addMsgIdx(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1);
    DataOutputBlobWriter x = Pipe.outputStream(pipe);
    DataOutputBlobWriter.openField(x);
    // write something
    // ->
    int numCapturedBytes = TrieParserReader.writeCapturedValuesToDataOutput(reader, x);
    // should
    // equal
    // 4
    // from
    // above
    // ex.
    x.closeLowLevelField();
    Pipe.confirmLowLevelWrite(pipe, size);
    Pipe.publishWrites(pipe);
    // /////////
    int msg = Pipe.takeMsgIdx(pipe);
    assertEquals(RawDataSchema.MSG_CHUNKEDSTREAM_1, msg);
    // String value = Pipe.takeUTF8(pipe); //something with UTF in method
    // name
    DataInputBlobReader y = Pipe.inputStream(pipe);
    y.openLowLevelAPIField();
    StringBuilder str = new StringBuilder();
    y.readUTFOfLength(y.available(), str);
    // assert("safdsfasdf", value);
    Pipe.confirmLowLevelRead(pipe, size);
    Pipe.releaseReadLock(pipe);
    assertEquals(numCapturedBytes, 4);
    // test to cover unsigned decimal value.
    reader = new TrieParserReader(3);
    map = new TrieParser(16);
    map.setUTF8Value("%i%.", 33);
    String test2 = "3.75";
    reader.query(map, test2);
    pipe = RawDataSchema.instance.newPipe(2, 64);
    pipe.initBuffers();
    size = Pipe.addMsgIdx(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1);
    x = Pipe.outputStream(pipe);
    DataOutputBlobWriter.openField(x);
    // write something
    // ->
    numCapturedBytes = TrieParserReader.writeCapturedValuesToDataOutput(reader, x);
    // should
    // equal
    // 4
    // from
    // above
    // ex.
    x.closeLowLevelField();
    Pipe.confirmLowLevelWrite(pipe, size);
    Pipe.publishWrites(pipe);
    // /////////
    msg = Pipe.takeMsgIdx(pipe);
    assertEquals(RawDataSchema.MSG_CHUNKEDSTREAM_1, msg);
    y = Pipe.inputStream(pipe);
    y.openLowLevelAPIField();
    // this will return 375 for "3.75
    long l1 = y.readPackedLong();
    // this will give us a -2, to tell us where the
    int b1 = y.read();
    // decimal should go.
    Pipe.confirmLowLevelRead(pipe, size);
    Pipe.releaseReadLock(pipe);
    assertEquals(l1, 375);
    assertEquals(b1, -2);
}

Example 10 with DataOutputBlobWriter

use of com.ociweb.pronghorn.pipe.DataOutputBlobWriter in project PronghornPipes by oci-pronghorn.

the class TrieParserTest method testparseCopy.

@Test
public void testparseCopy() {
    TrieParserReader reader = new TrieParserReader(3);
    TrieParser map = new TrieParser(16);
    map.setUTF8Value("12%b12", 33);
    CharSequence test = "12abcd12";
    // TrieParserReader that, byte[] source, int offset, int length, int
    // mask
    TrieParserReader.parseSetup(reader, "12%b12".getBytes(), 0, 6, 7);
    assertEquals(6, reader.sourceLen);
    Pipe<RawDataSchema> pipe = RawDataSchema.instance.newPipe(2, 64);
    pipe.initBuffers();
    int size = Pipe.addMsgIdx(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1);
    DataOutputBlobWriter x = Pipe.outputStream(pipe);
    DataOutputBlobWriter.openField(x);
    // will copy the
    int val = TrieParserReader.parseCopy(reader, 6, x);
    // value in mapping
    // to
    // dataoutputblobwriter
    x.closeLowLevelField();
    Pipe.confirmLowLevelWrite(pipe, size);
    Pipe.publishWrites(pipe);
    // /////////
    int msg = Pipe.takeMsgIdx(pipe);
    assertEquals(RawDataSchema.MSG_CHUNKEDSTREAM_1, msg);
    DataInputBlobReader y = Pipe.inputStream(pipe);
    y.openLowLevelAPIField();
    StringBuilder str = new StringBuilder();
    y.readUTFOfLength(y.available(), str);
    assertEquals(str.toString(), "12%b12");
    // asserting length returned by parseCopy is
    assertEquals(val, 6);
// length given. would not be the case if
// sourcelength was greater.
}