Examples with TransportException com.nextdoor.bender.ipc.TransportException used on opensource projects

Example 11 with TransportException

use of com.nextdoor.bender.ipc.TransportException in project bender by Nextdoor.

the class SNSS3HandlerTest method testExceptionHandlingd.

@Test
public void testExceptionHandlingd() throws Throwable {
    BaseHandler.CONFIG_FILE = "/com/nextdoor/bender/handler/config_test_sns.json";
    TestContext ctx = new TestContext();
    ctx.setFunctionName("unittest");
    ctx.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test-function:staging");
    /*
     * Invoke handler
     */
    SNSS3Handler fhandler = (SNSS3Handler) getHandler();
    fhandler.init(ctx);
    IpcSenderService ipcSpy = spy(fhandler.getIpcService());
    doThrow(new TransportException("expected")).when(ipcSpy).shutdown();
    fhandler.setIpcService(ipcSpy);
    AmazonSNSClient mockClient = mock(AmazonSNSClient.class);
    AmazonSNSClientFactory mockClientFactory = mock(AmazonSNSClientFactory.class);
    doReturn(mockClient).when(mockClientFactory).newInstance();
    fhandler.snsClientFactory = mockClientFactory;
    SNSEvent event = getTestEvent();
    try {
        fhandler.handler(event, ctx);
    } catch (Exception e) {
    }
    verify(mockClient, times(1)).publish("foo", "basic_input.log", "SNSS3Handler Failed");
}

Example 12 with TransportException

use of com.nextdoor.bender.ipc.TransportException in project bender by Nextdoor.

the class S3Transport method sendBatch.

@Override
public void sendBatch(TransportBuffer buffer, LinkedHashMap<String, String> partitions, Context context) throws TransportException {
    S3TransportBuffer buf = (S3TransportBuffer) buffer;
    /*
     * Create s3 key (filepath + filename)
     */
    LinkedHashMap<String, String> parts = new LinkedHashMap<String, String>(partitions);
    String filename = parts.remove(FILENAME_KEY);
    if (filename == null) {
        filename = context.getAwsRequestId();
    }
    String key = parts.entrySet().stream().map(s -> s.getKey() + "=" + s.getValue()).collect(Collectors.joining("/"));
    key = (key.equals("") ? filename : key + '/' + filename);
    if (this.basePath.endsWith("/")) {
        key = this.basePath + key;
    } else {
        key = this.basePath + '/' + key;
    }
    // TODO: make this dynamic
    if (key.endsWith(".gz")) {
        key = key.substring(0, key.length() - 3);
    }
    /*
     * Add or strip out compression format extension
     *
     * TODO: get this based on the compression codec
     */
    if (this.compress || buf.isCompressed()) {
        key += ".bz2";
    }
    ByteArrayOutputStream os = buf.getInternalBuffer();
    /*
     * Compress stream if needed. Don't compress a compressed stream.
     */
    ByteArrayOutputStream payload;
    if (this.compress && !buf.isCompressed()) {
        payload = compress(os);
    } else {
        payload = os;
    }
    /*
     * For memory efficiency convert the output stream into an InputStream. This is done using the
     * easystream library but under the hood it uses piped streams to facilitate this process. This
     * avoids copying the entire contents of the OutputStream to populate the InputStream. Note that
     * this process creates another thread to consume from the InputStream.
     */
    final String s3Key = key;
    /*
     * Write to OutputStream
     */
    final InputStreamFromOutputStream<String> isos = new InputStreamFromOutputStream<String>() {

        public String produce(final OutputStream dataSink) throws Exception {
            /*
         * Note this is executed in a different thread
         */
            payload.writeTo(dataSink);
            return null;
        }
    };
    /*
     * Consume InputStream
     */
    try {
        sendStream(isos, s3Key, payload.size());
    } finally {
        try {
            isos.close();
        } catch (IOException e) {
            throw new TransportException(e);
        } finally {
            buf.close();
        }
    }
}

Example 13 with TransportException

use of com.nextdoor.bender.ipc.TransportException in project bender by Nextdoor.

the class S3Transport method sendStream.

protected void sendStream(InputStream input, String key, long streamSize) throws TransportException {
    /*
     * Create metadata
     */
    ObjectMetadata metadata = new ObjectMetadata();
    /*
     * Find if a multipart upload has already begun or start a new one.
     */
    MultiPartUpload upload;
    synchronized (multiPartUploads) {
        if (!multiPartUploads.containsKey(key)) {
            InitiateMultipartUploadRequest uploadRequest = new InitiateMultipartUploadRequest(bucketName, key);
            uploadRequest.setObjectMetadata(metadata);
            InitiateMultipartUploadResult res = client.initiateMultipartUpload(uploadRequest);
            upload = new MultiPartUpload(bucketName, key, res.getUploadId());
            multiPartUploads.put(key, upload);
        } else {
            upload = multiPartUploads.get(key);
        }
    }
    /*
     * Write out to S3. Note that the S3 client auto closes the input stream.
     */
    UploadPartRequest req = upload.getUploadPartRequest().withInputStream(input).withPartSize(streamSize);
    try {
        UploadPartResult res = client.uploadPart(req);
        upload.addPartETag(res.getPartETag());
    } catch (AmazonClientException e) {
        client.abortMultipartUpload(upload.getAbortMultipartUploadRequest());
        throw new TransportException("unable to put file" + e, e);
    } finally {
        try {
            input.close();
        } catch (IOException e) {
            logger.warn("error encountered while closing input stream", e);
        }
    }
}

Example 14 with TransportException

use of com.nextdoor.bender.ipc.TransportException in project bender by Nextdoor.

the class HttpTransport method sendBatch.

public void sendBatch(byte[] raw) throws TransportException {
    /*
     * Wrap the call with retry logic to avoid intermittent ES issues.
     */
    Callable<HttpResponse> callable = () -> {
        HttpResponse resp;
        String responseString = null;
        HttpPost httpPost = new HttpPost(this.url);
        /*
       * Do the call, read response, release connection so it is available for use again, and
       * finally check the response.
       */
        try {
            if (this.useGzip) {
                resp = sendBatchCompressed(httpPost, raw);
            } else {
                resp = sendBatchUncompressed(httpPost, raw);
            }
            try {
                responseString = EntityUtils.toString(resp.getEntity());
            } catch (ParseException | IOException e) {
                throw new TransportException("http transport call failed because " + resp.getStatusLine().getReasonPhrase());
            }
        } finally {
            /*
         * Always release connection otherwise it blocks future requests.
         */
            httpPost.releaseConnection();
        }
        checkResponse(resp, responseString);
        return resp;
    };
    RetryConfig config = new RetryConfigBuilder().retryOnSpecificExceptions(TransportException.class).withMaxNumberOfTries(this.retries + 1).withDelayBetweenTries(this.retryDelayMs, ChronoUnit.MILLIS).withExponentialBackoff().build();
    try {
        new CallExecutor(config).execute(callable);
    } catch (RetriesExhaustedException ree) {
        logger.warn("transport failed after " + ree.getCallResults().getTotalTries() + " tries.");
        throw new TransportException(ree.getCallResults().getLastExceptionThatCausedRetry());
    } catch (UnexpectedException ue) {
        throw new TransportException(ue);
    }
}

Example 15 with TransportException

use of com.nextdoor.bender.ipc.TransportException in project bender by Nextdoor.

the class TcpTransport method sendBatch.

@Override
public void sendBatch(TransportBuffer buffer) throws TransportException {
    Buffer internalBuffer = ((TcpTransportBuffer) buffer).getInternalBuffer();
    Callable<Boolean> write = () -> {
        try {
            sink.write(internalBuffer, internalBuffer.size());
            return true;
        } catch (IOException ex) {
            throw new TransportException("Error while sending in tcp transport", ex);
        }
    };
    try {
        new CallExecutor(retryConfig).execute(write);
    } catch (RetriesExhaustedException | UnexpectedException ue) {
        throw new TransportException(ue);
    }
}