use of com.nextdoor.bender.ipc.TransportException in project bender by Nextdoor.
the class BaseHandler method processInternal.
/**
* Method called by Handler implementations to process records.
*
* @param context Lambda invocation context.
* @throws HandlerException
*/
private void processInternal(Context context) throws HandlerException {
Stat runtime = new Stat("runtime.ns");
runtime.start();
Source source = this.getSource();
DeserializerProcessor deser = source.getDeserProcessor();
List<OperationProcessor> operations = source.getOperationProcessors();
List<String> containsStrings = source.getContainsStrings();
List<Pattern> regexPatterns = source.getRegexPatterns();
this.getIpcService().setContext(context);
Iterator<InternalEvent> events = this.getInternalEventIterator();
/*
* For logging purposes log when the function started running
*/
this.monitor.invokeTimeNow();
AtomicLong eventCount = new AtomicLong(0);
AtomicLong oldestArrivalTime = new AtomicLong(System.currentTimeMillis());
AtomicLong oldestOccurrenceTime = new AtomicLong(System.currentTimeMillis());
/*
* Process each record
*/
int characteristics = Spliterator.IMMUTABLE;
Spliterator<InternalEvent> spliterator = Spliterators.spliteratorUnknownSize(events, characteristics);
Stream<InternalEvent> input = StreamSupport.stream(spliterator, false);
/*
* Filter out raw events
*/
Stream<InternalEvent> filtered = input.filter(/*
* Perform regex filter
*/
ievent -> {
eventCount.incrementAndGet();
String eventStr = ievent.getEventString();
/*
* Apply String contains filters before deserialization
*/
for (String containsString : containsStrings) {
if (eventStr.contains(containsString)) {
return false;
}
}
/*
* Apply regex patterns before deserialization
*/
for (Pattern regexPattern : regexPatterns) {
Matcher m = regexPattern.matcher(eventStr);
if (m.find()) {
return false;
}
}
return true;
});
/*
* Deserialize
*/
Stream<InternalEvent> deserialized = filtered.map(ievent -> {
DeserializedEvent data = deser.deserialize(ievent.getEventString());
if (data == null || data.getPayload() == null) {
logger.warn("Failed to deserialize: " + ievent.getEventString());
return null;
}
ievent.setEventObj(data);
return ievent;
}).filter(Objects::nonNull);
/*
* Perform Operations
*/
Stream<InternalEvent> operated = deserialized;
for (OperationProcessor operation : operations) {
operated = operation.perform(operated);
}
/*
* Serialize
*/
Stream<InternalEvent> serialized = operated.map(ievent -> {
try {
String raw = null;
raw = this.ser.serialize(this.wrapper.getWrapped(ievent));
ievent.setSerialized(raw);
return ievent;
} catch (SerializationException e) {
return null;
}
}).filter(Objects::nonNull);
/*
* Transport
*/
serialized.forEach(ievent -> {
/*
* Update times
*/
updateOldest(oldestArrivalTime, ievent.getArrivalTime());
updateOldest(oldestOccurrenceTime, ievent.getEventTime());
try {
this.getIpcService().add(ievent);
} catch (TransportException e) {
logger.warn("error adding event", e);
}
});
/*
* Wait for transporters to finish
*/
try {
this.getIpcService().shutdown();
} catch (TransportException e) {
throw new HandlerException("encounted TransportException while shutting down ipcService", e);
} catch (InterruptedException e) {
throw new HandlerException("thread was interruptedwhile shutting down ipcService", e);
} finally {
String evtSource = this.getSourceName();
runtime.stop();
if (!this.skipWriteStats) {
writeStats(eventCount.get(), oldestArrivalTime.get(), oldestOccurrenceTime.get(), evtSource, runtime);
}
if (logger.isTraceEnabled()) {
getGCStats();
}
}
}
use of com.nextdoor.bender.ipc.TransportException in project bender by Nextdoor.
the class HandlerTest method testExceptionHandling.
@Test(expected = TransportException.class)
public void testExceptionHandling() throws Throwable {
TestContext ctx = new TestContext();
ctx.setFunctionName("unittest");
ctx.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test-function:staging");
/*
* Invoke handler
*/
BaseHandler<T> fhandler = (BaseHandler<T>) getHandler();
fhandler.init(ctx);
IpcSenderService ipcSpy = spy(fhandler.getIpcService());
doThrow(new TransportException("expected")).when(ipcSpy).shutdown();
fhandler.setIpcService(ipcSpy);
T event = getTestEvent();
try {
fhandler.handler(event, ctx);
} catch (Exception e) {
throw e.getCause().getCause();
}
}
use of com.nextdoor.bender.ipc.TransportException in project bender by Nextdoor.
the class S3Transport method compress.
protected ByteArrayOutputStream compress(ByteArrayOutputStream raw) throws TransportException {
ByteArrayOutputStream compressed = new ByteArrayOutputStream();
BZip2CompressorOutputStream bcos = null;
try {
bcos = new BZip2CompressorOutputStream(compressed);
} catch (IOException e) {
throw new TransportException("unable to open compressed stream", e);
}
try {
raw.writeTo(bcos);
bcos.flush();
} catch (IOException e) {
throw new TransportException("unable to compress data", e);
} finally {
try {
bcos.close();
} catch (IOException e) {
}
}
return compressed;
}
use of com.nextdoor.bender.ipc.TransportException in project bender by Nextdoor.
the class BaseHandlerTest method testGeneralTransportExceptionOnShutdown.
@Test(expected = TransportException.class)
public void testGeneralTransportExceptionOnShutdown() throws Throwable {
BaseHandler.CONFIG_FILE = "/config/handler_config.json";
List<DummyEvent> events = new ArrayList<DummyEvent>(1);
events.add(new DummyEvent("foo", 0));
TestContext context = new TestContext();
context.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test:tag");
handler.init(context);
IpcSenderService spy = spy(handler.getIpcService());
handler.setIpcService(spy);
doThrow(new TransportException("expected")).when(spy).shutdown();
try {
handler.handler(events, context);
} catch (Exception e) {
throw e.getCause().getCause();
}
}
use of com.nextdoor.bender.ipc.TransportException in project bender by Nextdoor.
the class HttpTransportTest method testGzipErrorsResponse.
@Test(expected = TransportException.class)
public void testGzipErrorsResponse() throws TransportException, IOException {
byte[] respPayload = "gzip resp".getBytes(StandardCharsets.UTF_8);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream os = new GZIPOutputStream(baos);
os.write(respPayload);
os.close();
byte[] compressedResponse = baos.toByteArray();
HttpClient client = getMockClientWithResponse(compressedResponse, ContentType.DEFAULT_BINARY, HttpStatus.SC_INTERNAL_SERVER_ERROR, true);
HttpTransport transport = new HttpTransport(client, "", true, 1, 1);
try {
transport.sendBatch("foo".getBytes());
} catch (Exception e) {
assertEquals("http transport call failed because \"expected failure\" payload response \"gzip resp\"", e.getCause().getMessage());
throw e;
}
}
Aggregations