use of com.amazonaws.services.lambda.runtime.Context in project bender by Nextdoor.
the class BaseHandler method processInternal.
/**
* Method called by Handler implementations to process records.
*
* @param context Lambda invocation context.
* @throws HandlerException
*/
private void processInternal(Context context) throws HandlerException {
Stat runtime = new Stat("runtime.ns");
runtime.start();
Source source = this.getSource();
DeserializerProcessor deser = source.getDeserProcessor();
List<OperationProcessor> operations = source.getOperationProcessors();
List<String> containsStrings = source.getContainsStrings();
List<Pattern> regexPatterns = source.getRegexPatterns();
this.getIpcService().setContext(context);
Iterator<InternalEvent> events = this.getInternalEventIterator();
/*
* For logging purposes log when the function started running
*/
this.monitor.invokeTimeNow();
AtomicLong eventCount = new AtomicLong(0);
AtomicLong oldestArrivalTime = new AtomicLong(System.currentTimeMillis());
AtomicLong oldestOccurrenceTime = new AtomicLong(System.currentTimeMillis());
/*
* Process each record
*/
int characteristics = Spliterator.IMMUTABLE;
Spliterator<InternalEvent> spliterator = Spliterators.spliteratorUnknownSize(events, characteristics);
Stream<InternalEvent> input = StreamSupport.stream(spliterator, false);
/*
* Filter out raw events
*/
Stream<InternalEvent> filtered = input.filter(/*
* Perform regex filter
*/
ievent -> {
eventCount.incrementAndGet();
String eventStr = ievent.getEventString();
/*
* Apply String contains filters before deserialization
*/
for (String containsString : containsStrings) {
if (eventStr.contains(containsString)) {
return false;
}
}
/*
* Apply regex patterns before deserialization
*/
for (Pattern regexPattern : regexPatterns) {
Matcher m = regexPattern.matcher(eventStr);
if (m.find()) {
return false;
}
}
return true;
});
/*
* Deserialize
*/
Stream<InternalEvent> deserialized = filtered.map(ievent -> {
DeserializedEvent data = deser.deserialize(ievent.getEventString());
if (data == null || data.getPayload() == null) {
logger.warn("Failed to deserialize: " + ievent.getEventString());
return null;
}
ievent.setEventObj(data);
return ievent;
}).filter(Objects::nonNull);
/*
* Perform Operations
*/
Stream<InternalEvent> operated = deserialized;
for (OperationProcessor operation : operations) {
operated = operation.perform(operated);
}
/*
* Serialize
*/
Stream<InternalEvent> serialized = operated.map(ievent -> {
try {
String raw = null;
raw = this.ser.serialize(this.wrapper.getWrapped(ievent));
ievent.setSerialized(raw);
return ievent;
} catch (SerializationException e) {
return null;
}
}).filter(Objects::nonNull);
/*
* Transport
*/
serialized.forEach(ievent -> {
/*
* Update times
*/
updateOldest(oldestArrivalTime, ievent.getArrivalTime());
updateOldest(oldestOccurrenceTime, ievent.getEventTime());
try {
this.getIpcService().add(ievent);
} catch (TransportException e) {
logger.warn("error adding event", e);
}
});
/*
* Wait for transporters to finish
*/
try {
this.getIpcService().shutdown();
} catch (TransportException e) {
throw new HandlerException("encounted TransportException while shutting down ipcService", e);
} catch (InterruptedException e) {
throw new HandlerException("thread was interruptedwhile shutting down ipcService", e);
} finally {
String evtSource = this.getSourceName();
runtime.stop();
if (!this.skipWriteStats) {
writeStats(eventCount.get(), oldestArrivalTime.get(), oldestOccurrenceTime.get(), evtSource, runtime);
}
if (logger.isTraceEnabled()) {
getGCStats();
}
}
}
use of com.amazonaws.services.lambda.runtime.Context in project bender by Nextdoor.
the class S3Transport method sendBatch.
@Override
public void sendBatch(TransportBuffer buffer, LinkedHashMap<String, String> partitions, Context context) throws TransportException {
S3TransportBuffer buf = (S3TransportBuffer) buffer;
/*
* Create s3 key (filepath + filename)
*/
LinkedHashMap<String, String> parts = new LinkedHashMap<String, String>(partitions);
String filename = parts.remove(FILENAME_KEY);
if (filename == null) {
filename = context.getAwsRequestId();
}
String key = parts.entrySet().stream().map(s -> s.getKey() + "=" + s.getValue()).collect(Collectors.joining("/"));
key = (key.equals("") ? filename : key + '/' + filename);
if (this.basePath.endsWith("/")) {
key = this.basePath + key;
} else {
key = this.basePath + '/' + key;
}
// TODO: make this dynamic
if (key.endsWith(".gz")) {
key = key.substring(0, key.length() - 3);
}
/*
* Add or strip out compression format extension
*
* TODO: get this based on the compression codec
*/
if (this.compress || buf.isCompressed()) {
key += ".bz2";
}
ByteArrayOutputStream os = buf.getInternalBuffer();
/*
* Compress stream if needed. Don't compress a compressed stream.
*/
ByteArrayOutputStream payload;
if (this.compress && !buf.isCompressed()) {
payload = compress(os);
} else {
payload = os;
}
/*
* For memory efficiency convert the output stream into an InputStream. This is done using the
* easystream library but under the hood it uses piped streams to facilitate this process. This
* avoids copying the entire contents of the OutputStream to populate the InputStream. Note that
* this process creates another thread to consume from the InputStream.
*/
final String s3Key = key;
/*
* Write to OutputStream
*/
final InputStreamFromOutputStream<String> isos = new InputStreamFromOutputStream<String>() {
public String produce(final OutputStream dataSink) throws Exception {
/*
* Note this is executed in a different thread
*/
payload.writeTo(dataSink);
return null;
}
};
/*
* Consume InputStream
*/
try {
sendStream(isos, s3Key, payload.size());
} finally {
try {
isos.close();
} catch (IOException e) {
throw new TransportException(e);
} finally {
buf.close();
}
}
}
Aggregations