Search in sources :

Example 1 with Context

use of com.amazonaws.services.lambda.runtime.Context in project bender by Nextdoor.

the class BaseHandler method processInternal.

/**
 * Method called by Handler implementations to process records.
 *
 * @param context Lambda invocation context.
 * @throws HandlerException
 */
private void processInternal(Context context) throws HandlerException {
    Stat runtime = new Stat("runtime.ns");
    runtime.start();
    Source source = this.getSource();
    DeserializerProcessor deser = source.getDeserProcessor();
    List<OperationProcessor> operations = source.getOperationProcessors();
    List<String> containsStrings = source.getContainsStrings();
    List<Pattern> regexPatterns = source.getRegexPatterns();
    this.getIpcService().setContext(context);
    Iterator<InternalEvent> events = this.getInternalEventIterator();
    /*
     * For logging purposes log when the function started running
     */
    this.monitor.invokeTimeNow();
    AtomicLong eventCount = new AtomicLong(0);
    AtomicLong oldestArrivalTime = new AtomicLong(System.currentTimeMillis());
    AtomicLong oldestOccurrenceTime = new AtomicLong(System.currentTimeMillis());
    /*
     * Process each record
     */
    int characteristics = Spliterator.IMMUTABLE;
    Spliterator<InternalEvent> spliterator = Spliterators.spliteratorUnknownSize(events, characteristics);
    Stream<InternalEvent> input = StreamSupport.stream(spliterator, false);
    /*
     * Filter out raw events
     */
    Stream<InternalEvent> filtered = input.filter(/*
         * Perform regex filter
         */
    ievent -> {
        eventCount.incrementAndGet();
        String eventStr = ievent.getEventString();
        /*
           * Apply String contains filters before deserialization
           */
        for (String containsString : containsStrings) {
            if (eventStr.contains(containsString)) {
                return false;
            }
        }
        /*
           * Apply regex patterns before deserialization
           */
        for (Pattern regexPattern : regexPatterns) {
            Matcher m = regexPattern.matcher(eventStr);
            if (m.find()) {
                return false;
            }
        }
        return true;
    });
    /*
     * Deserialize
     */
    Stream<InternalEvent> deserialized = filtered.map(ievent -> {
        DeserializedEvent data = deser.deserialize(ievent.getEventString());
        if (data == null || data.getPayload() == null) {
            logger.warn("Failed to deserialize: " + ievent.getEventString());
            return null;
        }
        ievent.setEventObj(data);
        return ievent;
    }).filter(Objects::nonNull);
    /*
     * Perform Operations
     */
    Stream<InternalEvent> operated = deserialized;
    for (OperationProcessor operation : operations) {
        operated = operation.perform(operated);
    }
    /*
     * Serialize
     */
    Stream<InternalEvent> serialized = operated.map(ievent -> {
        try {
            String raw = null;
            raw = this.ser.serialize(this.wrapper.getWrapped(ievent));
            ievent.setSerialized(raw);
            return ievent;
        } catch (SerializationException e) {
            return null;
        }
    }).filter(Objects::nonNull);
    /*
     * Transport
     */
    serialized.forEach(ievent -> {
        /*
       * Update times
       */
        updateOldest(oldestArrivalTime, ievent.getArrivalTime());
        updateOldest(oldestOccurrenceTime, ievent.getEventTime());
        try {
            this.getIpcService().add(ievent);
        } catch (TransportException e) {
            logger.warn("error adding event", e);
        }
    });
    /*
     * Wait for transporters to finish
     */
    try {
        this.getIpcService().shutdown();
    } catch (TransportException e) {
        throw new HandlerException("encounted TransportException while shutting down ipcService", e);
    } catch (InterruptedException e) {
        throw new HandlerException("thread was interruptedwhile shutting down ipcService", e);
    } finally {
        String evtSource = this.getSourceName();
        runtime.stop();
        if (!this.skipWriteStats) {
            writeStats(eventCount.get(), oldestArrivalTime.get(), oldestOccurrenceTime.get(), evtSource, runtime);
        }
        if (logger.isTraceEnabled()) {
            getGCStats();
        }
    }
}
Also used : Monitor(com.nextdoor.bender.monitoring.Monitor) Spliterators(java.util.Spliterators) Wrapper(com.nextdoor.bender.wrapper.Wrapper) Context(com.amazonaws.services.lambda.runtime.Context) Stat(com.nextdoor.bender.monitoring.Stat) InternalEvent(com.nextdoor.bender.InternalEvent) OperationProcessor(com.nextdoor.bender.operation.OperationProcessor) ArrayList(java.util.ArrayList) IpcSenderService(com.nextdoor.bender.ipc.IpcSenderService) Logger(org.apache.log4j.Logger) Matcher(java.util.regex.Matcher) GarbageCollectorMXBean(java.lang.management.GarbageCollectorMXBean) AmazonS3ClientFactory(com.nextdoor.bender.aws.AmazonS3ClientFactory) TransportException(com.nextdoor.bender.ipc.TransportException) BenderConfig(com.nextdoor.bender.config.BenderConfig) StreamSupport(java.util.stream.StreamSupport) ManagementFactory(java.lang.management.ManagementFactory) DeserializedEvent(com.nextdoor.bender.deserializer.DeserializedEvent) Iterator(java.util.Iterator) IOException(java.io.IOException) SerializerProcessor(com.nextdoor.bender.serializer.SerializerProcessor) ConfigurationException(com.nextdoor.bender.config.ConfigurationException) Objects(java.util.Objects) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) Stream(java.util.stream.Stream) SerializationException(com.nextdoor.bender.serializer.SerializationException) BenderLayout(com.nextdoor.bender.logging.BenderLayout) Pattern(java.util.regex.Pattern) Source(com.nextdoor.bender.config.Source) Spliterator(java.util.Spliterator) DeserializerProcessor(com.nextdoor.bender.deserializer.DeserializerProcessor) AmazonS3URI(com.amazonaws.services.s3.AmazonS3URI) HandlerResources(com.nextdoor.bender.config.HandlerResources) Pattern(java.util.regex.Pattern) DeserializedEvent(com.nextdoor.bender.deserializer.DeserializedEvent) SerializationException(com.nextdoor.bender.serializer.SerializationException) Matcher(java.util.regex.Matcher) OperationProcessor(com.nextdoor.bender.operation.OperationProcessor) TransportException(com.nextdoor.bender.ipc.TransportException) Source(com.nextdoor.bender.config.Source) InternalEvent(com.nextdoor.bender.InternalEvent) AtomicLong(java.util.concurrent.atomic.AtomicLong) Stat(com.nextdoor.bender.monitoring.Stat) Objects(java.util.Objects) DeserializerProcessor(com.nextdoor.bender.deserializer.DeserializerProcessor)

Example 2 with Context

use of com.amazonaws.services.lambda.runtime.Context in project bender by Nextdoor.

the class S3Transport method sendBatch.

@Override
public void sendBatch(TransportBuffer buffer, LinkedHashMap<String, String> partitions, Context context) throws TransportException {
    S3TransportBuffer buf = (S3TransportBuffer) buffer;
    /*
     * Create s3 key (filepath + filename)
     */
    LinkedHashMap<String, String> parts = new LinkedHashMap<String, String>(partitions);
    String filename = parts.remove(FILENAME_KEY);
    if (filename == null) {
        filename = context.getAwsRequestId();
    }
    String key = parts.entrySet().stream().map(s -> s.getKey() + "=" + s.getValue()).collect(Collectors.joining("/"));
    key = (key.equals("") ? filename : key + '/' + filename);
    if (this.basePath.endsWith("/")) {
        key = this.basePath + key;
    } else {
        key = this.basePath + '/' + key;
    }
    // TODO: make this dynamic
    if (key.endsWith(".gz")) {
        key = key.substring(0, key.length() - 3);
    }
    /*
     * Add or strip out compression format extension
     *
     * TODO: get this based on the compression codec
     */
    if (this.compress || buf.isCompressed()) {
        key += ".bz2";
    }
    ByteArrayOutputStream os = buf.getInternalBuffer();
    /*
     * Compress stream if needed. Don't compress a compressed stream.
     */
    ByteArrayOutputStream payload;
    if (this.compress && !buf.isCompressed()) {
        payload = compress(os);
    } else {
        payload = os;
    }
    /*
     * For memory efficiency convert the output stream into an InputStream. This is done using the
     * easystream library but under the hood it uses piped streams to facilitate this process. This
     * avoids copying the entire contents of the OutputStream to populate the InputStream. Note that
     * this process creates another thread to consume from the InputStream.
     */
    final String s3Key = key;
    /*
     * Write to OutputStream
     */
    final InputStreamFromOutputStream<String> isos = new InputStreamFromOutputStream<String>() {

        public String produce(final OutputStream dataSink) throws Exception {
            /*
         * Note this is executed in a different thread
         */
            payload.writeTo(dataSink);
            return null;
        }
    };
    /*
     * Consume InputStream
     */
    try {
        sendStream(isos, s3Key, payload.size());
    } finally {
        try {
            isos.close();
        } catch (IOException e) {
            throw new TransportException(e);
        } finally {
            buf.close();
        }
    }
}
Also used : OutputStream(java.io.OutputStream) UploadPartRequest(com.amazonaws.services.s3.model.UploadPartRequest) PartitionedTransport(com.nextdoor.bender.ipc.PartitionedTransport) BZip2CompressorOutputStream(org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream) ByteArrayOutputStream(org.apache.commons.io.output.ByteArrayOutputStream) Context(com.amazonaws.services.lambda.runtime.Context) IOException(java.io.IOException) InputStreamFromOutputStream(com.gc.iotools.stream.is.InputStreamFromOutputStream) AmazonS3Client(com.amazonaws.services.s3.AmazonS3Client) Collectors(java.util.stream.Collectors) LinkedHashMap(java.util.LinkedHashMap) Logger(org.apache.log4j.Logger) TransportBuffer(com.nextdoor.bender.ipc.TransportBuffer) InitiateMultipartUploadRequest(com.amazonaws.services.s3.model.InitiateMultipartUploadRequest) TransportException(com.nextdoor.bender.ipc.TransportException) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata) Map(java.util.Map) InitiateMultipartUploadResult(com.amazonaws.services.s3.model.InitiateMultipartUploadResult) UploadPartResult(com.amazonaws.services.s3.model.UploadPartResult) AmazonClientException(com.amazonaws.AmazonClientException) InputStream(java.io.InputStream) InputStreamFromOutputStream(com.gc.iotools.stream.is.InputStreamFromOutputStream) OutputStream(java.io.OutputStream) BZip2CompressorOutputStream(org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream) ByteArrayOutputStream(org.apache.commons.io.output.ByteArrayOutputStream) InputStreamFromOutputStream(com.gc.iotools.stream.is.InputStreamFromOutputStream) ByteArrayOutputStream(org.apache.commons.io.output.ByteArrayOutputStream) IOException(java.io.IOException) TransportException(com.nextdoor.bender.ipc.TransportException) LinkedHashMap(java.util.LinkedHashMap)

Aggregations

Context (com.amazonaws.services.lambda.runtime.Context)2 TransportException (com.nextdoor.bender.ipc.TransportException)2 IOException (java.io.IOException)2 Logger (org.apache.log4j.Logger)2 AmazonClientException (com.amazonaws.AmazonClientException)1 AmazonS3Client (com.amazonaws.services.s3.AmazonS3Client)1 AmazonS3URI (com.amazonaws.services.s3.AmazonS3URI)1 InitiateMultipartUploadRequest (com.amazonaws.services.s3.model.InitiateMultipartUploadRequest)1 InitiateMultipartUploadResult (com.amazonaws.services.s3.model.InitiateMultipartUploadResult)1 ObjectMetadata (com.amazonaws.services.s3.model.ObjectMetadata)1 UploadPartRequest (com.amazonaws.services.s3.model.UploadPartRequest)1 UploadPartResult (com.amazonaws.services.s3.model.UploadPartResult)1 InputStreamFromOutputStream (com.gc.iotools.stream.is.InputStreamFromOutputStream)1 InternalEvent (com.nextdoor.bender.InternalEvent)1 AmazonS3ClientFactory (com.nextdoor.bender.aws.AmazonS3ClientFactory)1 BenderConfig (com.nextdoor.bender.config.BenderConfig)1 ConfigurationException (com.nextdoor.bender.config.ConfigurationException)1 HandlerResources (com.nextdoor.bender.config.HandlerResources)1 Source (com.nextdoor.bender.config.Source)1 DeserializedEvent (com.nextdoor.bender.deserializer.DeserializedEvent)1