Examples with Record com.amazon.dataprepper.model.record.Record used on opensource projects

Example 81 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class AggregateProcessor method doExecute.

@Override
public Collection<Record<Event>> doExecute(Collection<Record<Event>> records) {
    final List<Record<Event>> recordsOut = new LinkedList<>();
    final List<Map.Entry<AggregateIdentificationKeysHasher.IdentificationHash, AggregateGroup>> groupsToConclude = aggregateGroupManager.getGroupsToConclude();
    for (final Map.Entry<AggregateIdentificationKeysHasher.IdentificationHash, AggregateGroup> groupEntry : groupsToConclude) {
        final Optional<Event> concludeGroupEvent = aggregateActionSynchronizer.concludeGroup(groupEntry.getKey(), groupEntry.getValue());
        if (concludeGroupEvent.isPresent()) {
            recordsOut.add(new Record(concludeGroupEvent.get()));
            actionConcludeGroupEventsOutCounter.increment();
        } else {
            actionConcludeGroupEventsDroppedCounter.increment();
        }
    }
    int handleEventsOut = 0;
    int handleEventsDropped = 0;
    for (final Record<Event> record : records) {
        final Event event = record.getData();
        final AggregateIdentificationKeysHasher.IdentificationHash identificationKeysHash = aggregateIdentificationKeysHasher.createIdentificationKeyHashFromEvent(event);
        final AggregateGroup aggregateGroupForEvent = aggregateGroupManager.getAggregateGroup(identificationKeysHash);
        final AggregateActionResponse handleEventResponse = aggregateActionSynchronizer.handleEventForGroup(event, identificationKeysHash, aggregateGroupForEvent);
        final Event aggregateActionResponseEvent = handleEventResponse.getEvent();
        if (aggregateActionResponseEvent != null) {
            recordsOut.add(new Record<>(aggregateActionResponseEvent, record.getMetadata()));
            handleEventsOut++;
        } else {
            handleEventsDropped++;
        }
    }
    actionHandleEventsOutCounter.increment(handleEventsOut);
    actionHandleEventsDroppedCounter.increment(handleEventsDropped);
    return recordsOut;
}

Example 82 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class StringPrepper method execute.

@Override
public Collection<Record<Event>> execute(final Collection<Record<Event>> records) {
    final Collection<Record<Event>> modifiedRecords = new ArrayList<>(records.size());
    for (Record<Event> record : records) {
        final Event recordEvent = record.getData();
        final String eventJson = recordEvent.toJsonString();
        try {
            final Map<String, Object> newData = processEventJson(eventJson);
            final Event newRecordEvent = JacksonEvent.builder().withEventMetadata(recordEvent.getMetadata()).withData(newData).build();
            modifiedRecords.add(new Record<>(newRecordEvent));
        } catch (JsonProcessingException e) {
            LOG.error("Unable to process Event data: {}", eventJson, e);
        }
    }
    return modifiedRecords;
}

Example 83 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class TestBuffer method read.

@Override
public Map.Entry<Collection<Record<Event>>, CheckpointState> read(int timeoutInMillis) {
    final List<Record<Event>> records = new ArrayList<>();
    int index = 0;
    Record<Event> record;
    while (index < batchSize && (record = buffer.poll()) != null) {
        records.add(record);
        index++;
    }
    final CheckpointState checkpointState = new CheckpointState(records.size());
    return new AbstractMap.SimpleEntry<>(records, checkpointState);
}

Example 84 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class StringPrepperTests method testStringPrepperUpperCase.

@Test
public void testStringPrepperUpperCase() {
    configuration.setUpperCase(true);
    final StringPrepper stringPrepper = createObjectUnderTest();
    final List<Record<Event>> modifiedRecords = (List<Record<Event>>) stringPrepper.execute(TEST_RECORDS);
    stringPrepper.shutdown();
    final List<Event> modifiedRecordEvents = modifiedRecords.stream().map(Record::getData).collect(Collectors.toList());
    assertThat(modifiedRecordEvents.size(), equalTo(2));
    final Event firstEvent = modifiedRecordEvents.get(0);
    final Event secondEvent = modifiedRecordEvents.get(1);
    assertTrue(firstEvent.containsKey(TEST_KEY));
    assertThat(firstEvent.getMetadata().getEventType(), equalTo(TEST_EVENT_TYPE));
    assertThat(firstEvent.get(TEST_KEY, String.class), equalTo(UPPERCASE_TEST_STRING.toUpperCase()));
    assertTrue(secondEvent.containsKey(TEST_KEY));
    assertThat(secondEvent.getMetadata().getEventType(), equalTo(TEST_EVENT_TYPE));
    assertThat(secondEvent.get(TEST_KEY, String.class), equalTo(LOWERCASE_TEST_STRING));
}

Example 85 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class StdOutSinkTests method testSinkWithCustomType.

// TODO: remove with the completion of: https://github.com/opensearch-project/data-prepper/issues/546
@Test
public void testSinkWithCustomType() {
    final StdOutSink stdOutSink = new StdOutSink(new PluginSetting(PLUGIN_NAME, new HashMap<>()));
    stdOutSink.output(Collections.singletonList(new Record<Object>(new TestObject())));
}