Examples with OrderValidation io.confluent.examples.streams.avro.microservices.OrderValidation used on opensource projects

Example 1 with OrderValidation

use of io.confluent.examples.streams.avro.microservices.OrderValidation in project kafka-streams-examples by confluentinc.

the class FraudService method processStreams.

private KafkaStreams processStreams(final String bootstrapServers, final String stateDir) {
    // Latch onto instances of the orders and inventory topics
    StreamsBuilder builder = new StreamsBuilder();
    KStream<String, Order> orders = builder.stream(ORDERS.name(), Consumed.with(ORDERS.keySerde(), ORDERS.valueSerde())).filter((id, order) -> OrderState.CREATED.equals(order.getState()));
    // Create an aggregate of the total value by customer and hold it with the order. We use session windows to
    // detect periods of activity.
    KTable<Windowed<Long>, OrderValue> aggregate = orders.groupBy((id, order) -> order.getCustomerId(), Serialized.with(Serdes.Long(), ORDERS.valueSerde())).windowedBy(SessionWindows.with(60 * MIN)).aggregate(OrderValue::new, // Calculate running total for each customer within this window
    (custId, order, total) -> new OrderValue(order, total.getValue() + order.getQuantity() * order.getPrice()), // include a merger as we're using session windows.
    (k, a, b) -> simpleMerge(a, b), Materialized.with(null, Schemas.ORDER_VALUE_SERDE));
    // Ditch the windowing and rekey
    KStream<String, OrderValue> ordersWithTotals = aggregate.toStream((windowedKey, orderValue) -> windowedKey.key()).filter(// When elements are evicted from a session window they create delete events. Filter these out.
    (k, v) -> v != null).selectKey((id, orderValue) -> orderValue.getOrder().getId());
    // Now branch the stream into two, for pass and fail, based on whether the windowed total is over Fraud Limit
    KStream<String, OrderValue>[] forks = ordersWithTotals.branch((id, orderValue) -> orderValue.getValue() >= FRAUD_LIMIT, (id, orderValue) -> orderValue.getValue() < FRAUD_LIMIT);
    forks[0].mapValues(orderValue -> new OrderValidation(orderValue.getOrder().getId(), FRAUD_CHECK, FAIL)).to(ORDER_VALIDATIONS.name(), Produced.with(ORDER_VALIDATIONS.keySerde(), ORDER_VALIDATIONS.valueSerde()));
    forks[1].mapValues(orderValue -> new OrderValidation(orderValue.getOrder().getId(), FRAUD_CHECK, PASS)).to(ORDER_VALIDATIONS.name(), Produced.with(ORDER_VALIDATIONS.keySerde(), ORDER_VALIDATIONS.valueSerde()));
    // disable caching to ensure a complete aggregate changelog. This is a little trick we need to apply
    // as caching in Kafka Streams will conflate subsequent updates for the same key. Disabling caching ensures
    // we get a complete "changelog" from the aggregate(...) step above (i.e. every input event will have a
    // corresponding output event.
    Properties props = baseStreamsConfig(bootstrapServers, stateDir, FRAUD_SERVICE_APP_ID);
    props.setProperty(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, "0");
    return new KafkaStreams(builder.build(), props);
}

Example 2 with OrderValidation

use of io.confluent.examples.streams.avro.microservices.OrderValidation in project kafka-streams-examples by confluentinc.

the class ValidationsAggregatorService method aggregateOrderValidations.

private KafkaStreams aggregateOrderValidations(String bootstrapServers, String stateDir) {
    // TODO put into a KTable to make dynamically configurable
    final int numberOfRules = 3;
    StreamsBuilder builder = new StreamsBuilder();
    KStream<String, OrderValidation> validations = builder.stream(ORDER_VALIDATIONS.name(), serdes1);
    KStream<String, Order> orders = builder.stream(ORDERS.name(), serdes2).filter((id, order) -> OrderState.CREATED.equals(order.getState()));
    // If all rules pass then validate the order
    validations.groupByKey(serdes3).windowedBy(SessionWindows.with(5 * MIN)).aggregate(() -> 0L, (id, result, total) -> PASS.equals(result.getValidationResult()) ? total + 1 : total, // include a merger as we're using session windows.
    (k, a, b) -> b == null ? a : b, Materialized.with(null, Serdes.Long())).toStream((windowedKey, total) -> windowedKey.key()).filter((k1, v) -> v != null).filter((k, total) -> total >= numberOfRules).join(orders, (id, order) -> newBuilder(order).setState(VALIDATED).build(), JoinWindows.of(5 * MIN), serdes4).to(ORDERS.name(), serdes5);
    // If any rule fails then fail the order
    validations.filter((id, rule) -> FAIL.equals(rule.getValidationResult())).join(orders, (id, order) -> newBuilder(order).setState(OrderState.FAILED).build(), JoinWindows.of(5 * MIN), serdes7).groupByKey(serdes6).reduce((order, v1) -> order).toStream().to(ORDERS.name(), Produced.with(ORDERS.keySerde(), ORDERS.valueSerde()));
    return new KafkaStreams(builder.build(), baseStreamsConfig(bootstrapServers, stateDir, ORDERS_SERVICE_APP_ID));
}

Example 3 with OrderValidation

use of io.confluent.examples.streams.avro.microservices.OrderValidation in project kafka-streams-examples by confluentinc.

the class FraudServiceTest method shouldValidateWhetherOrderAmountExceedsFraudLimitOverWindow.

@Test
public void shouldValidateWhetherOrderAmountExceedsFraudLimitOverWindow() throws Exception {
    // Given
    fraudService = new FraudService();
    List<Order> orders = asList(new Order(id(0L), 0L, CREATED, UNDERPANTS, 3, 5.00d), new Order(id(1L), 0L, CREATED, JUMPERS, 1, 75.00d), new Order(id(2L), 1L, CREATED, JUMPERS, 1, 75.00d), new Order(id(3L), 1L, CREATED, JUMPERS, 1, 75.00d), // Should fail as over limit
    new Order(id(4L), 1L, CREATED, JUMPERS, 50, 75.00d), // First should pass
    new Order(id(5L), 2L, CREATED, UNDERPANTS, 10, 100.00d), // Second should fail as rolling total by customer is over limit
    new Order(id(6L), 2L, CREATED, UNDERPANTS, 10, 100.00d), // Third should fail as rolling total by customer is still over limit
    new Order(id(7L), 2L, CREATED, UNDERPANTS, 1, 5.00d));
    sendOrders(orders);
    // When
    fraudService.start(CLUSTER.bootstrapServers());
    // Then there should be failures for the two orders that push customers over their limit.
    List<OrderValidation> expected = asList(new OrderValidation(id(0L), FRAUD_CHECK, PASS), new OrderValidation(id(1L), FRAUD_CHECK, PASS), new OrderValidation(id(2L), FRAUD_CHECK, PASS), new OrderValidation(id(3L), FRAUD_CHECK, PASS), new OrderValidation(id(4L), FRAUD_CHECK, FAIL), new OrderValidation(id(5L), FRAUD_CHECK, PASS), new OrderValidation(id(6L), FRAUD_CHECK, FAIL), new OrderValidation(id(7L), FRAUD_CHECK, FAIL));
    List<OrderValidation> read = read(Topics.ORDER_VALIDATIONS, 8, CLUSTER.bootstrapServers());
    assertThat(read).isEqualTo(expected);
}

Example 4 with OrderValidation

use of io.confluent.examples.streams.avro.microservices.OrderValidation in project kafka-streams-examples by confluentinc.

the class ValidationsAggregatorServiceTest method shouldAggregateRuleSuccesses.

@Test
public void shouldAggregateRuleSuccesses() throws Exception {
    // Given
    ordersService = new ValidationsAggregatorService();
    orders = asList(new Order(id(0L), 0L, CREATED, UNDERPANTS, 3, 5.00d), new Order(id(1L), 0L, CREATED, JUMPERS, 1, 75.00d));
    sendOrders(orders);
    ruleResults = asList(new OrderValidation(id(0L), OrderValidationType.FRAUD_CHECK, OrderValidationResult.PASS), new OrderValidation(id(0L), OrderValidationType.ORDER_DETAILS_CHECK, OrderValidationResult.PASS), new OrderValidation(id(0L), OrderValidationType.INVENTORY_CHECK, OrderValidationResult.PASS), new OrderValidation(id(1L), OrderValidationType.FRAUD_CHECK, OrderValidationResult.PASS), new OrderValidation(id(1L), OrderValidationType.ORDER_DETAILS_CHECK, OrderValidationResult.FAIL), new OrderValidation(id(1L), OrderValidationType.INVENTORY_CHECK, OrderValidationResult.PASS));
    sendOrderValuations(ruleResults);
    // When
    ordersService.start(CLUSTER.bootstrapServers());
    // Then
    List<KeyValue<String, Order>> finalOrders = MicroserviceTestUtils.readKeyValues(Topics.ORDERS, 4, CLUSTER.bootstrapServers());
    assertThat(finalOrders.size()).isEqualTo(4);
    // And the first order should have been validated but the second should have failed
    assertThat(finalOrders.stream().map(kv -> kv.value).collect(Collectors.toList())).contains(new Order(id(0L), 0L, VALIDATED, UNDERPANTS, 3, 5.00d), new Order(id(1L), 0L, FAILED, JUMPERS, 1, 75.00d));
}