Examples with MyStatefulApplication org.apache.samza.storage.MyStatefulApplication used on opensource projects

Example 1 with MyStatefulApplication

use of org.apache.samza.storage.MyStatefulApplication in project samza by apache.

the class TransactionalStateIntegrationTest method secondRun.

private void secondRun(String changelogTopic, List<String> expectedChangelogMessages, List<String> expectedInitialStoreContents, Map<String, String> overriddenConfigs) {
    // clear the local store directory
    if (!hostAffinity) {
        new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR));
    }
    // produce the second batch of input messages
    List<String> inputMessages = Arrays.asList("4", "5", "5", ":shutdown");
    inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m));
    // run the application
    RunApplicationContext context = runApplication(new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, Collections.singletonMap(STORE_NAME, changelogTopic)), "myApp", overriddenConfigs);
    // wait for the application to finish
    context.getRunner().waitForFinish();
    // consume and verify any additional changelog messages
    List<ConsumerRecord<String, String>> changelogRecords = consumeMessages(changelogTopic, expectedChangelogMessages.size());
    List<String> changelogMessages = changelogRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
    Assert.assertEquals(expectedChangelogMessages, changelogMessages);
    // verify the store contents during startup (this is after changelog verification to ensure init has completed)
    Assert.assertEquals(expectedInitialStoreContents, MyStatefulApplication.getInitialStoreContents().get(STORE_NAME));
}

Example 2 with MyStatefulApplication

use of org.apache.samza.storage.MyStatefulApplication in project samza by apache.

the class CheckpointVersionIntegrationTest method runStatefulApp.

private void runStatefulApp(List<String> inputMessages, List<String> expectedInputTopicMessages, List<String> expectedChangelogMessages, Map<String, String> configs) {
    // create input topic and produce the first batch of input messages
    createTopic(INPUT_TOPIC, 1);
    inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m));
    // verify that the input messages were produced successfully
    if (inputMessages.size() > 0) {
        List<ConsumerRecord<String, String>> inputRecords = consumeMessages(INPUT_TOPIC, inputMessages.size());
        List<String> readInputMessages = inputRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
        Assert.assertEquals(expectedInputTopicMessages, readInputMessages);
    }
    // run the application
    RunApplicationContext context = runApplication(new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, Collections.singletonMap(STORE_NAME, CHANGELOG_TOPIC)), "myApp", configs);
    // wait for the application to finish
    context.getRunner().waitForFinish();
    // consume and verify the changelog messages
    if (expectedChangelogMessages.size() > 0) {
        List<ConsumerRecord<String, String>> changelogRecords = consumeMessages(CHANGELOG_TOPIC, expectedChangelogMessages.size());
        List<String> changelogMessages = changelogRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
        Assert.assertEquals(expectedChangelogMessages, changelogMessages);
    }
    LOG.info("Finished initial run");
}

Example 3 with MyStatefulApplication

use of org.apache.samza.storage.MyStatefulApplication in project samza by apache.

the class CheckpointVersionIntegrationTest method finalRun.

private void finalRun(String changelogTopic, List<String> expectedChangelogMessages, List<String> expectedInitialStoreContents, List<String> inputMessages, Map<String, String> overriddenConfigs) {
    // remove previous files so restore is from the checkpointV2
    new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR));
    // produce the second batch of input messages
    inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m));
    // run the application
    RunApplicationContext context = runApplication(new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, Collections.singletonMap(STORE_NAME, changelogTopic)), "myApp", overriddenConfigs);
    // wait for the application to finish
    context.getRunner().waitForFinish();
    // verify the store contents during startup (this is after changelog verification to ensure init has completed)
    Assert.assertEquals(expectedInitialStoreContents, MyStatefulApplication.getInitialStoreContents().get(STORE_NAME));
    // consume and verify any additional changelog messages
    List<ConsumerRecord<String, String>> changelogRecords = consumeMessages(changelogTopic, expectedChangelogMessages.size());
    List<String> changelogMessages = changelogRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
    Assert.assertEquals(expectedChangelogMessages, changelogMessages);
}

Example 4 with MyStatefulApplication

use of org.apache.samza.storage.MyStatefulApplication in project samza by apache.

the class TransactionalStateMultiStoreIntegrationTest method initialRun.

private void initialRun(List<String> inputMessages, List<String> expectedChangelogMessages) {
    // create input topic and produce the first batch of input messages
    createTopic(INPUT_TOPIC, 1);
    inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m));
    // verify that the input messages were produced successfully
    if (inputMessages.size() > 0) {
        List<ConsumerRecord<String, String>> inputRecords = consumeMessages(INPUT_TOPIC, inputMessages.size());
        List<String> readInputMessages = inputRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
        Assert.assertEquals(inputMessages, readInputMessages);
    }
    SamzaApplication app = new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, ImmutableMap.of(STORE_1_NAME, STORE_1_CHANGELOG, STORE_2_NAME, STORE_2_CHANGELOG));
    // run the application
    RunApplicationContext context = runApplication(app, APP_NAME, CONFIGS);
    // consume and verify the changelog messages
    if (expectedChangelogMessages.size() > 0) {
        List<ConsumerRecord<String, String>> changelogRecords = consumeMessages(STORE_1_CHANGELOG, expectedChangelogMessages.size());
        List<String> changelogMessages = changelogRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
        Assert.assertEquals(expectedChangelogMessages, changelogMessages);
    }
    // wait for the application to finish
    context.getRunner().waitForFinish();
    LOG.info("Finished initial run");
}

Example 5 with MyStatefulApplication

use of org.apache.samza.storage.MyStatefulApplication in project samza by apache.

the class TransactionalStateMultiStoreIntegrationTest method secondRun.

private void secondRun(String changelogTopic, List<String> expectedChangelogMessages, List<String> expectedInitialStoreContents) {
    // clear the local store directory
    if (!hostAffinity) {
        new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR));
    }
    // produce the second batch of input messages
    List<String> inputMessages = Arrays.asList("4", "5", "5", ":shutdown");
    inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m));
    SamzaApplication app = new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, ImmutableMap.of(STORE_1_NAME, changelogTopic, STORE_2_NAME, STORE_2_CHANGELOG));
    // run the application
    RunApplicationContext context = runApplication(app, APP_NAME, CONFIGS);
    // wait for the application to finish
    context.getRunner().waitForFinish();
    // consume and verify any additional changelog messages
    List<ConsumerRecord<String, String>> changelogRecords = consumeMessages(changelogTopic, expectedChangelogMessages.size());
    List<String> changelogMessages = changelogRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
    Assert.assertEquals(expectedChangelogMessages, changelogMessages);
    // verify the store contents during startup (this is after changelog verification to ensure init has completed)
    Assert.assertEquals(expectedInitialStoreContents, MyStatefulApplication.getInitialStoreContents().get(STORE_1_NAME));
}