Examples with MockCoordinatorExecutor org.apache.hudi.sink.utils.MockCoordinatorExecutor used on opensource projects

Example 1 with MockCoordinatorExecutor

use of org.apache.hudi.sink.utils.MockCoordinatorExecutor in project hudi by apache.

the class TestStreamWriteOperatorCoordinator method before.

@BeforeEach
public void before() throws Exception {
    OperatorCoordinator.Context context = new MockOperatorCoordinatorContext(new OperatorID(), 2);
    coordinator = new StreamWriteOperatorCoordinator(TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()), context);
    coordinator.start();
    coordinator.setExecutor(new MockCoordinatorExecutor(context));
    coordinator.handleEventFromOperator(0, WriteMetadataEvent.emptyBootstrap(0));
    coordinator.handleEventFromOperator(1, WriteMetadataEvent.emptyBootstrap(1));
}

Example 2 with MockCoordinatorExecutor

use of org.apache.hudi.sink.utils.MockCoordinatorExecutor in project hudi by apache.

the class TestStreamWriteOperatorCoordinator method testSyncMetadataTable.

@Test
void testSyncMetadataTable() throws Exception {
    // reset
    reset();
    // override the default configuration
    Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath());
    conf.setBoolean(FlinkOptions.METADATA_ENABLED, true);
    conf.setInteger(FlinkOptions.METADATA_COMPACTION_DELTA_COMMITS, 5);
    OperatorCoordinator.Context context = new MockOperatorCoordinatorContext(new OperatorID(), 1);
    coordinator = new StreamWriteOperatorCoordinator(conf, context);
    coordinator.start();
    coordinator.setExecutor(new MockCoordinatorExecutor(context));
    final WriteMetadataEvent event0 = WriteMetadataEvent.emptyBootstrap(0);
    coordinator.handleEventFromOperator(0, event0);
    String instant = coordinator.getInstant();
    assertNotEquals("", instant);
    final String metadataTableBasePath = HoodieTableMetadata.getMetadataTableBasePath(tempFile.getAbsolutePath());
    HoodieTableMetaClient metadataTableMetaClient = StreamerUtil.createMetaClient(metadataTableBasePath);
    HoodieTimeline completedTimeline = metadataTableMetaClient.getActiveTimeline().filterCompletedInstants();
    assertThat("One instant need to sync to metadata table", completedTimeline.getInstants().count(), is(1L));
    assertThat(completedTimeline.lastInstant().get().getTimestamp(), is(HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP));
    // write another 3 commits
    for (int i = 1; i < 4; i++) {
        instant = mockWriteWithMetadata();
        metadataTableMetaClient.reloadActiveTimeline();
        completedTimeline = metadataTableMetaClient.getActiveTimeline().filterCompletedInstants();
        assertThat("One instant need to sync to metadata table", completedTimeline.getInstants().count(), is(i + 1L));
        assertThat(completedTimeline.lastInstant().get().getTimestamp(), is(instant));
    }
    // the 5th commit triggers the compaction
    instant = mockWriteWithMetadata();
    metadataTableMetaClient.reloadActiveTimeline();
    completedTimeline = metadataTableMetaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
    assertThat("One instant need to sync to metadata table", completedTimeline.getInstants().count(), is(6L));
    assertThat(completedTimeline.lastInstant().get().getTimestamp(), is(instant + "001"));
    assertThat(completedTimeline.lastInstant().get().getAction(), is(HoodieTimeline.COMMIT_ACTION));
    // write another 2 commits
    for (int i = 6; i < 8; i++) {
        instant = mockWriteWithMetadata();
        metadataTableMetaClient.reloadActiveTimeline();
        completedTimeline = metadataTableMetaClient.getActiveTimeline().filterCompletedInstants();
        assertThat("One instant need to sync to metadata table", completedTimeline.getInstants().count(), is(i + 1L));
        assertThat(completedTimeline.lastInstant().get().getTimestamp(), is(instant));
    }
    // write another commit to trigger clean
    instant = mockWriteWithMetadata();
    metadataTableMetaClient.reloadActiveTimeline();
    completedTimeline = metadataTableMetaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
    assertThat("One instant need to sync to metadata table", completedTimeline.getInstants().count(), is(10L));
    assertThat(completedTimeline.lastInstant().get().getTimestamp(), is(instant + "002"));
    assertThat(completedTimeline.lastInstant().get().getAction(), is(HoodieTimeline.CLEAN_ACTION));
    // write another commit
    mockWriteWithMetadata();
    // write another commit to trigger compaction
    instant = mockWriteWithMetadata();
    metadataTableMetaClient.reloadActiveTimeline();
    completedTimeline = metadataTableMetaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
    assertThat("One instant need to sync to metadata table", completedTimeline.getInstants().count(), is(13L));
    assertThat(completedTimeline.lastInstant().get().getTimestamp(), is(instant + "001"));
    assertThat(completedTimeline.lastInstant().get().getAction(), is(HoodieTimeline.COMMIT_ACTION));
}

Example 3 with MockCoordinatorExecutor

use of org.apache.hudi.sink.utils.MockCoordinatorExecutor in project hudi by apache.

the class TestStreamWriteOperatorCoordinator method testHiveSyncInvoked.

@Test
public void testHiveSyncInvoked() throws Exception {
    // reset
    reset();
    // override the default configuration
    Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath());
    conf.setBoolean(FlinkOptions.HIVE_SYNC_ENABLED, true);
    OperatorCoordinator.Context context = new MockOperatorCoordinatorContext(new OperatorID(), 1);
    coordinator = new StreamWriteOperatorCoordinator(conf, context);
    coordinator.start();
    coordinator.setExecutor(new MockCoordinatorExecutor(context));
    final WriteMetadataEvent event0 = WriteMetadataEvent.emptyBootstrap(0);
    coordinator.handleEventFromOperator(0, event0);
    String instant = mockWriteWithMetadata();
    assertNotEquals("", instant);
    // never throw for hive synchronization now
    assertDoesNotThrow(() -> coordinator.notifyCheckpointComplete(1));
}