Examples with BoundedInMemoryQueueConsumer org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer used on opensource projects

Example 1 with BoundedInMemoryQueueConsumer

use of org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer in project hudi by apache.

the class TestBoundedInMemoryExecutorInSpark method testExecutor.

@Test
public void testExecutor() {
    final List<HoodieRecord> hoodieRecords = dataGen.generateInserts(instantTime, 100);
    HoodieWriteConfig hoodieWriteConfig = mock(HoodieWriteConfig.class);
    when(hoodieWriteConfig.getWriteBufferLimitBytes()).thenReturn(1024);
    BoundedInMemoryQueueConsumer<HoodieLazyInsertIterable.HoodieInsertValueGenResult<HoodieRecord>, Integer> consumer = new BoundedInMemoryQueueConsumer<HoodieLazyInsertIterable.HoodieInsertValueGenResult<HoodieRecord>, Integer>() {

        private int count = 0;

        @Override
        protected void consumeOneRecord(HoodieLazyInsertIterable.HoodieInsertValueGenResult<HoodieRecord> record) {
            count++;
        }

        @Override
        protected void finish() {
        }

        @Override
        protected Integer getResult() {
            return count;
        }
    };
    BoundedInMemoryExecutor<HoodieRecord, Tuple2<HoodieRecord, Option<IndexedRecord>>, Integer> executor = null;
    try {
        executor = new BoundedInMemoryExecutor(hoodieWriteConfig.getWriteBufferLimitBytes(), hoodieRecords.iterator(), consumer, getTransformFunction(HoodieTestDataGenerator.AVRO_SCHEMA), getPreExecuteRunnable());
        int result = executor.execute();
        // It should buffer and write 100 records
        assertEquals(100, result);
        // There should be no remaining records in the buffer
        assertFalse(executor.isRemaining());
    } finally {
        if (executor != null) {
            executor.shutdownNow();
        }
    }
}

Example 2 with BoundedInMemoryQueueConsumer

use of org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer in project hudi by apache.

the class TestBoundedInMemoryExecutorInSpark method testInterruptExecutor.

@Test
public void testInterruptExecutor() {
    final List<HoodieRecord> hoodieRecords = dataGen.generateInserts(instantTime, 100);
    HoodieWriteConfig hoodieWriteConfig = mock(HoodieWriteConfig.class);
    when(hoodieWriteConfig.getWriteBufferLimitBytes()).thenReturn(1024);
    BoundedInMemoryQueueConsumer<HoodieLazyInsertIterable.HoodieInsertValueGenResult<HoodieRecord>, Integer> consumer = new BoundedInMemoryQueueConsumer<HoodieLazyInsertIterable.HoodieInsertValueGenResult<HoodieRecord>, Integer>() {

        @Override
        protected void consumeOneRecord(HoodieLazyInsertIterable.HoodieInsertValueGenResult<HoodieRecord> record) {
            try {
                while (true) {
                    Thread.sleep(1000);
                }
            } catch (InterruptedException ie) {
                return;
            }
        }

        @Override
        protected void finish() {
        }

        @Override
        protected Integer getResult() {
            return 0;
        }
    };
    BoundedInMemoryExecutor<HoodieRecord, Tuple2<HoodieRecord, Option<IndexedRecord>>, Integer> executor = null;
    try {
        executor = new BoundedInMemoryExecutor(hoodieWriteConfig.getWriteBufferLimitBytes(), hoodieRecords.iterator(), consumer, getTransformFunction(HoodieTestDataGenerator.AVRO_SCHEMA), getPreExecuteRunnable());
        BoundedInMemoryExecutor<HoodieRecord, Tuple2<HoodieRecord, Option<IndexedRecord>>, Integer> finalExecutor = executor;
        Thread.currentThread().interrupt();
        assertThrows(HoodieException.class, () -> finalExecutor.execute());
        assertTrue(Thread.interrupted());
    } finally {
        if (executor != null) {
            executor.shutdownNow();
        }
    }
}