Examples with Entry org.apache.samza.storage.kv.Entry used on opensource projects

Example 1 with Entry

use of org.apache.samza.storage.kv.Entry in project samza by apache.

the class TaskSideInputHandler method process.

/**
 * Processes the incoming side input message envelope and updates the last processed offset for its SSP.
 * Synchronized inorder to be exclusive with flush().
 *
 * @param envelope incoming envelope to be processed
 */
public synchronized void process(IncomingMessageEnvelope envelope) {
    SystemStreamPartition envelopeSSP = envelope.getSystemStreamPartition();
    String envelopeOffset = envelope.getOffset();
    for (String store : this.sspToStores.get(envelopeSSP)) {
        SideInputsProcessor storeProcessor = this.storeToProcessor.get(store);
        KeyValueStore keyValueStore = (KeyValueStore) this.taskSideInputStorageManager.getStore(store);
        Collection<Entry<?, ?>> entriesToBeWritten = storeProcessor.process(envelope, keyValueStore);
        // TODO: SAMZA-2255: optimize writes to side input stores
        for (Entry entry : entriesToBeWritten) {
            // If the key is null we ignore, if the value is null, we issue a delete, else we issue a put
            if (entry.getKey() != null) {
                if (entry.getValue() != null) {
                    keyValueStore.put(entry.getKey(), entry.getValue());
                } else {
                    keyValueStore.delete(entry.getKey());
                }
            }
        }
    }
    this.lastProcessedOffsets.put(envelopeSSP, envelopeOffset);
    checkCaughtUp(envelopeSSP, envelopeOffset, SystemStreamMetadata.OffsetType.NEWEST);
}

Example 2 with Entry

use of org.apache.samza.storage.kv.Entry in project samza by apache.

the class TestAsyncRetriableTable method testUpdateAllWithOneRetry.

@Test
public void testUpdateAllWithOneRetry() {
    TableRetryPolicy policy = new TableRetryPolicy();
    policy.withFixedBackoff(Duration.ofMillis(10));
    TableReadFunction<String, String> readFn = mock(TableReadFunction.class);
    TableWriteFunction<String, String, String> writeFn = mock(TableWriteFunction.class);
    doReturn(true).when(writeFn).isRetriable(any());
    AtomicInteger times = new AtomicInteger();
    doAnswer(invocation -> {
        CompletableFuture<Map<String, String>> future = new CompletableFuture();
        if (times.get() > 0) {
            future.complete(null);
        } else {
            times.incrementAndGet();
            future.completeExceptionally(new RuntimeException("test exception"));
        }
        return future;
    }).when(writeFn).updateAllAsync(any());
    AsyncReadWriteUpdateTable delegate = new AsyncRemoteTable(readFn, writeFn);
    AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn);
    table.init(TestRemoteTable.getMockContext());
    table.updateAllAsync(Arrays.asList(new Entry(1, 2))).join();
    verify(writeFn, times(2)).updateAllAsync(any());
    assertEquals(1, table.writeRetryMetrics.retryCount.getCount());
    assertEquals(0, table.writeRetryMetrics.successCount.getCount());
    assertEquals(0, table.writeRetryMetrics.permFailureCount.getCount());
    assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0);
}

Example 3 with Entry

use of org.apache.samza.storage.kv.Entry in project samza by apache.

the class TestAsyncRetriableTable method testPutAllWithOneRetry.

@Test
public void testPutAllWithOneRetry() {
    TableRetryPolicy policy = new TableRetryPolicy();
    policy.withFixedBackoff(Duration.ofMillis(10));
    TableReadFunction<String, String> readFn = mock(TableReadFunction.class);
    TableWriteFunction<String, String, String> writeFn = mock(TableWriteFunction.class);
    doReturn(true).when(writeFn).isRetriable(any());
    AtomicInteger times = new AtomicInteger();
    doAnswer(invocation -> {
        CompletableFuture<Map<String, String>> future = new CompletableFuture();
        if (times.get() > 0) {
            future.complete(null);
        } else {
            times.incrementAndGet();
            future.completeExceptionally(new RuntimeException("test exception"));
        }
        return future;
    }).when(writeFn).putAllAsync(any());
    AsyncReadWriteUpdateTable delegate = new AsyncRemoteTable(readFn, writeFn);
    AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn);
    table.init(TestRemoteTable.getMockContext());
    table.putAllAsync(Arrays.asList(new Entry(1, 2))).join();
    verify(writeFn, times(2)).putAllAsync(any());
    assertEquals(1, table.writeRetryMetrics.retryCount.getCount());
    assertEquals(0, table.writeRetryMetrics.successCount.getCount());
    assertEquals(0, table.writeRetryMetrics.permFailureCount.getCount());
    assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0);
}

Example 4 with Entry

use of org.apache.samza.storage.kv.Entry in project samza by apache.

the class TestAsyncRetriableTable method testPutWithPermFailureOnMaxCount.

@Test
public void testPutWithPermFailureOnMaxCount() {
    TableRetryPolicy policy = new TableRetryPolicy();
    policy.withFixedBackoff(Duration.ofMillis(5));
    policy.withStopAfterAttempts(10);
    TableReadFunction<String, String> readFn = mock(TableReadFunction.class);
    TableWriteFunction<String, String, String> writeFn = mock(TableWriteFunction.class);
    doReturn(true).when(writeFn).isRetriable(any());
    CompletableFuture<String> future = new CompletableFuture();
    future.completeExceptionally(new RuntimeException("test exception"));
    doReturn(future).when(writeFn).putAllAsync(any());
    AsyncReadWriteUpdateTable delegate = new AsyncRemoteTable(readFn, writeFn);
    AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn);
    table.init(TestRemoteTable.getMockContext());
    try {
        table.putAllAsync(Arrays.asList(new Entry(1, 2))).join();
        fail();
    } catch (Throwable t) {
    }
    verify(writeFn, atLeast(11)).putAllAsync(any());
    assertEquals(10, table.writeRetryMetrics.retryCount.getCount());
    assertEquals(0, table.writeRetryMetrics.successCount.getCount());
    assertEquals(1, table.writeRetryMetrics.permFailureCount.getCount());
    assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0);
}

Example 5 with Entry

use of org.apache.samza.storage.kv.Entry in project samza by apache.

the class TestAsyncRateLimitedTable method testPutAllAsync.

@Test
public void testPutAllAsync() {
    writeTable.putAllAsync(Arrays.asList(new Entry("1", "2"))).join();
    verify(writeFn, times(1)).putAllAsync(anyCollection());
    verify(writeFn, times(0)).putAllAsync(anyCollection(), any());
    verify(writeRateLimiter, times(0)).throttle(anyString());
    verify(writeRateLimiter, times(0)).throttle(anyString(), anyString());
    verify(writeRateLimiter, times(0)).throttle(anyCollection());
    verify(writeRateLimiter, times(1)).throttleRecords(anyCollection());
    verify(writeRateLimiter, times(0)).throttle(anyInt(), any());
    verifyReadPartNotCalled();
}