use of io.pravega.shared.controller.event.ControllerEvent in project pravega by pravega.
the class ControllerEventProcessors method handleOrphanedReaders.
private CompletableFuture<Void> handleOrphanedReaders(final EventProcessorGroup<? extends ControllerEvent> group, final Supplier<Set<String>> processes) {
return withRetriesAsync(() -> CompletableFuture.supplyAsync(() -> {
try {
return group.getProcesses();
} catch (CheckpointStoreException e) {
if (e.getType().equals(CheckpointStoreException.Type.NoNode)) {
return Collections.<String>emptySet();
}
throw new CompletionException(e);
}
}, executor), RETRYABLE_PREDICATE, Integer.MAX_VALUE, executor).thenComposeAsync(groupProcesses -> withRetriesAsync(() -> CompletableFuture.supplyAsync(() -> {
try {
return new ImmutablePair<>(processes.get(), groupProcesses);
} catch (Exception e) {
log.error(String.format("Error fetching current processes%s", group.toString()), e);
throw new CompletionException(e);
}
}, executor), RETRYABLE_PREDICATE, Integer.MAX_VALUE, executor)).thenComposeAsync(pair -> {
Set<String> activeProcesses = pair.getLeft();
Set<String> registeredProcesses = pair.getRight();
if (registeredProcesses == null || registeredProcesses.isEmpty()) {
return CompletableFuture.completedFuture(null);
}
if (activeProcesses != null) {
registeredProcesses.removeAll(activeProcesses);
}
List<CompletableFuture<Void>> futureList = new ArrayList<>();
for (String process : registeredProcesses) {
futureList.add(withRetriesAsync(() -> CompletableFuture.runAsync(() -> {
try {
group.notifyProcessFailure(process);
} catch (CheckpointStoreException e) {
log.error(String.format("Error notifying failure of process=%s in event processor group %s", process, group.toString()), e);
throw new CompletionException(e);
}
}, executor), RETRYABLE_PREDICATE, Integer.MAX_VALUE, executor));
}
return Futures.allOf(futureList);
});
}
use of io.pravega.shared.controller.event.ControllerEvent in project pravega by pravega.
the class AbstractRequestProcessor method withCompletion.
protected <T extends ControllerEvent> CompletableFuture<Void> withCompletion(StreamTask<T> task, T event, String scope, String stream, Predicate<Throwable> writeBackPredicate) {
Preconditions.checkNotNull(task);
Preconditions.checkNotNull(event);
Preconditions.checkNotNull(scope);
Preconditions.checkNotNull(stream);
Preconditions.checkNotNull(writeBackPredicate);
CompletableFuture<Void> resultFuture = new CompletableFuture<>();
OperationContext context = streamMetadataStore.createStreamContext(scope, stream, RequestTag.NON_EXISTENT_ID);
CompletableFuture<String> waitingProcFuture = suppressException(streamMetadataStore.getWaitingRequestProcessor(scope, stream, context, executor), null, "Exception while trying to fetch waiting request. Logged and ignored.");
CompletableFuture<Boolean> hasTaskStarted = task.hasTaskStarted(event);
CompletableFuture.allOf(waitingProcFuture, hasTaskStarted).thenAccept(v -> {
boolean hasStarted = hasTaskStarted.join();
String waitingRequestProcessor = waitingProcFuture.join();
if (hasStarted || waitingRequestProcessor == null || waitingRequestProcessor.equals(getProcessorName())) {
withRetries(() -> task.execute(event), executor).whenComplete((r, ex) -> {
if (ex != null && writeBackPredicate.test(ex)) {
suppressException(streamMetadataStore.createWaitingRequestIfAbsent(scope, stream, getProcessorName(), context, executor), null, "Exception while trying to create waiting request. Logged and ignored.").thenCompose(ignore -> retryIndefinitelyThenComplete(() -> task.writeBack(event), resultFuture, ex));
} else {
// Processing was done for this event, whether it succeeded or failed, we should remove
// the waiting request if it matches the current processor.
// If we don't delete it then some other processor will never be able to do the work.
// So we need to retry indefinitely until deleted.
retryIndefinitelyThenComplete(() -> streamMetadataStore.deleteWaitingRequestConditionally(scope, stream, getProcessorName(), context, executor), resultFuture, ex);
}
});
} else {
log.debug("Found another processing requested by a different processor {}. Will postpone the event {}.", waitingRequestProcessor, event);
// This is done to guarantee fairness. If another processor has requested for processing
// on this stream, we will back off and postpone the work for later.
retryIndefinitelyThenComplete(() -> task.writeBack(event), resultFuture, StoreException.create(StoreException.Type.OPERATION_NOT_ALLOWED, "Postponed " + event + " so that waiting processor" + waitingRequestProcessor + " can work. "));
}
}).exceptionally(e -> {
resultFuture.completeExceptionally(e);
return null;
});
return resultFuture;
}
use of io.pravega.shared.controller.event.ControllerEvent in project pravega by pravega.
the class ControllerEventProcessorsTest method testHandleOrphaned.
@Test(timeout = 10000)
public void testHandleOrphaned() throws CheckpointStoreException {
LocalController localController = mock(LocalController.class);
CheckpointStore checkpointStore = mock(CheckpointStore.class);
StreamMetadataStore streamStore = mock(StreamMetadataStore.class);
BucketStore bucketStore = mock(BucketStore.class);
ConnectionPool connectionPool = mock(ConnectionPool.class);
StreamMetadataTasks streamMetadataTasks = mock(StreamMetadataTasks.class);
StreamTransactionMetadataTasks streamTransactionMetadataTasks = mock(StreamTransactionMetadataTasks.class);
KVTableMetadataStore kvtStore = mock(KVTableMetadataStore.class);
TableMetadataTasks kvtTasks = mock(TableMetadataTasks.class);
ControllerEventProcessorConfig config = ControllerEventProcessorConfigImpl.withDefault();
EventProcessorSystem system = mock(EventProcessorSystem.class);
EventProcessorGroup<ControllerEvent> processor = getProcessor();
EventProcessorGroup<ControllerEvent> mockProcessor = spy(processor);
doThrow(new CheckpointStoreException("host not found")).when(mockProcessor).notifyProcessFailure("host3");
when(system.createEventProcessorGroup(any(), any(), any())).thenReturn(mockProcessor);
@Cleanup ControllerEventProcessors processors = new ControllerEventProcessors("host1", config, localController, checkpointStore, streamStore, bucketStore, connectionPool, streamMetadataTasks, streamTransactionMetadataTasks, kvtStore, kvtTasks, system, executorService());
// check for a case where init is not initialized so that kvtRequestProcessors don't get initialized and will be null
assertTrue(Futures.await(processors.sweepFailedProcesses(() -> Sets.newHashSet("host1"))));
Assert.assertFalse(processors.isReady());
Assert.assertFalse(processors.isBootstrapCompleted());
Assert.assertFalse(processors.isMetadataServiceConnected());
processors.startAsync();
processors.awaitRunning();
assertTrue(Futures.await(processors.sweepFailedProcesses(() -> Sets.newHashSet("host1"))));
assertTrue(Futures.await(processors.handleFailedProcess("host1")));
AssertExtensions.assertFutureThrows("host not found", processors.handleFailedProcess("host3"), e -> e instanceof CheckpointStoreException);
processors.shutDown();
}
use of io.pravega.shared.controller.event.ControllerEvent in project pravega by pravega.
the class ControllerEventProcessorTest method testCommitAndStreamProcessorFairness.
@Test(timeout = 30000)
public void testCommitAndStreamProcessorFairness() {
List<VersionedTransactionData> txnDataList1 = createAndCommitTransactions(3);
int epoch = txnDataList1.get(0).getEpoch();
streamStore.startCommitTransactions(SCOPE, STREAM, 100, null, executor).join();
EventStreamWriterMock<ControllerEvent> requestEventWriter = new EventStreamWriterMock<>();
streamMetadataTasks.setRequestEventWriter(requestEventWriter);
CommitRequestHandler commitEventProcessor = new CommitRequestHandler(streamStore, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), null, null, null, null, null, null, null, streamStore, null, executor);
// set some processor name so that the processing gets postponed
streamStore.createWaitingRequestIfAbsent(SCOPE, STREAM, "test", null, executor).join();
AssertExtensions.assertFutureThrows("Operation should be disallowed", commitEventProcessor.processEvent(new CommitEvent(SCOPE, STREAM, epoch)), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
streamStore.deleteWaitingRequestConditionally(SCOPE, STREAM, "test1", null, executor).join();
assertEquals("test", streamStore.getWaitingRequestProcessor(SCOPE, STREAM, null, executor).join());
// now remove the barrier but change the state so that processing can not happen.
streamStore.deleteWaitingRequestConditionally(SCOPE, STREAM, "test", null, executor).join();
assertNull(streamStore.getWaitingRequestProcessor(SCOPE, STREAM, null, executor).join());
streamStore.setState(SCOPE, STREAM, State.SCALING, null, executor).join();
AssertExtensions.assertFutureThrows("Operation should be disallowed", commitEventProcessor.processEvent(new CommitEvent(SCOPE, STREAM, epoch)), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
assertEquals(commitEventProcessor.getProcessorName(), streamStore.getWaitingRequestProcessor(SCOPE, STREAM, null, executor).join());
streamStore.setState(SCOPE, STREAM, State.ACTIVE, null, executor).join();
// verify that we are able to process if the waiting processor name is same as ours.
commitEventProcessor.processEvent(new CommitEvent(SCOPE, STREAM, epoch)).join();
// verify that waiting processor is cleaned up.
assertNull(streamStore.getWaitingRequestProcessor(SCOPE, STREAM, null, executor).join());
// now set the state to COMMITTING_TXN and try the same with scaling
streamStore.setState(SCOPE, STREAM, State.COMMITTING_TXN, null, executor).join();
// verify that event that does not try to use `processor.withCompletion` runs without contention
assertTrue(Futures.await(streamRequestHandler.processEvent(new AutoScaleEvent(SCOPE, STREAM, 0L, AutoScaleEvent.UP, 0L, 2, true, 0L))));
// now same event's processing in face of a barrier should get postponed
streamStore.createWaitingRequestIfAbsent(SCOPE, STREAM, commitEventProcessor.getProcessorName(), null, executor).join();
assertTrue(Futures.await(streamRequestHandler.processEvent(new AutoScaleEvent(SCOPE, STREAM, 0L, AutoScaleEvent.UP, 0L, 2, true, 0L))));
AssertExtensions.assertFutureThrows("Operation should be disallowed", streamRequestHandler.processEvent(new ScaleOpEvent(SCOPE, STREAM, Collections.emptyList(), Collections.emptyList(), false, 0L, 0L)), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
assertEquals(commitEventProcessor.getProcessorName(), streamStore.getWaitingRequestProcessor(SCOPE, STREAM, null, executor).join());
}
use of io.pravega.shared.controller.event.ControllerEvent in project pravega by pravega.
the class StreamMetadataTasksTest method testWorkflowCompletionTimeout.
@Test(timeout = 30000)
public void testWorkflowCompletionTimeout() {
EventHelper helper = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamStorePartialMock).getHostTaskIndex());
StreamMetadataTasks streamMetadataTask = new StreamMetadataTasks(streamStorePartialMock, bucketStore, TaskStoreFactory.createZKStore(zkClient, executor), SegmentHelperMock.getSegmentHelperMock(), executor, "host", new GrpcAuthHelper(authEnabled, "key", 300), helper);
streamMetadataTask.setCompletionTimeoutMillis(500L);
StreamConfiguration configuration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
String completion = "completion";
streamStorePartialMock.createStream(SCOPE, completion, configuration, System.currentTimeMillis(), null, executor).join();
streamStorePartialMock.setState(SCOPE, completion, State.ACTIVE, null, executor).join();
WriterMock requestEventWriter = new WriterMock(streamMetadataTask, executor);
streamMetadataTask.setRequestEventWriter(requestEventWriter);
StreamConfiguration configuration2 = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(3)).build();
AssertExtensions.assertFutureThrows("update timedout", streamMetadataTask.updateStream(SCOPE, completion, configuration2, 0L), e -> Exceptions.unwrap(e) instanceof TimeoutException);
ControllerEvent event = requestEventWriter.eventQueue.poll();
assertTrue(event instanceof UpdateStreamEvent);
VersionedMetadata<StreamConfigurationRecord> configurationRecord = streamStorePartialMock.getConfigurationRecord(SCOPE, completion, null, executor).join();
assertTrue(configurationRecord.getObject().isUpdating());
Map<Long, Long> streamCut = Collections.singletonMap(0L, 0L);
AssertExtensions.assertFutureThrows("truncate timedout", streamMetadataTask.truncateStream(SCOPE, completion, streamCut, 0L), e -> Exceptions.unwrap(e) instanceof TimeoutException);
event = requestEventWriter.eventQueue.poll();
assertTrue(event instanceof TruncateStreamEvent);
VersionedMetadata<StreamTruncationRecord> truncationRecord = streamStorePartialMock.getTruncationRecord(SCOPE, completion, null, executor).join();
assertTrue(truncationRecord.getObject().isUpdating());
AssertExtensions.assertFutureThrows("seal timedout", streamMetadataTask.sealStream(SCOPE, completion, 0L), e -> Exceptions.unwrap(e) instanceof TimeoutException);
event = requestEventWriter.eventQueue.poll();
assertTrue(event instanceof SealStreamEvent);
VersionedMetadata<State> state = streamStorePartialMock.getVersionedState(SCOPE, completion, null, executor).join();
assertEquals(state.getObject(), State.SEALING);
streamStorePartialMock.setState(SCOPE, completion, State.SEALED, null, executor).join();
AssertExtensions.assertFutureThrows("delete timedout", streamMetadataTask.deleteStream(SCOPE, completion, 0L), e -> Exceptions.unwrap(e) instanceof TimeoutException);
event = requestEventWriter.eventQueue.poll();
assertTrue(event instanceof DeleteStreamEvent);
}
Aggregations