Examples with CheckpointStoreException io.pravega.controller.store.checkpoint.CheckpointStoreException used on opensource projects

Example 1 with CheckpointStoreException

use of io.pravega.controller.store.checkpoint.CheckpointStoreException in project pravega by pravega.

the class ControllerEventProcessors method handleOrphanedReaders.

private CompletableFuture<Void> handleOrphanedReaders(final EventProcessorGroup<? extends ControllerEvent> group, final Supplier<Set<String>> processes) {
    return withRetriesAsync(() -> CompletableFuture.supplyAsync(() -> {
        try {
            return group.getProcesses();
        } catch (CheckpointStoreException e) {
            if (e.getType().equals(CheckpointStoreException.Type.NoNode)) {
                return Collections.<String>emptySet();
            }
            throw new CompletionException(e);
        }
    }, executor), RETRYABLE_PREDICATE, Integer.MAX_VALUE, executor).thenComposeAsync(groupProcesses -> withRetriesAsync(() -> CompletableFuture.supplyAsync(() -> {
        try {
            return new ImmutablePair<>(processes.get(), groupProcesses);
        } catch (Exception e) {
            log.error(String.format("Error fetching current processes%s", group.toString()), e);
            throw new CompletionException(e);
        }
    }, executor), RETRYABLE_PREDICATE, Integer.MAX_VALUE, executor)).thenComposeAsync(pair -> {
        Set<String> activeProcesses = pair.getLeft();
        Set<String> registeredProcesses = pair.getRight();
        if (registeredProcesses == null || registeredProcesses.isEmpty()) {
            return CompletableFuture.completedFuture(null);
        }
        if (activeProcesses != null) {
            registeredProcesses.removeAll(activeProcesses);
        }
        List<CompletableFuture<Void>> futureList = new ArrayList<>();
        for (String process : registeredProcesses) {
            futureList.add(withRetriesAsync(() -> CompletableFuture.runAsync(() -> {
                try {
                    group.notifyProcessFailure(process);
                } catch (CheckpointStoreException e) {
                    log.error(String.format("Error notifying failure of process=%s in event processor group %s", process, group.toString()), e);
                    throw new CompletionException(e);
                }
            }, executor), RETRYABLE_PREDICATE, Integer.MAX_VALUE, executor));
        }
        return Futures.allOf(futureList);
    });
}

Example 2 with CheckpointStoreException

use of io.pravega.controller.store.checkpoint.CheckpointStoreException in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method failOverTests.

@Test
public void failOverTests() throws CheckpointStoreException, InterruptedException {
    // Create mock writer objects.
    EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
    EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
    EventStreamReader<CommitEvent> commitReader = commitWriter.getReader();
    EventStreamReader<AbortEvent> abortReader = abortWriter.getReader();
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
    // Create test scope and stream.
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
    Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
    Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, System.currentTimeMillis()).join());
    // Set up txn task for creating transactions from a failedHost.
    StreamTransactionMetadataTasks failedTxnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "failedHost", connectionFactory, false, "");
    failedTxnTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
    // Create 3 transactions from failedHost.
    VersionedTransactionData tx1 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
    VersionedTransactionData tx2 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
    VersionedTransactionData tx3 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
    // Ping another txn from failedHost.
    UUID txnId = UUID.randomUUID();
    streamStore.createTransaction(SCOPE, STREAM, txnId, 10000, 30000, 30000, null, executor).join();
    PingTxnStatus pingStatus = failedTxnTasks.pingTxn(SCOPE, STREAM, txnId, 10000, null).join();
    VersionedTransactionData tx4 = streamStore.getTransactionData(SCOPE, STREAM, txnId, null, executor).join();
    // Validate versions of all txn
    Assert.assertEquals(0, tx1.getVersion());
    Assert.assertEquals(0, tx2.getVersion());
    Assert.assertEquals(0, tx3.getVersion());
    Assert.assertEquals(1, tx4.getVersion());
    Assert.assertEquals(PingTxnStatus.Status.OK, pingStatus.getStatus());
    // Validate the txn index.
    Assert.assertEquals(1, streamStore.listHostsOwningTxn().join().size());
    // Change state of one txn to COMMITTING.
    TxnStatus txnStatus2 = streamStore.sealTransaction(SCOPE, STREAM, tx2.getId(), true, Optional.empty(), null, executor).thenApply(AbstractMap.SimpleEntry::getKey).join();
    Assert.assertEquals(TxnStatus.COMMITTING, txnStatus2);
    // Change state of another txn to ABORTING.
    TxnStatus txnStatus3 = streamStore.sealTransaction(SCOPE, STREAM, tx3.getId(), false, Optional.empty(), null, executor).thenApply(AbstractMap.SimpleEntry::getKey).join();
    Assert.assertEquals(TxnStatus.ABORTING, txnStatus3);
    // Create transaction tasks for sweeping txns from failedHost.
    txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    TxnSweeper txnSweeper = new TxnSweeper(streamStore, txnTasks, 100, executor);
    // Before initializing, txnSweeper.sweepFailedHosts would throw an error
    AssertExtensions.assertThrows("IllegalStateException before initialization", txnSweeper.sweepFailedProcesses(() -> Collections.singleton("host")), ex -> ex instanceof IllegalStateException);
    // Initialize stream writers.
    txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
    // Validate that txnTasks is ready.
    assertTrue(txnTasks.isReady());
    // Sweep txns that were being managed by failedHost.
    txnSweeper.sweepFailedProcesses(() -> Collections.singleton("host")).join();
    // Validate that sweeping completes correctly.
    Assert.assertEquals(0, streamStore.listHostsOwningTxn().join().size());
    Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx1.getId(), null, executor).join());
    Assert.assertEquals(TxnStatus.COMMITTING, streamStore.transactionStatus(SCOPE, STREAM, tx2.getId(), null, executor).join());
    Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx3.getId(), null, executor).join());
    Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx4.getId(), null, executor).join());
    // Create commit and abort event processors.
    ConnectionFactory connectionFactory = Mockito.mock(ConnectionFactory.class);
    BlockingQueue<CommitEvent> processedCommitEvents = new LinkedBlockingQueue<>();
    BlockingQueue<AbortEvent> processedAbortEvents = new LinkedBlockingQueue<>();
    createEventProcessor("commitRG", "commitStream", commitReader, commitWriter, () -> new CommitEventProcessor(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, connectionFactory, processedCommitEvents));
    createEventProcessor("abortRG", "abortStream", abortReader, abortWriter, () -> new ConcurrentEventProcessor<>(new AbortRequestHandler(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, connectionFactory, processedAbortEvents), executor));
    // Wait until the commit event is processed and ensure that the txn state is COMMITTED.
    CommitEvent commitEvent = processedCommitEvents.take();
    assertEquals(tx2.getId(), commitEvent.getTxid());
    assertEquals(TxnStatus.COMMITTED, streamStore.transactionStatus(SCOPE, STREAM, tx2.getId(), null, executor).join());
    // Wait until 3 abort events are processed and ensure that the txn state is ABORTED.
    Predicate<AbortEvent> predicate = event -> event.getTxid().equals(tx1.getId()) || event.getTxid().equals(tx3.getId()) || event.getTxid().equals(tx4.getId());
    AbortEvent abortEvent1 = processedAbortEvents.take();
    assertTrue(predicate.test(abortEvent1));
    AbortEvent abortEvent2 = processedAbortEvents.take();
    assertTrue(predicate.test(abortEvent2));
    AbortEvent abortEvent3 = processedAbortEvents.take();
    assertTrue(predicate.test(abortEvent3));
    assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx1.getId(), null, executor).join());
    assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx3.getId(), null, executor).join());
    assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx4.getId(), null, executor).join());
}

Example 3 with CheckpointStoreException

use of io.pravega.controller.store.checkpoint.CheckpointStoreException in project pravega by pravega.

the class ZKCheckpointStoreTests method failingTests.

@Test
public void failingTests() {
    final String process1 = UUID.randomUUID().toString();
    final String readerGroup1 = UUID.randomUUID().toString();
    final String readerGroup2 = UUID.randomUUID().toString();
    final String reader1 = UUID.randomUUID().toString();
    cli.close();
    Predicate<Throwable> predicate = e -> e instanceof CheckpointStoreException && e.getCause() instanceof IllegalStateException;
    AssertExtensions.assertThrows("failed getProcesses", () -> checkpointStore.getProcesses(), predicate);
    AssertExtensions.assertThrows("failed addReaderGroup", () -> checkpointStore.addReaderGroup(process1, readerGroup1), predicate);
    AssertExtensions.assertThrows("failed getReaderGroups", () -> checkpointStore.getReaderGroups(process1), predicate);
    AssertExtensions.assertThrows("failed addReader", () -> checkpointStore.addReader(process1, readerGroup1, reader1), predicate);
    Position position = new PositionImpl(Collections.emptyMap());
    AssertExtensions.assertThrows("failed setPosition", () -> checkpointStore.setPosition(process1, readerGroup1, reader1, position), predicate);
    AssertExtensions.assertThrows("failed getPositions", () -> checkpointStore.getPositions(process1, readerGroup1), predicate);
    AssertExtensions.assertThrows("failed sealReaderGroup", () -> checkpointStore.sealReaderGroup(process1, readerGroup2), predicate);
    AssertExtensions.assertThrows("failed removeReader", () -> checkpointStore.removeReader(process1, readerGroup1, reader1), predicate);
    AssertExtensions.assertThrows("failed removeReaderGroup", () -> checkpointStore.removeReaderGroup(process1, readerGroup1), predicate);
}

Example 4 with CheckpointStoreException

use of io.pravega.controller.store.checkpoint.CheckpointStoreException in project pravega by pravega.

the class ZkCheckpointStoreConnectivityTest method connectivityFailureTests.

@Test
public void connectivityFailureTests() throws IOException {
    final String process1 = UUID.randomUUID().toString();
    final String readerGroup1 = UUID.randomUUID().toString();
    final String reader1 = UUID.randomUUID().toString();
    Predicate<Throwable> predicate = e -> e instanceof CheckpointStoreException && ((CheckpointStoreException) e).getType().equals(CheckpointStoreException.Type.Connectivity);
    AssertExtensions.assertThrows("failed getProcesses", () -> checkpointStore.getProcesses(), predicate);
    AssertExtensions.assertThrows("failed addReaderGroup", () -> checkpointStore.addReaderGroup(process1, readerGroup1), predicate);
    AssertExtensions.assertThrows("failed addReader", () -> checkpointStore.addReader(process1, readerGroup1, reader1), predicate);
    AssertExtensions.assertThrows("failed sealReaderGroup", () -> checkpointStore.sealReaderGroup(process1, readerGroup1), predicate);
    AssertExtensions.assertThrows("failed removeReader", () -> checkpointStore.removeReader(process1, readerGroup1, reader1), predicate);
    AssertExtensions.assertThrows("failed getPositions", () -> checkpointStore.getPositions(process1, readerGroup1), predicate);
    Position position = new PositionImpl(Collections.emptyMap());
    AssertExtensions.assertThrows("failed setPosition", () -> checkpointStore.setPosition(process1, readerGroup1, reader1, position), predicate);
    AssertExtensions.assertThrows("failed removeReader", () -> checkpointStore.removeReader(process1, readerGroup1, reader1), predicate);
    AssertExtensions.assertThrows("failed removeReaderGroup", () -> checkpointStore.removeReaderGroup(process1, readerGroup1), predicate);
}

Example 5 with CheckpointStoreException

use of io.pravega.controller.store.checkpoint.CheckpointStoreException in project pravega by pravega.

the class ControllerEventProcessorsTest method testHandleOrphaned.

@Test(timeout = 10000)
public void testHandleOrphaned() {
    Controller localController = mock(Controller.class);
    CheckpointStore checkpointStore = mock(CheckpointStore.class);
    StreamMetadataStore streamStore = mock(StreamMetadataStore.class);
    HostControllerStore hostStore = mock(HostControllerStore.class);
    ConnectionFactory connectionFactory = mock(ConnectionFactory.class);
    StreamMetadataTasks streamMetadataTasks = mock(StreamMetadataTasks.class);
    StreamTransactionMetadataTasks streamTransactionMetadataTasks = mock(StreamTransactionMetadataTasks.class);
    executor = Executors.newSingleThreadScheduledExecutor();
    ControllerEventProcessorConfig config = ControllerEventProcessorConfigImpl.withDefault();
    EventProcessorSystem system = mock(EventProcessorSystem.class);
    EventProcessorGroup<ControllerEvent> processor = new EventProcessorGroup<ControllerEvent>() {

        @Override
        public void notifyProcessFailure(String process) throws CheckpointStoreException {
        }

        @Override
        public EventStreamWriter<ControllerEvent> getWriter() {
            return null;
        }

        @Override
        public Set<String> getProcesses() throws CheckpointStoreException {
            return Sets.newHashSet("host1", "host2");
        }

        @Override
        public Service startAsync() {
            return null;
        }

        @Override
        public boolean isRunning() {
            return false;
        }

        @Override
        public State state() {
            return null;
        }

        @Override
        public Service stopAsync() {
            return null;
        }

        @Override
        public void awaitRunning() {
        }

        @Override
        public void awaitRunning(long timeout, TimeUnit unit) throws TimeoutException {
        }

        @Override
        public void awaitTerminated() {
        }

        @Override
        public void awaitTerminated(long timeout, TimeUnit unit) throws TimeoutException {
        }

        @Override
        public Throwable failureCause() {
            return null;
        }

        @Override
        public void addListener(Listener listener, Executor executor) {
        }

        @Override
        public void close() throws Exception {
        }
    };
    try {
        when(system.createEventProcessorGroup(any(), any())).thenReturn(processor);
    } catch (CheckpointStoreException e) {
        e.printStackTrace();
    }
    ControllerEventProcessors processors = new ControllerEventProcessors("host1", config, localController, checkpointStore, streamStore, hostStore, SegmentHelperMock.getSegmentHelperMock(), connectionFactory, streamMetadataTasks, system, executor);
    processors.startAsync();
    processors.awaitRunning();
    assertTrue(Futures.await(processors.sweepFailedProcesses(() -> Sets.newHashSet("host1"))));
    assertTrue(Futures.await(processors.handleFailedProcess("host1")));
    processors.shutDown();
}