Examples with ZookeeperBucketStore io.pravega.controller.store.stream.ZookeeperBucketStore used on opensource projects

Example 1 with ZookeeperBucketStore

use of io.pravega.controller.store.stream.ZookeeperBucketStore in project pravega by pravega.

the class BucketServiceFactory method createWatermarkingService.

public BucketManager createWatermarkingService(Duration executionDuration, BucketWork work, ScheduledExecutorService executorService) {
    switch(bucketStore.getStoreType()) {
        case Zookeeper:
            ZookeeperBucketStore zkBucketStore = (ZookeeperBucketStore) bucketStore;
            Function<Integer, BucketService> zkSupplier = bucket -> new ZooKeeperBucketService(BucketStore.ServiceType.WatermarkingService, bucket, zkBucketStore, executorService, maxConcurrentExecutions, executionDuration, work);
            return new ZooKeeperBucketManager(hostId, zkBucketStore, BucketStore.ServiceType.WatermarkingService, executorService, zkSupplier);
        case InMemory:
            InMemoryBucketStore inMemoryBucketStore = (InMemoryBucketStore) bucketStore;
            Function<Integer, BucketService> inMemorySupplier = bucket -> new InMemoryBucketService(BucketStore.ServiceType.WatermarkingService, bucket, inMemoryBucketStore, executorService, maxConcurrentExecutions, executionDuration, work);
            return new InMemoryBucketManager(hostId, (InMemoryBucketStore) bucketStore, BucketStore.ServiceType.WatermarkingService, executorService, inMemorySupplier);
        default:
            throw new IllegalArgumentException(String.format("store type %s not supported", bucketStore.getStoreType().name()));
    }
}

Example 2 with ZookeeperBucketStore

use of io.pravega.controller.store.stream.ZookeeperBucketStore in project pravega by pravega.

the class BucketServiceFactory method createRetentionService.

public BucketManager createRetentionService(Duration executionDuration, BucketWork work, ScheduledExecutorService executorService) {
    switch(bucketStore.getStoreType()) {
        case Zookeeper:
            ZookeeperBucketStore zkBucketStore = (ZookeeperBucketStore) bucketStore;
            Function<Integer, BucketService> zkSupplier = bucket -> new ZooKeeperBucketService(BucketStore.ServiceType.RetentionService, bucket, zkBucketStore, executorService, maxConcurrentExecutions, executionDuration, work);
            return new ZooKeeperBucketManager(hostId, zkBucketStore, BucketStore.ServiceType.RetentionService, executorService, zkSupplier);
        case InMemory:
            InMemoryBucketStore inMemoryBucketStore = (InMemoryBucketStore) bucketStore;
            Function<Integer, BucketService> inMemorySupplier = bucket -> new InMemoryBucketService(BucketStore.ServiceType.RetentionService, bucket, inMemoryBucketStore, executorService, maxConcurrentExecutions, executionDuration, work);
            return new InMemoryBucketManager(hostId, (InMemoryBucketStore) bucketStore, BucketStore.ServiceType.RetentionService, executorService, inMemorySupplier);
        default:
            throw new IllegalArgumentException(String.format("store type %s not supported", bucketStore.getStoreType().name()));
    }
}

Example 3 with ZookeeperBucketStore

use of io.pravega.controller.store.stream.ZookeeperBucketStore in project pravega by pravega.

the class ZkStoreBucketServiceTest method testMaxConcurrentJobs.

@Test(timeout = 30000)
public void testMaxConcurrentJobs() {
    String scope = "scope";
    String stream1 = "stream1";
    String stream2 = "stream2";
    String stream3 = "stream3";
    String stream4 = "stream4";
    ConcurrentHashMap<String, CompletableFuture<Void>> streamWorkLatch = new ConcurrentHashMap<>();
    ConcurrentHashMap<String, CompletableFuture<Void>> streamWorkFuture = new ConcurrentHashMap<>();
    streamWorkFuture.put(stream1, new CompletableFuture<>());
    streamWorkFuture.put(stream2, new CompletableFuture<>());
    streamWorkFuture.put(stream3, new CompletableFuture<>());
    streamWorkFuture.put(stream4, new CompletableFuture<>());
    streamWorkLatch.put(stream1, new CompletableFuture<>());
    streamWorkLatch.put(stream2, new CompletableFuture<>());
    streamWorkLatch.put(stream3, new CompletableFuture<>());
    streamWorkLatch.put(stream4, new CompletableFuture<>());
    BucketWork bucketWork = x -> {
        // indicate that the work has been called
        streamWorkLatch.get(x.getStreamName()).complete(null);
        return streamWorkFuture.get(x.getStreamName());
    };
    BucketService service = new ZooKeeperBucketService(BucketStore.ServiceType.RetentionService, 0, (ZookeeperBucketStore) bucketStore, executor, 2, Duration.ofSeconds(2), bucketWork);
    service.startAsync();
    service.awaitRunning();
    // verify that we add a new stream and it gets added
    // send notification for stream1 and verify that it's work gets called
    BucketService.StreamNotification notification = new BucketService.StreamNotification(scope, stream1, BucketService.NotificationType.StreamAdded);
    service.notify(notification);
    AtomicBoolean added = new AtomicBoolean(false);
    Futures.loop(added::get, () -> {
        added.set(service.getKnownStreams().stream().anyMatch(x -> x.getStreamName().equals(stream1)));
        return CompletableFuture.completedFuture(null);
    }, executor).join();
    // wait for stream work to be called
    streamWorkLatch.get(stream1).join();
    // complete stream 1 work
    streamWorkFuture.get(stream1).complete(null);
    Set<Stream> streams = service.getKnownStreams();
    assertEquals(streams.size(), 1);
    Stream stream1obj = streams.iterator().next();
    // now send notification to remove the stream
    notification = new BucketService.StreamNotification(scope, stream1, BucketService.NotificationType.StreamRemoved);
    service.notify(notification);
    // whenever notification loop kicks in, it should remove the stream from known streams and worker queue.
    AtomicBoolean removed = new AtomicBoolean(false);
    Futures.loop(removed::get, () -> {
        removed.set(!service.getKnownStreams().contains(stream1obj));
        return CompletableFuture.completedFuture(null);
    }, executor).join();
    // the work should also be removed from worker queue
    Futures.loop(removed::get, () -> {
        removed.set(service.getWorkerQueue().stream().noneMatch(x -> x.getStream().equals(stream1obj)));
        return CompletableFuture.completedFuture(null);
    }, executor).join();
    // send notification for addition of stream2, stream3
    notification = new BucketService.StreamNotification(scope, stream2, BucketService.NotificationType.StreamAdded);
    service.notify(notification);
    notification = new BucketService.StreamNotification(scope, stream3, BucketService.NotificationType.StreamAdded);
    service.notify(notification);
    // wait for two "work" to be called.. wait on those latches to complete
    streamWorkLatch.get(stream2).join();
    streamWorkLatch.get(stream3).join();
    assertEquals(service.getWorkerQueue().size(), 0);
    // send notification for addition of stream4
    notification = new BucketService.StreamNotification(scope, stream4, BucketService.NotificationType.StreamAdded);
    service.notify(notification);
    // its work should not get called!
    Collection<BucketService.QueueElement> workerQueueAfterDelay = Futures.delayedFuture(() -> CompletableFuture.completedFuture(service.getWorkerQueue()), 5000, executor).join();
    assertFalse(streamWorkLatch.get(stream4).isDone());
    // worker queue should still have the element for stream4 waiting
    assertEquals(workerQueueAfterDelay.size(), 1);
    assertEquals(workerQueueAfterDelay.iterator().next().getStream().getStreamName(), stream4);
    // finish one of the work and we shall have worker queue pick up work from stream4
    streamWorkFuture.get(stream2).complete(null);
    // stream 4's work should be called and completed
    streamWorkLatch.get(stream4).join();
}

Example 4 with ZookeeperBucketStore

use of io.pravega.controller.store.stream.ZookeeperBucketStore in project pravega by pravega.

the class WatermarkingServiceHealthContibutorTest method setup.

@Before
public void setup() {
    ZookeeperBucketStore bucketStore = mock(ZookeeperBucketStore.class);
    doReturn(StoreType.Zookeeper).when(bucketStore).getStoreType();
    String hostId = UUID.randomUUID().toString();
    BucketServiceFactory bucketStoreFactory = new BucketServiceFactory(hostId, bucketStore, 2);
    ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
    PeriodicWatermarking periodicWatermarking = mock(PeriodicWatermarking.class);
    watermarkingService = spy(bucketStoreFactory.createWatermarkingService(Duration.ofMillis(10), periodicWatermarking::watermark, executor));
    doReturn(CompletableFuture.completedFuture(null)).when((ZooKeeperBucketManager) watermarkingService).initializeService();
    doNothing().when((ZooKeeperBucketManager) watermarkingService).startBucketOwnershipListener();
    doReturn(true).when(watermarkingService).isHealthy();
    contributor = new WatermarkingServiceHealthContributor("watermarkingservice", watermarkingService);
    builder = Health.builder().name("watermark");
}