Examples with BookKeeperLogFactory io.pravega.segmentstore.storage.impl.bookkeeper.BookKeeperLogFactory used on opensource projects

Example 11 with BookKeeperLogFactory

use of io.pravega.segmentstore.storage.impl.bookkeeper.BookKeeperLogFactory in project pravega by pravega.

the class S3IntegrationTest method createBuilder.

// endregion
// region StreamSegmentStoreTestBase Implementation
@Override
protected ServiceBuilder createBuilder(ServiceBuilderConfig.Builder configBuilder, int instanceId, boolean useChunkedSegmentStorage) {
    Preconditions.checkState(useChunkedSegmentStorage);
    ServiceBuilderConfig builderConfig = getBuilderConfig(configBuilder, instanceId);
    return ServiceBuilder.newInMemoryBuilder(builderConfig).withStorageFactory(setup -> new LocalS3SimpleStorageFactory(setup.getConfig(S3StorageConfig::builder), setup.getStorageExecutor())).withDataLogFactory(setup -> new BookKeeperLogFactory(setup.getConfig(BookKeeperConfig::builder), getBookkeeper().getZkClient(), setup.getCoreExecutor()));
}

Example 12 with BookKeeperLogFactory

use of io.pravega.segmentstore.storage.impl.bookkeeper.BookKeeperLogFactory in project pravega by pravega.

the class DurableLogRecoveryCommand method execute.

@Override
public void execute() throws Exception {
    @Cleanup Storage storage = this.storageFactory.createStorageAdapter();
    @Cleanup val zkClient = createZKClient();
    val bkConfig = getCommandArgs().getState().getConfigBuilder().include(BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, getServiceConfig().getZkURL())).build().getConfig(BookKeeperConfig::builder);
    @Cleanup val dataLogFactory = new BookKeeperLogFactory(bkConfig, zkClient, executorService);
    output("Container Count = %d", this.containerCount);
    dataLogFactory.initialize();
    output("Started ZK Client at %s.", getServiceConfig().getZkURL());
    storage.initialize(CONTAINER_EPOCH);
    output("Loaded %s Storage.", getServiceConfig().getStorageImplementation());
    output("Starting recovery...");
    // create back up of metadata segments
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, this.containerCount, executorService, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    @Cleanup Context context = createContext(executorService);
    // create debug segment container instances using new new dataLog and old storage.
    Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainerMap = startDebugSegmentContainers(context, dataLogFactory);
    output("Containers started. Recovering all segments...");
    ContainerRecoveryUtils.recoverAllSegments(storage, debugStreamSegmentContainerMap, executorService, TIMEOUT);
    output("All segments recovered.");
    // Update core attributes from the backUp Metadata segments
    output("Updating core attributes for segments registered.");
    ContainerRecoveryUtils.updateCoreAttributes(backUpMetadataSegments, debugStreamSegmentContainerMap, executorService, TIMEOUT);
    // Flush new metadata segment to the storage
    flushToStorage(debugStreamSegmentContainerMap);
    // Waits for metadata segments to be flushed to LTS and then stops the debug segment containers
    stopDebugSegmentContainers(debugStreamSegmentContainerMap);
    output("Segments have been recovered.");
    output("Recovery Done!");
}

Example 13 with BookKeeperLogFactory

use of io.pravega.segmentstore.storage.impl.bookkeeper.BookKeeperLogFactory in project pravega by pravega.

the class BookKeeperCommand method createContext.

/**
 * Creates a new Context to be used by the BookKeeper command.
 *
 * @return A new Context.
 * @throws DurableDataLogException If the BookKeeperLogFactory could not be initialized.
 */
@VisibleForTesting
public Context createContext() throws DurableDataLogException {
    val serviceConfig = getServiceConfig();
    val bkConfig = getCommandArgs().getState().getConfigBuilder().include(BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, serviceConfig.getZkURL())).build().getConfig(BookKeeperConfig::builder);
    val zkClient = createZKClient();
    val factory = new BookKeeperLogFactory(bkConfig, zkClient, getCommandArgs().getState().getExecutor());
    try {
        factory.initialize();
    } catch (DurableDataLogException ex) {
        zkClient.close();
        throw ex;
    }
    val bkAdmin = new BookKeeperAdmin((BookKeeper) factory.getBookKeeperClient());
    return new Context(serviceConfig, bkConfig, zkClient, factory, bkAdmin);
}

Example 14 with BookKeeperLogFactory

use of io.pravega.segmentstore.storage.impl.bookkeeper.BookKeeperLogFactory in project pravega by pravega.

the class DataRecoveryTest method testDataRecoveryCommand.

/**
 * Tests DurableLog recovery command.
 * @throws Exception    In case of any exception thrown while execution.
 */
@Test
public void testDataRecoveryCommand() throws Exception {
    int instanceId = 0;
    int bookieCount = 3;
    int containerCount = 1;
    @Cleanup TestUtils.PravegaRunner pravegaRunner = new TestUtils.PravegaRunner(bookieCount, containerCount);
    pravegaRunner.startBookKeeperRunner(instanceId++);
    pravegaRunner.startControllerAndSegmentStore(this.storageFactory, null);
    String streamName = "testDataRecoveryCommand";
    TestUtils.createScopeStream(pravegaRunner.getControllerRunner().getController(), SCOPE, streamName, config);
    try (val clientRunner = new TestUtils.ClientRunner(pravegaRunner.getControllerRunner(), SCOPE)) {
        // Write events to the streams.
        TestUtils.writeEvents(streamName, clientRunner.getClientFactory());
    }
    // Shut down the controller
    pravegaRunner.shutDownControllerRunner();
    // Flush all Tier 1 to LTS
    ServiceBuilder.ComponentSetup componentSetup = new ServiceBuilder.ComponentSetup(pravegaRunner.getSegmentStoreRunner().getServiceBuilder());
    for (int containerId = 0; containerId < containerCount; containerId++) {
        componentSetup.getContainerRegistry().getContainer(containerId).flushToStorage(TIMEOUT).join();
    }
    // Shutdown SegmentStore
    pravegaRunner.shutDownSegmentStoreRunner();
    // Shutdown BookKeeper & ZooKeeper
    pravegaRunner.shutDownBookKeeperRunner();
    // start a new BookKeeper and ZooKeeper.
    pravegaRunner.startBookKeeperRunner(instanceId++);
    // set pravega properties for the test
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.storage.impl.name", "FILESYSTEM");
    pravegaProperties.setProperty("pravegaservice.storage.layout", "ROLLING_STORAGE");
    pravegaProperties.setProperty("filesystem.root", this.baseDir.getAbsolutePath());
    pravegaProperties.setProperty("pravegaservice.zk.connect.uri", "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort());
    pravegaProperties.setProperty("bookkeeper.ledger.path", pravegaRunner.getBookKeeperRunner().getLedgerPath());
    pravegaProperties.setProperty("bookkeeper.zk.metadata.path", pravegaRunner.getBookKeeperRunner().getLogMetaNamespace());
    pravegaProperties.setProperty("pravegaservice.clusterName", pravegaRunner.getBookKeeperRunner().getBaseNamespace());
    STATE.get().getConfigBuilder().include(pravegaProperties);
    // Command under test
    TestUtils.executeCommand("data-recovery durableLog-recovery", STATE.get());
    // Start a new segment store and controller
    this.factory = new BookKeeperLogFactory(pravegaRunner.getBookKeeperRunner().getBkConfig().get(), pravegaRunner.getBookKeeperRunner().getZkClient().get(), executorService());
    pravegaRunner.startControllerAndSegmentStore(this.storageFactory, this.factory);
    log.info("Started a controller and segment store.");
    // Create the client with new controller.
    try (val clientRunner = new TestUtils.ClientRunner(pravegaRunner.getControllerRunner(), SCOPE)) {
        // Try reading all events to verify that the recovery was successful.
        TestUtils.readAllEvents(SCOPE, streamName, clientRunner.getClientFactory(), clientRunner.getReaderGroupManager(), "RG", "R");
        log.info("Read all events again to verify that segments were recovered.");
    }
    Assert.assertNotNull(StorageListSegmentsCommand.descriptor());
}

Example 15 with BookKeeperLogFactory

use of io.pravega.segmentstore.storage.impl.bookkeeper.BookKeeperLogFactory in project pravega by pravega.

the class SegmentStoreAdapter method attachDataLogFactory.

private ServiceBuilder attachDataLogFactory(ServiceBuilder builder) {
    if (this.config.getBookieCount() > 0) {
        // We were instructed to start at least one Bookie.
        this.zkClient = CuratorFrameworkFactory.builder().connectString("localhost:" + this.config.getZkPort()).namespace("pravega").retryPolicy(new ExponentialBackoffRetry(1000, 5)).sessionTimeoutMs(5000).connectionTimeoutMs(5000).build();
        this.zkClient.start();
        return builder.withDataLogFactory(setup -> {
            BookKeeperConfig bkConfig = setup.getConfig(BookKeeperConfig::builder);
            return new BookKeeperLogFactory(bkConfig, this.zkClient, setup.getCoreExecutor());
        });
    } else {
        // No Bookies -> InMemory Tier1.
        return builder.withDataLogFactory(setup -> new InMemoryDurableDataLogFactory(setup.getCoreExecutor()));
    }
}