Examples with LogSegmentMetadata org.apache.distributedlog.LogSegmentMetadata used on opensource projects

Example 21 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestDLCK method testCheckAndRepairDLNamespace.

@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testCheckAndRepairDLNamespace() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    confLocal.setLogSegmentCacheEnabled(false);
    URI uri = createDLMURI("/check-and-repair-dl-namespace");
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().name("dlck-tool").numThreads(1).build();
    ExecutorService executorService = Executors.newCachedThreadPool();
    String streamName = "check-and-repair-dl-namespace";
    // Create completed log segments
    DistributedLogManager dlm = namespace.openLog(streamName);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 1L, 1L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 2L, 11L, 10, true);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 3L, 21L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 4L, 31L, 10, true);
    // dryrun
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, namespace, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(namespace)), scheduler, false, false);
    Map<Long, LogSegmentMetadata> segments = getLogSegments(dlm);
    LOG.info("segments after drynrun {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 16L, 0L), 2L, 9, 19L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 16L, 0L), 4L, 9, 39L);
    // check and repair
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, namespace, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(namespace)), scheduler, false, false);
    segments = getLogSegments(dlm);
    LOG.info("segments after repair {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 18L, 0L), 2L, 10, 20L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 18L, 0L), 4L, 10, 40L);
    dlm.close();
    SchedulerUtils.shutdownScheduler(executorService, 5, TimeUnit.MINUTES);
    namespace.close();
}

Example 22 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestDLCK method verifyLogSegment.

static void verifyLogSegment(Map<Long, LogSegmentMetadata> segments, DLSN lastDLSN, long logSegmentSequenceNumber, int recordCount, long lastTxId) {
    LogSegmentMetadata segment = segments.get(logSegmentSequenceNumber);
    assertNotNull(segment);
    assertEquals(lastDLSN, segment.getLastDLSN());
    assertEquals(recordCount, segment.getRecordCount());
    assertEquals(lastTxId, segment.getLastTxId());
}

Example 23 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestDLCK method getLogSegments.

static Map<Long, LogSegmentMetadata> getLogSegments(DistributedLogManager dlm) throws Exception {
    Map<Long, LogSegmentMetadata> logSegmentMap = new HashMap<Long, LogSegmentMetadata>();
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    for (LogSegmentMetadata segment : segments) {
        logSegmentMap.put(segment.getLogSegmentSequenceNumber(), segment);
    }
    return logSegmentMap;
}

Example 24 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestZKLogSegmentMetadataStore method testGetLogSegmentNames.

@Test(timeout = 60000)
public void testGetLogSegmentNames() throws Exception {
    Transaction<Object> createTxn = lsmStore.transaction();
    List<LogSegmentMetadata> createdSegments = Lists.newArrayListWithExpectedSize(10);
    for (int i = 0; i < 10; i++) {
        LogSegmentMetadata segment = createLogSegment(i);
        createdSegments.add(segment);
        lsmStore.createLogSegment(createTxn, segment, null);
    }
    Utils.ioResult(createTxn.execute());
    String rootPath = "/" + runtime.getMethodName();
    List<String> children = zkc.get().getChildren(rootPath, false);
    Collections.sort(children);
    assertEquals("Should find 10 log segments", 10, children.size());
    List<String> logSegmentNames = Utils.ioResult(lsmStore.getLogSegmentNames(rootPath, null)).getValue();
    Collections.sort(logSegmentNames);
    assertEquals("Should find 10 log segments", 10, logSegmentNames.size());
    assertEquals(children, logSegmentNames);
    List<CompletableFuture<LogSegmentMetadata>> getFutures = Lists.newArrayListWithExpectedSize(10);
    for (int i = 0; i < 10; i++) {
        getFutures.add(lsmStore.getLogSegment(rootPath + "/" + logSegmentNames.get(i)));
    }
    List<LogSegmentMetadata> segments = Utils.ioResult(FutureUtils.collect(getFutures));
    for (int i = 0; i < 10; i++) {
        assertEquals(createdSegments.get(i), segments.get(i));
    }
}

Example 25 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestZKLogSegmentMetadataStore method testLogSegmentNamesListener.

@Test(timeout = 60000)
public void testLogSegmentNamesListener() throws Exception {
    int numSegments = 3;
    Transaction<Object> createTxn = lsmStore.transaction();
    for (int i = 0; i < numSegments; i++) {
        LogSegmentMetadata segment = createLogSegment(i);
        lsmStore.createLogSegment(createTxn, segment, null);
    }
    Utils.ioResult(createTxn.execute());
    String rootPath = "/" + runtime.getMethodName();
    List<String> children = zkc.get().getChildren(rootPath, false);
    Collections.sort(children);
    final AtomicInteger numNotifications = new AtomicInteger(0);
    final List<List<String>> segmentLists = Lists.newArrayListWithExpectedSize(2);
    LogSegmentNamesListener listener = new LogSegmentNamesListener() {

        @Override
        public void onSegmentsUpdated(Versioned<List<String>> segments) {
            logger.info("Received segments : {}", segments);
            segmentLists.add(segments.getValue());
            numNotifications.incrementAndGet();
        }

        @Override
        public void onLogStreamDeleted() {
        // no-op;
        }
    };
    lsmStore.getLogSegmentNames(rootPath, listener);
    assertEquals(1, lsmStore.listeners.size());
    assertTrue("Should contain listener", lsmStore.listeners.containsKey(rootPath));
    assertTrue("Should contain listener", lsmStore.listeners.get(rootPath).containsKey(listener));
    while (numNotifications.get() < 1) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals("Should receive one segment list update", 1, numNotifications.get());
    List<String> firstSegmentList = segmentLists.get(0);
    Collections.sort(firstSegmentList);
    assertEquals("List of segments should be same", children, firstSegmentList);
    logger.info("Create another {} segments.", numSegments);
    // create another log segment, it should trigger segment list updated
    Transaction<Object> anotherCreateTxn = lsmStore.transaction();
    for (int i = numSegments; i < 2 * numSegments; i++) {
        LogSegmentMetadata segment = createLogSegment(i);
        lsmStore.createLogSegment(anotherCreateTxn, segment, null);
    }
    Utils.ioResult(anotherCreateTxn.execute());
    List<String> newChildren = zkc.get().getChildren(rootPath, false);
    Collections.sort(newChildren);
    logger.info("All log segments become {}", newChildren);
    while (numNotifications.get() < 2) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals("Should receive second segment list update", 2, numNotifications.get());
    List<String> secondSegmentList = segmentLists.get(1);
    Collections.sort(secondSegmentList);
    assertEquals("List of segments should be updated", 2 * numSegments, secondSegmentList.size());
    assertEquals("List of segments should be updated", newChildren, secondSegmentList);
}