Examples with LogSegmentMetadata com.twitter.distributedlog.LogSegmentMetadata used on opensource projects

Example 16 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class TestZKLogSegmentMetadataStore method testUpdateLogSegment.

@Test(timeout = 60000)
public void testUpdateLogSegment() throws Exception {
    LogSegmentMetadata segment = createLogSegment(1L, 99L);
    Transaction<Object> createTxn = lsmStore.transaction();
    lsmStore.createLogSegment(createTxn, segment);
    FutureUtils.result(createTxn.execute());
    // the log segment should be created
    assertNotNull("LogSegment " + segment + " should be created", zkc.get().exists(segment.getZkPath(), false));
    LogSegmentMetadata modifiedSegment = createLogSegment(1L, 999L);
    Transaction<Object> updateTxn = lsmStore.transaction();
    lsmStore.updateLogSegment(updateTxn, modifiedSegment);
    FutureUtils.result(updateTxn.execute());
    // the log segment should be updated
    LogSegmentMetadata readSegment = FutureUtils.result(LogSegmentMetadata.read(zkc, segment.getZkPath(), true));
    assertEquals("Last entry id should be changed from 99L to 999L", 999L, readSegment.getLastEntryId());
}

Example 17 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class TestZKLogSegmentMetadataStore method testLogSegmentNamesListener.

@Test(timeout = 60000)
public void testLogSegmentNamesListener() throws Exception {
    int numSegments = 3;
    Transaction<Object> createTxn = lsmStore.transaction();
    for (int i = 0; i < numSegments; i++) {
        LogSegmentMetadata segment = createLogSegment(i);
        lsmStore.createLogSegment(createTxn, segment);
    }
    FutureUtils.result(createTxn.execute());
    String rootPath = "/" + runtime.getMethodName();
    List<String> children = zkc.get().getChildren(rootPath, false);
    Collections.sort(children);
    final AtomicInteger numNotifications = new AtomicInteger(0);
    final List<List<String>> segmentLists = Lists.newArrayListWithExpectedSize(2);
    LogSegmentNamesListener listener = new LogSegmentNamesListener() {

        @Override
        public void onSegmentsUpdated(List<String> segments) {
            logger.info("Received segments : {}", segments);
            segmentLists.add(segments);
            numNotifications.incrementAndGet();
        }
    };
    lsmStore.registerLogSegmentListener(rootPath, listener);
    assertEquals(1, lsmStore.listeners.size());
    assertTrue("Should contain listener", lsmStore.listeners.containsKey(rootPath));
    assertTrue("Should contain listener", lsmStore.listeners.get(rootPath).contains(listener));
    while (numNotifications.get() < 1) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals("Should receive one segment list update", 1, numNotifications.get());
    List<String> firstSegmentList = segmentLists.get(0);
    Collections.sort(firstSegmentList);
    assertEquals("List of segments should be same", children, firstSegmentList);
    logger.info("Create another {} segments.", numSegments);
    // create another log segment, it should trigger segment list updated
    Transaction<Object> anotherCreateTxn = lsmStore.transaction();
    for (int i = numSegments; i < 2 * numSegments; i++) {
        LogSegmentMetadata segment = createLogSegment(i);
        lsmStore.createLogSegment(anotherCreateTxn, segment);
    }
    FutureUtils.result(anotherCreateTxn.execute());
    List<String> newChildren = zkc.get().getChildren(rootPath, false);
    Collections.sort(newChildren);
    logger.info("All log segments become {}", newChildren);
    while (numNotifications.get() < 2) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals("Should receive second segment list update", 2, numNotifications.get());
    List<String> secondSegmentList = segmentLists.get(1);
    Collections.sort(secondSegmentList);
    assertEquals("List of segments should be updated", 2 * numSegments, secondSegmentList.size());
    assertEquals("List of segments should be updated", newChildren, secondSegmentList);
}

Example 18 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class DistributedLogAdmin method checkStream.

private static StreamCandidate checkStream(final com.twitter.distributedlog.DistributedLogManagerFactory factory, final String streamName, final ExecutorService executorService, final BookKeeperClient bkc, String digestpw) throws IOException {
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        if (segments.isEmpty()) {
            return null;
        }
        List<Future<LogSegmentCandidate>> futures = new ArrayList<Future<LogSegmentCandidate>>(segments.size());
        for (LogSegmentMetadata segment : segments) {
            futures.add(checkLogSegment(streamName, segment, executorService, bkc, digestpw));
        }
        List<LogSegmentCandidate> segmentCandidates;
        try {
            segmentCandidates = Await.result(Future.collect(futures));
        } catch (Exception e) {
            throw new IOException("Failed on checking stream " + streamName, e);
        }
        StreamCandidate streamCandidate = new StreamCandidate(streamName);
        for (LogSegmentCandidate segmentCandidate : segmentCandidates) {
            if (null != segmentCandidate) {
                streamCandidate.addLogSegmentCandidate(segmentCandidate);
            }
        }
        if (streamCandidate.segmentCandidates.isEmpty()) {
            return null;
        }
        return streamCandidate;
    } finally {
        dlm.close();
    }
}

Example 19 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class DistributedLogAdmin method repairStream.

private static boolean repairStream(MetadataUpdater metadataUpdater, StreamCandidate streamCandidate, boolean verbose, boolean interactive) throws IOException {
    if (verbose) {
        System.out.println("Stream " + streamCandidate.streamName + " : ");
        for (LogSegmentCandidate segmentCandidate : streamCandidate.segmentCandidates) {
            System.out.println("  " + segmentCandidate.metadata.getLogSegmentSequenceNumber() + " : metadata = " + segmentCandidate.metadata + ", last dlsn = " + segmentCandidate.lastRecord.getDlsn());
        }
        System.out.println("-------------------------------------------");
    }
    if (interactive && !IOUtils.confirmPrompt("Do you want to fix the stream " + streamCandidate.streamName + " (Y/N) : ")) {
        return false;
    }
    for (LogSegmentCandidate segmentCandidate : streamCandidate.segmentCandidates) {
        LogSegmentMetadata newMetadata = FutureUtils.result(metadataUpdater.updateLastRecord(segmentCandidate.metadata, segmentCandidate.lastRecord));
        if (verbose) {
            System.out.println("  Fixed segment " + segmentCandidate.metadata.getLogSegmentSequenceNumber() + " : ");
            System.out.println("    old metadata : " + segmentCandidate.metadata);
            System.out.println("    new metadata : " + newMetadata);
        }
    }
    if (verbose) {
        System.out.println("-------------------------------------------");
    }
    return true;
}

Example 20 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class DLAuditor method collectLedgersFromStream.

private List<Long> collectLedgersFromStream(com.twitter.distributedlog.DistributedLogManagerFactory factory, String stream, Set<Long> ledgers) throws IOException {
    DistributedLogManager dlm = factory.createDistributedLogManager(stream, com.twitter.distributedlog.DistributedLogManagerFactory.ClientSharingOption.SharedClients);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        List<Long> sLedgers = new ArrayList<Long>();
        for (LogSegmentMetadata segment : segments) {
            synchronized (ledgers) {
                ledgers.add(segment.getLedgerId());
            }
            sLedgers.add(segment.getLedgerId());
        }
        return sLedgers;
    } finally {
        dlm.close();
    }
}