Examples with LogSegmentMetadata com.twitter.distributedlog.LogSegmentMetadata used on opensource projects

Example 11 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class LogSegmentCache method add.

/**
     * Add the segment <i>metadata</i> for <i>name</i> in the cache.
     *
     * @param name
     *          segment name.
     * @param metadata
     *          segment metadata.
     */
public void add(String name, LogSegmentMetadata metadata) {
    synchronized (logSegments) {
        if (!logSegments.containsKey(name)) {
            logSegments.put(name, metadata);
            LOG.info("{} added log segment ({} : {}) to cache.", new Object[] { streamName, name, metadata });
        }
        LogSegmentMetadata oldMetadata = lid2LogSegments.remove(metadata.getLedgerId());
        if (null == oldMetadata) {
            lid2LogSegments.put(metadata.getLedgerId(), metadata);
        } else {
            if (oldMetadata.isInProgress() && !metadata.isInProgress()) {
                lid2LogSegments.put(metadata.getLedgerId(), metadata);
            } else {
                lid2LogSegments.put(oldMetadata.getLedgerId(), oldMetadata);
            }
        }
    }
}

Example 12 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class TestDLCK method testCheckAndRepairDLNamespace.

@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testCheckAndRepairDLNamespace() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    URI uri = createDLMURI("/check-and-repair-dl-namespace");
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    com.twitter.distributedlog.DistributedLogManagerFactory factory = new com.twitter.distributedlog.DistributedLogManagerFactory(confLocal, uri);
    ExecutorService executorService = Executors.newCachedThreadPool();
    String streamName = "check-and-repair-dl-namespace";
    // Create completed log segments
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 1L, 1L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 2L, 11L, 10, true);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 3L, 21L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 4L, 31L, 10, true);
    // dryrun
    BookKeeperClient bkc = getBookKeeperClient(factory);
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, factory, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(factory)), executorService, bkc, confLocal.getBKDigestPW(), false, false);
    Map<Long, LogSegmentMetadata> segments = getLogSegments(dlm);
    LOG.info("segments after drynrun {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 16L, 0L), 2L, 9, 19L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 16L, 0L), 4L, 9, 39L);
    // check and repair
    bkc = getBookKeeperClient(factory);
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, factory, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(factory)), executorService, bkc, confLocal.getBKDigestPW(), false, false);
    segments = getLogSegments(dlm);
    LOG.info("segments after repair {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 18L, 0L), 2L, 10, 20L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 18L, 0L), 4L, 10, 40L);
    dlm.close();
    SchedulerUtils.shutdownScheduler(executorService, 5, TimeUnit.MINUTES);
    factory.close();
}

Example 13 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class TestDLCK method getLogSegments.

static Map<Long, LogSegmentMetadata> getLogSegments(DistributedLogManager dlm) throws Exception {
    Map<Long, LogSegmentMetadata> logSegmentMap = new HashMap<Long, LogSegmentMetadata>();
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    for (LogSegmentMetadata segment : segments) {
        logSegmentMap.put(segment.getLogSegmentSequenceNumber(), segment);
    }
    return logSegmentMap;
}

Example 14 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class TestZKLogSegmentMetadataStore method testDeleteLogSegment.

@Test(timeout = 60000)
public void testDeleteLogSegment() throws Exception {
    LogSegmentMetadata segment = createLogSegment(1L);
    Transaction<Object> createTxn = lsmStore.transaction();
    lsmStore.createLogSegment(createTxn, segment);
    FutureUtils.result(createTxn.execute());
    // the log segment should be created
    assertNotNull("LogSegment " + segment + " should be created", zkc.get().exists(segment.getZkPath(), false));
    Transaction<Object> deleteTxn = lsmStore.transaction();
    lsmStore.deleteLogSegment(deleteTxn, segment);
    FutureUtils.result(deleteTxn.execute());
    assertNull("LogSegment " + segment + " should be deleted", zkc.get().exists(segment.getZkPath(), false));
}

Example 15 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class TestZKLogSegmentMetadataStore method testRegisterListenerAfterLSMStoreClosed.

@Test(timeout = 60000)
public void testRegisterListenerAfterLSMStoreClosed() throws Exception {
    lsmStore.close();
    LogSegmentMetadata segment = createLogSegment(1L);
    lsmStore.registerLogSegmentListener(segment.getZkPath(), new LogSegmentNamesListener() {

        @Override
        public void onSegmentsUpdated(List<String> segments) {
        // no-op;
        }
    });
    assertTrue("No listener is registered", lsmStore.listeners.isEmpty());
}