Examples with DistributedLogNamespace com.twitter.distributedlog.namespace.DistributedLogNamespace used on opensource projects

Example 6 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class TestZKLogMetadataForWriter method testCreateLogMetadataWithCustomMetadata.

@Test(timeout = 60000)
public void testCreateLogMetadataWithCustomMetadata() throws Exception {
    URI uri = DLMTestUtil.createDLMURI(zkPort, "");
    String logName = testName.getMethodName();
    String logIdentifier = "<default>";
    List<String> pathsToDelete = Lists.newArrayList();
    DLMetadata.create(new BKDLConfig(zkServers, "/ledgers")).update(uri);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(new DistributedLogConfiguration()).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(logName);
    dlm.createOrUpdateMetadata(logName.getBytes("UTF-8"));
    dlm.close();
    testCreateLogMetadataWithMissingPaths(uri, logName, logIdentifier, pathsToDelete, true, false);
}

Example 7 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class TestStreamManager method testCreateStream.

@Test
public void testCreateStream() throws Exception {
    Stream mockStream = mock(Stream.class);
    final String streamName = "stream1";
    when(mockStream.getStreamName()).thenReturn(streamName);
    StreamFactory mockStreamFactory = mock(StreamFactory.class);
    StreamPartitionConverter mockPartitionConverter = mock(StreamPartitionConverter.class);
    StreamConfigProvider mockStreamConfigProvider = mock(StreamConfigProvider.class);
    when(mockStreamFactory.create((String) any(), (DynamicDistributedLogConfiguration) any(), (StreamManager) any())).thenReturn(mockStream);
    DistributedLogNamespace dlNamespace = mock(DistributedLogNamespace.class);
    ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1);
    StreamManager streamManager = new StreamManagerImpl("", new DistributedLogConfiguration(), executorService, mockStreamFactory, mockPartitionConverter, mockStreamConfigProvider, dlNamespace);
    assertTrue(Await.ready(streamManager.createStreamAsync(streamName)).isReturn());
    verify(dlNamespace).createLog(streamName);
}

Example 8 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class ConsoleWriter method main.

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }
    String dlUriStr = args[0];
    final String streamName = args[1];
    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.setImmediateFlushEnabled(true);
    conf.setOutputBufferSize(0);
    conf.setPeriodicFlushFrequencyMilliSeconds(0);
    conf.setLockTimeout(DistributedLogConstants.LOCK_IMMEDIATE);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).regionId(DistributedLogConstants.LOCAL_REGION_ID).clientId("console-writer").build();
    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);
    try {
        AsyncLogWriter writer = null;
        try {
            writer = FutureUtils.result(dlm.openAsyncLogWriter());
            ConsoleReader reader = new ConsoleReader();
            String line;
            while ((line = reader.readLine(PROMPT_MESSAGE)) != null) {
                writer.write(new LogRecord(System.currentTimeMillis(), line.getBytes(UTF_8))).addEventListener(new FutureEventListener<DLSN>() {

                    @Override
                    public void onFailure(Throwable cause) {
                        System.out.println("Encountered error on writing data");
                        cause.printStackTrace(System.err);
                        Runtime.getRuntime().exit(0);
                    }

                    @Override
                    public void onSuccess(DLSN value) {
                    // done
                    }
                });
            }
        } finally {
            if (null != writer) {
                FutureUtils.result(writer.asyncClose(), Duration.apply(5, TimeUnit.SECONDS));
            }
        }
    } finally {
        dlm.close();
        namespace.close();
    }
}

Example 9 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class StreamTransformer method main.

public static void main(String[] args) throws Exception {
    if (3 != args.length) {
        System.out.println(HELP);
        return;
    }
    String dlUriStr = args[0];
    final String srcStreamName = args[1];
    final String targetStreamName = args[2];
    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    // 16KB
    conf.setOutputBufferSize(16 * 1024);
    // 5ms
    conf.setPeriodicFlushFrequencyMilliSeconds(5);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    // open the dlm
    System.out.println("Opening log stream " + srcStreamName);
    DistributedLogManager srcDlm = namespace.openLog(srcStreamName);
    System.out.println("Opening log stream " + targetStreamName);
    DistributedLogManager targetDlm = namespace.openLog(targetStreamName);
    Transformer<byte[], byte[]> replicationTransformer = new IdenticalTransformer<byte[]>();
    LogRecordWithDLSN lastTargetRecord;
    DLSN srcDlsn;
    try {
        lastTargetRecord = targetDlm.getLastLogRecord();
        TransformedRecord lastTransformedRecord = new TransformedRecord();
        try {
            lastTransformedRecord.read(protocolFactory.getProtocol(new TIOStreamTransport(new ByteArrayInputStream(lastTargetRecord.getPayload()))));
            srcDlsn = DLSN.deserializeBytes(lastTransformedRecord.getSrcDlsn());
            System.out.println("Last transformed record is " + srcDlsn);
        } catch (TException e) {
            System.err.println("Error on reading last transformed record");
            e.printStackTrace(System.err);
            srcDlsn = DLSN.InitialDLSN;
        }
    } catch (LogNotFoundException lnfe) {
        srcDlsn = DLSN.InitialDLSN;
    } catch (LogEmptyException lee) {
        srcDlsn = DLSN.InitialDLSN;
    }
    AsyncLogWriter targetWriter = FutureUtils.result(targetDlm.openAsyncLogWriter());
    try {
        readLoop(srcDlm, srcDlsn, targetWriter, replicationTransformer);
    } finally {
        FutureUtils.result(targetWriter.asyncClose(), Duration.apply(5, TimeUnit.SECONDS));
        targetDlm.close();
        srcDlm.close();
        namespace.close();
    }
}

Example 10 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class TestBKDistributedLogManager method testCheckLogExists.

@Test(timeout = 60000)
public void testCheckLogExists() throws Exception {
    String name = "distrlog-check-log-exists";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
    }
    writer.setReadyToFlush();
    writer.flushAndSync();
    writer.close();
    dlm.createOrUpdateMetadata(name.getBytes());
    assertEquals(name, new String(dlm.getMetadata()));
    URI uri = createDLMURI("/" + name);
    BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder().conf(conf).uri(uri).build();
    assertTrue(namespace.logExists(name));
    assertFalse(namespace.logExists("non-existent-log"));
    URI nonExistentUri = createDLMURI("/" + "non-existent-ns");
    DistributedLogNamespace nonExistentNS = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(nonExistentUri).build();
    assertFalse(nonExistentNS.logExists(name));
    int logCount = 0;
    Iterator<String> logIter = namespace.getLogs();
    while (logIter.hasNext()) {
        String log = logIter.next();
        logCount++;
        assertEquals(name, log);
    }
    assertEquals(1, logCount);
    for (Map.Entry<String, byte[]> logEntry : namespace.enumerateLogsWithMetadataInNamespace().entrySet()) {
        assertEquals(name, new String(logEntry.getValue()));
    }
}