Examples with DistributedLogNamespace com.twitter.distributedlog.namespace.DistributedLogNamespace used on opensource projects

Example 16 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class StreamBenchmark method run.

protected void run(String[] args) throws Exception {
    logger.info("Parsing arguments for benchmark : {}", args);
    // parse command line
    parseCommandLine(args);
    statsProvider.start(conf);
    // run the benchmark
    StatsLogger statsLogger = statsProvider.getStatsLogger("dl");
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).statsLogger(statsLogger).build();
    try {
        benchmark(namespace, streamName, statsProvider.getStatsLogger("benchmark"));
    } finally {
        namespace.close();
        statsProvider.stop();
    }
}

Example 17 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class MultiReader method main.

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }
    String dlUriStr = args[0];
    final String streamList = args[1];
    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    String[] streamNameList = StringUtils.split(streamList, ',');
    DistributedLogManager[] managers = new DistributedLogManager[streamNameList.length];
    for (int i = 0; i < managers.length; i++) {
        String streamName = streamNameList[i];
        // open the dlm
        System.out.println("Opening log stream " + streamName);
        managers[i] = namespace.openLog(streamName);
    }
    final CountDownLatch keepAliveLatch = new CountDownLatch(1);
    for (DistributedLogManager dlm : managers) {
        final DistributedLogManager manager = dlm;
        dlm.getLastLogRecordAsync().addEventListener(new FutureEventListener<LogRecordWithDLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                if (cause instanceof LogNotFoundException) {
                    System.err.println("Log stream " + manager.getStreamName() + " is not found. Please create it first.");
                    keepAliveLatch.countDown();
                } else if (cause instanceof LogEmptyException) {
                    System.err.println("Log stream " + manager.getStreamName() + " is empty.");
                    readLoop(manager, DLSN.InitialDLSN, keepAliveLatch);
                } else {
                    System.err.println("Encountered exception on process stream " + manager.getStreamName());
                    keepAliveLatch.countDown();
                }
            }

            @Override
            public void onSuccess(LogRecordWithDLSN record) {
                readLoop(manager, record.getDlsn(), keepAliveLatch);
            }
        });
    }
    keepAliveLatch.await();
    for (DistributedLogManager dlm : managers) {
        dlm.close();
    }
    namespace.close();
}

Example 18 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class StreamRewinder method main.

public static void main(String[] args) throws Exception {
    if (3 != args.length) {
        System.out.println(HELP);
        return;
    }
    String dlUriStr = args[0];
    final String streamName = args[1];
    final int rewindSeconds = Integer.parseInt(args[2]);
    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);
    try {
        readLoop(dlm, rewindSeconds);
    } finally {
        dlm.close();
        namespace.close();
    }
}

Example 19 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class TailReader method main.

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }
    String dlUriStr = args[0];
    final String streamName = args[1];
    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);
    // get the last record
    LogRecordWithDLSN lastRecord;
    DLSN dlsn;
    try {
        lastRecord = dlm.getLastLogRecord();
        dlsn = lastRecord.getDlsn();
        readLoop(dlm, dlsn);
    } catch (LogNotFoundException lnfe) {
        System.err.println("Log stream " + streamName + " is not found. Please create it first.");
        return;
    } catch (LogEmptyException lee) {
        System.err.println("Log stream " + streamName + " is empty.");
        dlsn = DLSN.InitialDLSN;
        readLoop(dlm, dlsn);
    } finally {
        dlm.close();
        namespace.close();
    }
}

Example 20 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class ReaderWithOffsets method main.

public static void main(String[] args) throws Exception {
    if (4 != args.length) {
        System.out.println(HELP);
        return;
    }
    String dlUriStr = args[0];
    final String streamName = args[1];
    final String readerId = args[2];
    final String offsetStoreFile = args[3];
    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);
    // open the offset store
    Options options = new Options();
    options.createIfMissing(true);
    final DB offsetDB = factory.open(new File(offsetStoreFile), options);
    final AtomicReference<DLSN> lastDLSN = new AtomicReference<DLSN>(null);
    // offset updater
    final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    executorService.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            if (null != lastDLSN.get()) {
                offsetDB.put(readerId.getBytes(UTF_8), lastDLSN.get().serializeBytes());
                System.out.println("Updated reader " + readerId + " offset to " + lastDLSN.get());
            }
        }
    }, 10, 10, TimeUnit.SECONDS);
    try {
        byte[] offset = offsetDB.get(readerId.getBytes(UTF_8));
        DLSN dlsn;
        if (null == offset) {
            dlsn = DLSN.InitialDLSN;
        } else {
            dlsn = DLSN.deserializeBytes(offset);
        }
        readLoop(dlm, dlsn, lastDLSN);
    } finally {
        offsetDB.close();
        dlm.close();
        namespace.close();
    }
}