Examples with DistributedLogClient com.twitter.distributedlog.service.DistributedLogClient used on opensource projects

Example 11 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class ConsoleProxyMultiWriter method main.

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }
    String finagleNameStr = args[0];
    final String streamList = args[1];
    DistributedLogClient client = DistributedLogClientBuilder.newBuilder().clientId(ClientId$.MODULE$.apply("console-proxy-writer")).name("console-proxy-writer").thriftmux(true).finagleNameStr(finagleNameStr).build();
    String[] streamNameList = StringUtils.split(streamList, ',');
    DistributedLogMultiStreamWriter multiStreamWriter = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList(streamNameList)).bufferSize(0).client(client).flushIntervalMs(0).firstSpeculativeTimeoutMs(10000).maxSpeculativeTimeoutMs(20000).requestTimeoutMs(50000).build();
    ConsoleReader reader = new ConsoleReader();
    String line;
    while ((line = reader.readLine(PROMPT_MESSAGE)) != null) {
        multiStreamWriter.write(ByteBuffer.wrap(line.getBytes(UTF_8))).addEventListener(new FutureEventListener<DLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                System.out.println("Encountered error on writing data");
                cause.printStackTrace(System.err);
                Runtime.getRuntime().exit(0);
            }

            @Override
            public void onSuccess(DLSN value) {
            // done
            }
        });
    }
    multiStreamWriter.close();
    client.close();
}

Example 12 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class RecordGenerator method main.

public static void main(String[] args) throws Exception {
    if (3 != args.length) {
        System.out.println(HELP);
        return;
    }
    String finagleNameStr = args[0];
    final String streamName = args[1];
    double rate = Double.parseDouble(args[2]);
    RateLimiter limiter = RateLimiter.create(rate);
    DistributedLogClient client = DistributedLogClientBuilder.newBuilder().clientId(ClientId$.MODULE$.apply("record-generator")).name("record-generator").thriftmux(true).finagleNameStr(finagleNameStr).build();
    final CountDownLatch keepAliveLatch = new CountDownLatch(1);
    final AtomicLong numWrites = new AtomicLong(0);
    final AtomicBoolean running = new AtomicBoolean(true);
    while (running.get()) {
        limiter.acquire();
        String record = "record-" + System.currentTimeMillis();
        client.write(streamName, ByteBuffer.wrap(record.getBytes(UTF_8))).addEventListener(new FutureEventListener<DLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                System.out.println("Encountered error on writing data");
                cause.printStackTrace(System.err);
                running.set(false);
                keepAliveLatch.countDown();
            }

            @Override
            public void onSuccess(DLSN value) {
                long numSuccesses = numWrites.incrementAndGet();
                if (numSuccesses % 100 == 0) {
                    System.out.println("Write " + numSuccesses + " records.");
                }
            }
        });
    }
    keepAliveLatch.await();
    client.close();
}

Example 13 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class SimpleBalancer method balance.

@Override
public void balance(int rebalanceWaterMark, double rebalanceTolerancePercentage, int rebalanceConcurrency, Optional<RateLimiter> rebalanceRateLimiter) {
    // get the ownership distributions from individual targets
    Map<SocketAddress, Set<String>> distribution1 = targetMonitor1.getStreamOwnershipDistribution();
    Map<SocketAddress, Set<String>> distribution2 = targetMonitor2.getStreamOwnershipDistribution();
    // get stream counts
    int proxyCount1 = distribution1.size();
    int streamCount1 = countNumberStreams(distribution1);
    int proxyCount2 = distribution2.size();
    int streamCount2 = countNumberStreams(distribution2);
    logger.info("'{}' has {} streams by {} proxies; while '{}' has {} streams by {} proxies.", new Object[] { target1, streamCount1, proxyCount1, target2, streamCount2, proxyCount2 });
    String source, target;
    Map<SocketAddress, Set<String>> srcDistribution;
    DistributedLogClient srcClient, targetClient;
    MonitorServiceClient srcMonitor, targetMonitor;
    int srcStreamCount, targetStreamCount;
    if (streamCount1 > streamCount2) {
        source = target1;
        srcStreamCount = streamCount1;
        srcClient = targetClient1;
        srcMonitor = targetMonitor1;
        srcDistribution = distribution1;
        target = target2;
        targetStreamCount = streamCount2;
        targetClient = targetClient2;
        targetMonitor = targetMonitor2;
    } else {
        source = target2;
        srcStreamCount = streamCount2;
        srcClient = targetClient2;
        srcMonitor = targetMonitor2;
        srcDistribution = distribution2;
        target = target1;
        targetStreamCount = streamCount1;
        targetClient = targetClient1;
        targetMonitor = targetMonitor1;
    }
    Map<String, Integer> loadDistribution = new HashMap<String, Integer>();
    loadDistribution.put(source, srcStreamCount);
    loadDistribution.put(target, targetStreamCount);
    // Calculate how many streams to be rebalanced from src region to target region
    int numStreamsToRebalance = BalancerUtils.calculateNumStreamsToRebalance(source, loadDistribution, rebalanceWaterMark, rebalanceTolerancePercentage);
    if (numStreamsToRebalance <= 0) {
        logger.info("No streams need to be rebalanced from '{}' to '{}'.", source, target);
        return;
    }
    StreamChooser streamChooser = LimitedStreamChooser.of(new CountBasedStreamChooser(srcDistribution), numStreamsToRebalance);
    StreamMover streamMover = new StreamMoverImpl(source, srcClient, srcMonitor, target, targetClient, targetMonitor);
    moveStreams(streamChooser, streamMover, rebalanceConcurrency, rebalanceRateLimiter);
}

Example 14 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class SimpleBalancer method balanceAll.

@Override
public void balanceAll(String source, int rebalanceConcurrency, Optional<RateLimiter> rebalanceRateLimiter) {
    String target;
    DistributedLogClient sourceClient, targetClient;
    MonitorServiceClient sourceMonitor, targetMonitor;
    if (target1.equals(source)) {
        sourceClient = targetClient1;
        sourceMonitor = targetMonitor1;
        target = target2;
        targetClient = targetClient2;
        targetMonitor = targetMonitor2;
    } else if (target2.equals(source)) {
        sourceClient = targetClient2;
        sourceMonitor = targetMonitor2;
        target = target1;
        targetClient = targetClient1;
        targetMonitor = targetMonitor1;
    } else {
        throw new IllegalArgumentException("Unknown target " + source);
    }
    // get the ownership distributions from individual targets
    Map<SocketAddress, Set<String>> distribution = sourceMonitor.getStreamOwnershipDistribution();
    if (distribution.isEmpty()) {
        return;
    }
    StreamChooser streamChooser = new CountBasedStreamChooser(distribution);
    StreamMover streamMover = new StreamMoverImpl(source, sourceClient, sourceMonitor, target, targetClient, targetMonitor);
    moveStreams(streamChooser, streamMover, rebalanceConcurrency, rebalanceRateLimiter);
}