Examples with StreamPartitionConverter com.twitter.distributedlog.service.streamset.StreamPartitionConverter used on opensource projects

Example 1 with StreamPartitionConverter

use of com.twitter.distributedlog.service.streamset.StreamPartitionConverter in project distributedlog by twitter.

the class DistributedLogServer method runServer.

public void runServer() throws ConfigurationException, IllegalArgumentException, IOException {
    if (!uri.isPresent()) {
        throw new IllegalArgumentException("No distributedlog uri provided.");
    }
    URI dlUri = URI.create(uri.get());
    DistributedLogConfiguration dlConf = new DistributedLogConfiguration();
    if (conf.isPresent()) {
        String configFile = conf.get();
        try {
            dlConf.loadConf(new File(configFile).toURI().toURL());
        } catch (ConfigurationException e) {
            throw new IllegalArgumentException("Failed to load distributedlog configuration from " + configFile + ".");
        } catch (MalformedURLException e) {
            throw new IllegalArgumentException("Failed to load distributedlog configuration from malformed " + configFile + ".");
        }
    }
    this.configExecutorService = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("DistributedLogService-Dyncfg-%d").setDaemon(true).build());
    // server configuration and dynamic configuration
    ServerConfiguration serverConf = new ServerConfiguration();
    serverConf.loadConf(dlConf);
    // overwrite the shard id if it is provided in the args
    if (shardId.isPresent()) {
        serverConf.setServerShardId(shardId.get());
    }
    serverConf.validate();
    DynamicDistributedLogConfiguration dynDlConf = getServiceDynConf(dlConf);
    logger.info("Starting stats provider : {}", statsProvider.getClass());
    statsProvider.start(dlConf);
    if (announceServerSet.isPresent() && announceServerSet.get()) {
        announcer = new ServerSetAnnouncer(dlUri, port.or(0), statsPort.or(0), shardId.or(0));
    } else {
        announcer = new NOPAnnouncer();
    }
    // Build the stream partition converter
    StreamPartitionConverter converter;
    try {
        converter = ReflectionUtils.newInstance(serverConf.getStreamPartitionConverterClass());
    } catch (ConfigurationException e) {
        logger.warn("Failed to load configured stream-to-partition converter. Fallback to use {}", IdentityStreamPartitionConverter.class.getName());
        converter = new IdentityStreamPartitionConverter();
    }
    StreamConfigProvider streamConfProvider = getStreamConfigProvider(dlConf, converter);
    // pre-run
    preRun(dlConf, serverConf);
    Pair<DistributedLogServiceImpl, Server> serverPair = runServer(serverConf, dlConf, dynDlConf, dlUri, converter, statsProvider, port.or(0), keepAliveLatch, statsReceiver, thriftmux.isPresent(), streamConfProvider);
    this.dlService = serverPair.getLeft();
    this.server = serverPair.getRight();
    // announce the service
    announcer.announce();
}

Example 2 with StreamPartitionConverter

use of com.twitter.distributedlog.service.streamset.StreamPartitionConverter in project distributedlog by twitter.

the class TestStreamManager method testCreateStream.

@Test
public void testCreateStream() throws Exception {
    Stream mockStream = mock(Stream.class);
    final String streamName = "stream1";
    when(mockStream.getStreamName()).thenReturn(streamName);
    StreamFactory mockStreamFactory = mock(StreamFactory.class);
    StreamPartitionConverter mockPartitionConverter = mock(StreamPartitionConverter.class);
    StreamConfigProvider mockStreamConfigProvider = mock(StreamConfigProvider.class);
    when(mockStreamFactory.create((String) any(), (DynamicDistributedLogConfiguration) any(), (StreamManager) any())).thenReturn(mockStream);
    DistributedLogNamespace dlNamespace = mock(DistributedLogNamespace.class);
    ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1);
    StreamManager streamManager = new StreamManagerImpl("", new DistributedLogConfiguration(), executorService, mockStreamFactory, mockPartitionConverter, mockStreamConfigProvider, dlNamespace);
    assertTrue(Await.ready(streamManager.createStreamAsync(streamName)).isReturn());
    verify(dlNamespace).createLog(streamName);
}

Example 3 with StreamPartitionConverter

use of com.twitter.distributedlog.service.streamset.StreamPartitionConverter in project distributedlog by twitter.

the class TestStreamManager method testCollectionMethods.

@Test(timeout = 60000)
public void testCollectionMethods() throws Exception {
    Stream mockStream = mock(Stream.class);
    when(mockStream.getStreamName()).thenReturn("stream1");
    when(mockStream.getPartition()).thenReturn(new Partition("stream1", 0));
    StreamFactory mockStreamFactory = mock(StreamFactory.class);
    StreamPartitionConverter mockPartitionConverter = mock(StreamPartitionConverter.class);
    StreamConfigProvider mockStreamConfigProvider = mock(StreamConfigProvider.class);
    when(mockStreamFactory.create((String) any(), (DynamicDistributedLogConfiguration) any(), (StreamManager) any())).thenReturn(mockStream);
    StreamManager streamManager = new StreamManagerImpl("", new DistributedLogConfiguration(), mockExecutorService, mockStreamFactory, mockPartitionConverter, mockStreamConfigProvider, mock(DistributedLogNamespace.class));
    assertFalse(streamManager.isAcquired("stream1"));
    assertEquals(0, streamManager.numAcquired());
    assertEquals(0, streamManager.numCached());
    streamManager.notifyAcquired(mockStream);
    assertTrue(streamManager.isAcquired("stream1"));
    assertEquals(1, streamManager.numAcquired());
    assertEquals(0, streamManager.numCached());
    streamManager.notifyReleased(mockStream);
    assertFalse(streamManager.isAcquired("stream1"));
    assertEquals(0, streamManager.numAcquired());
    assertEquals(0, streamManager.numCached());
    streamManager.notifyAcquired(mockStream);
    assertTrue(streamManager.isAcquired("stream1"));
    assertEquals(1, streamManager.numAcquired());
    assertEquals(0, streamManager.numCached());
    streamManager.notifyAcquired(mockStream);
    assertTrue(streamManager.isAcquired("stream1"));
    assertEquals(1, streamManager.numAcquired());
    assertEquals(0, streamManager.numCached());
    streamManager.notifyReleased(mockStream);
    assertFalse(streamManager.isAcquired("stream1"));
    assertEquals(0, streamManager.numAcquired());
    assertEquals(0, streamManager.numCached());
    streamManager.notifyReleased(mockStream);
    assertFalse(streamManager.isAcquired("stream1"));
    assertEquals(0, streamManager.numAcquired());
    assertEquals(0, streamManager.numCached());
}