Examples with DatabusHttpClientImpl com.linkedin.databus.client.DatabusHttpClientImpl used on opensource projects

Example 1 with DatabusHttpClientImpl

use of com.linkedin.databus.client.DatabusHttpClientImpl in project databus by linkedin.

the class IntegratedDummyDatabusConsumer method initConn.

public void initConn(List<String> sources) throws IOException, InvalidConfigException, DatabusClientException, DatabusException {
    StringBuilder sourcesString = new StringBuilder();
    boolean firstSrc = true;
    for (String source : sources) {
        if (!firstSrc)
            sourcesString.append(",");
        firstSrc = false;
        sourcesString.append(source);
    }
    _fileBasedCallback.init();
    ArrayList<DatabusBootstrapConsumer> bootstrapCallbacks = new ArrayList<DatabusBootstrapConsumer>();
    bootstrapCallbacks.add(this);
    ArrayList<DatabusStreamConsumer> streamCallbacks = new ArrayList<DatabusStreamConsumer>();
    streamCallbacks.add(this);
    DatabusHttpClientImpl.Config clientConfigBuilder = new DatabusHttpClientImpl.Config();
    clientConfigBuilder.getContainer().getJmx().setJmxServicePort(5555);
    clientConfigBuilder.getContainer().setId(545454);
    clientConfigBuilder.getContainer().setHttpPort(8082);
    clientConfigBuilder.getCheckpointPersistence().setType(ProviderType.FILE_SYSTEM.toString());
    clientConfigBuilder.getCheckpointPersistence().getFileSystem().setRootDirectory("./integratedconsumer-checkpoints");
    clientConfigBuilder.getCheckpointPersistence().setClearBeforeUse(true);
    clientConfigBuilder.getRuntime().getBootstrap().setEnabled(true);
    DatabusSourcesConnection.Config srcDefaultConfig = new DatabusSourcesConnection.Config();
    srcDefaultConfig.setFreeBufferThreshold((int) (_maxEventBufferSize * 0.05));
    srcDefaultConfig.setCheckpointThresholdPct(80);
    // 60 sec before retries (unless disabled)
    srcDefaultConfig.setConsumerTimeBudgetMs(_useConsumerTimeout ? 60000 : 0);
    // max of 3 retries
    srcDefaultConfig.getDispatcherRetries().setMaxRetryNum(3);
    clientConfigBuilder.setConnectionDefaults(srcDefaultConfig);
    DbusEventBuffer.Config eventBufferConfig = clientConfigBuilder.getConnectionDefaults().getEventBuffer();
    eventBufferConfig.setMaxSize(_maxEventBufferSize);
    eventBufferConfig.setAverageEventSize(_maxReadBufferSize);
    // TODO: the following shall be used once we can set eventbuffer for bootstrap through the config builder (DDSDBUS-82)
    // For now, bootstrap buffer will use the same config as relay buffer.
    //clientConfigBuilder.getConnectionDefaults().
    DatabusHttpClientImpl.StaticConfig clientConfig = clientConfigBuilder.build();
    ServerInfoBuilder relayBuilder = clientConfig.getRuntime().getRelay("1");
    relayBuilder.setName("DefaultRelay");
    relayBuilder.setHost("localhost");
    relayBuilder.setPort(9000);
    relayBuilder.setSources(sourcesString.toString());
    ServerInfoBuilder bootstrapBuilder = clientConfig.getRuntime().getBootstrap().getService("2");
    bootstrapBuilder.setName("DefaultBootstrapServices");
    bootstrapBuilder.setHost("localhost");
    bootstrapBuilder.setPort(6060);
    bootstrapBuilder.setSources(sourcesString.toString());
    _dbusClient = new DatabusHttpClientImpl(clientConfig);
    _dbusClient.registerDatabusStreamListener(this, sources, null);
    _dbusClient.registerDatabusBootstrapListener(this, sources, null);
    // add pause processor
    try {
        _dbusClient.getProcessorRegistry().register(ConsumerPauseRequestProcessor.COMMAND_NAME, new ConsumerPauseRequestProcessor(null, this));
    } catch (ProcessorRegistrationConflictException e) {
        LOG.error("Failed to register " + ConsumerPauseRequestProcessor.COMMAND_NAME);
    }
}

Example 2 with DatabusHttpClientImpl

use of com.linkedin.databus.client.DatabusHttpClientImpl in project databus by linkedin.

the class TestDatabusV2RegistrationImpl method testMultiConsumerRegistrationOps.

@Test
public void testMultiConsumerRegistrationOps() throws Exception {
    DatabusHttpClientImpl client = null;
    try {
        DatabusHttpClientImpl.Config clientConfig = new DatabusHttpClientImpl.Config();
        clientConfig.getContainer().getJmx().setRmiEnabled(false);
        clientConfig.getContainer().setHttpPort(12003);
        client = new DatabusHttpClientImpl(clientConfig);
        registerRelay(1, "relay1", new InetSocketAddress("localhost", 8888), "S1,S2", client);
        registerRelay(2, "relay2", new InetSocketAddress("localhost", 7777), "S1,S3", client);
        registerRelay(3, "relay1.1", new InetSocketAddress("localhost", 8887), "S1,S2", client);
        registerRelay(4, "relay3", new InetSocketAddress("localhost", 6666), "S3,S4,S5", client);
        TestConsumer listener1 = new TestConsumer();
        TestConsumer listener2 = new TestConsumer();
        List<DatabusCombinedConsumer> listeners = new ArrayList<DatabusCombinedConsumer>();
        listeners.add(listener1);
        listeners.add(listener2);
        DatabusRegistration reg = client.register(listeners, "S1", "S2");
        assertEquals("Registered State", RegistrationState.REGISTERED, reg.getState());
        assertEquals("Component Name", "Status_TestConsumer_922c5e28", reg.getStatus().getComponentName());
        assertEquals("Component Status", Status.INITIALIZING, reg.getStatus().getStatus());
        // Start
        boolean started = reg.start();
        assertEquals("Started", true, started);
        assertEquals("Registered State", RegistrationState.STARTED, reg.getState());
        assertEquals("Component Status", Status.RUNNING, reg.getStatus().getStatus());
        //Start again
        started = reg.start();
        assertEquals("Started", false, started);
        assertEquals("Registered State", RegistrationState.STARTED, reg.getState());
        // Pause
        reg.pause();
        assertEquals("Registered State", RegistrationState.PAUSED, reg.getState());
        assertEquals("Component Status", Status.PAUSED, reg.getStatus().getStatus());
        // resume
        reg.resume();
        assertEquals("Registered State", RegistrationState.RESUMED, reg.getState());
        assertEquals("Component Status", Status.RUNNING, reg.getStatus().getStatus());
        // suspend due to error
        reg.suspendOnError(new Exception("dummy"));
        assertEquals("Registered State", RegistrationState.SUSPENDED_ON_ERROR, reg.getState());
        assertEquals("Component Status", Status.SUSPENDED_ON_ERROR, reg.getStatus().getStatus());
        // SHutdown
        reg.shutdown();
        assertEquals("Registered State", RegistrationState.SHUTDOWN, reg.getState());
        assertEquals("Component Status", Status.SHUTDOWN, reg.getStatus().getStatus());
        reg.deregister();
    } finally {
        if (null != client)
            client.shutdown();
    }
}

Example 3 with DatabusHttpClientImpl

use of com.linkedin.databus.client.DatabusHttpClientImpl in project databus by linkedin.

the class TestDatabusV2ClusterRegistrationImpl method testRegistrationStartFromInvalidState.

@Test
public void testRegistrationStartFromInvalidState() throws Exception {
    DatabusHttpClientImpl client = null;
    try {
        DatabusHttpClientImpl.Config clientConfig = new DatabusHttpClientImpl.Config();
        clientConfig.getContainer().getJmx().setRmiEnabled(false);
        clientConfig.getContainer().setHttpPort(12003);
        client = new DatabusHttpClientImpl(clientConfig);
        registerRelay(1, "relay1", new InetSocketAddress("localhost", 8888), "S1,S2", client);
        registerRelay(2, "relay2", new InetSocketAddress("localhost", 7777), "S1,S3", client);
        registerRelay(3, "relay1.1", new InetSocketAddress("localhost", 8887), "S1,S2", client);
        registerRelay(4, "relay3", new InetSocketAddress("localhost", 6666), "S3,S4,S5", client);
        TestDbusPartitionListener listener = new TestDbusPartitionListener();
        StaticConfig ckptConfig = new StaticConfig("localhost:1356", "dummy", 1, 1);
        DbusClusterInfo clusterInfo = new DbusClusterInfo("dummy", 10, 1);
        DatabusV2ClusterRegistrationImpl reg = new TestableDatabusV2ClusterRegistrationImpl(null, client, ckptConfig, clusterInfo, new TestDbusClusterConsumerFactory(), new TestDbusServerSideFilterFactory(), listener, "S1", "S3");
        try {
            // Invoking start from INIT state in illegal
            reg.start();
            Assert.fail();
        } catch (IllegalStateException e) {
        }
        try {
            // Invoking start from DEREGISTERED state in illegal
            reg.onRegister();
            reg.start();
            reg.shutdown();
            reg.deregister();
            reg.start();
            Assert.fail();
        } catch (IllegalStateException e) {
        }
    } finally {
        if (null != client) {
            client.shutdown();
        }
    }
}

Example 4 with DatabusHttpClientImpl

use of com.linkedin.databus.client.DatabusHttpClientImpl in project databus by linkedin.

the class TestDatabusV2ClusterRegistrationImpl method testRegistration.

@Test
public void testRegistration() throws Exception {
    DatabusHttpClientImpl client = null;
    try {
        DatabusHttpClientImpl.Config clientConfig = new DatabusHttpClientImpl.Config();
        clientConfig.getContainer().getJmx().setRmiEnabled(false);
        clientConfig.getContainer().setHttpPort(12003);
        client = new DatabusHttpClientImpl(clientConfig);
        registerRelay(1, "relay1", new InetSocketAddress("localhost", 8888), "S1,S2", client);
        registerRelay(2, "relay2", new InetSocketAddress("localhost", 7777), "S1,S3", client);
        registerRelay(3, "relay1.1", new InetSocketAddress("localhost", 8887), "S1,S2", client);
        registerRelay(4, "relay3", new InetSocketAddress("localhost", 6666), "S3,S4,S5", client);
        TestDbusPartitionListener listener = new TestDbusPartitionListener();
        StaticConfig ckptConfig = new StaticConfig("localhost:1356", "dummy", 1, 1);
        DbusClusterInfo clusterInfo = new DbusClusterInfo("dummy", 10, 1);
        DatabusV2ClusterRegistrationImpl reg = new TestableDatabusV2ClusterRegistrationImpl(null, client, ckptConfig, clusterInfo, new TestDbusClusterConsumerFactory(), new TestDbusServerSideFilterFactory(), listener, "S1", "S3");
        reg.onRegister();
        // Start
        reg.start();
        assertEquals("State CHeck", reg.getState(), RegistrationState.STARTED);
        // Add Partition(s)
        reg.onGainedPartitionOwnership(1);
        assertEquals("Listener called ", listener.isAddPartitionCalled(1), true);
        reg.onGainedPartitionOwnership(2);
        assertEquals("Listener called ", listener.isAddPartitionCalled(2), true);
        assertEquals("Partition Regs size ", 2, reg.getPartitionRegs().size());
        reg.onGainedPartitionOwnership(3);
        assertEquals("Listener called ", listener.isAddPartitionCalled(3), true);
        assertEquals("Partition Regs size ", 3, reg.getPartitionRegs().size());
        reg.onGainedPartitionOwnership(4);
        assertEquals("Listener called ", listener.isAddPartitionCalled(4), true);
        //duplicate call
        listener.clearCallbacks();
        reg.onGainedPartitionOwnership(4);
        assertEquals("Listener called ", listener.isAddPartitionCalled(4), false);
        assertEquals("Partition Regs size ", 4, reg.getPartitionRegs().size());
        List<String> gotPartitionList = new ArrayList<String>();
        for (DbusPartitionInfo p : reg.getPartitions()) gotPartitionList.add(p.toString());
        Collections.sort(gotPartitionList);
        assertEquals("Partitions Check", gotPartitionList.toString(), "[1, 2, 3, 4]");
        // Drop Partitions
        reg.onLostPartitionOwnership(1);
        gotPartitionList.clear();
        for (DbusPartitionInfo p : reg.getPartitions()) gotPartitionList.add(p.toString());
        Collections.sort(gotPartitionList);
        assertEquals("Partitions Check", "[2, 3, 4]", gotPartitionList.toString());
        assertEquals("Listener called ", true, listener.isDropPartitionCalled(1));
        reg.onLostPartitionOwnership(2);
        assertEquals("Listener called ", true, listener.isDropPartitionCalled(2));
        //duplicate call
        listener.clearCallbacks();
        reg.onLostPartitionOwnership(2);
        assertEquals("Listener called ", false, listener.isDropPartitionCalled(2));
        assertEquals("Partitions Check", "[3, 4]", reg.getPartitions().toString());
        assertEquals("Partition Regs size ", 2, reg.getPartitionRegs().size());
        reg.onReset(3);
        assertEquals("Listener called ", true, listener.isDropPartitionCalled(3));
        //duplicate call
        listener.clearCallbacks();
        reg.onReset(3);
        assertEquals("Listener called ", false, listener.isDropPartitionCalled(3));
        assertEquals("Partitions Check", "[4]", reg.getPartitions().toString());
        assertEquals("Partition Regs size ", 1, reg.getPartitionRegs().size());
        reg.onError(4);
        assertEquals("Listener called ", true, listener.isDropPartitionCalled(4));
        //duplicate call
        listener.clearCallbacks();
        reg.onError(4);
        assertEquals("Listener called ", false, listener.isDropPartitionCalled(4));
        assertEquals("Partitions Check", "[]", reg.getPartitions().toString());
        assertEquals("Partition Regs size ", 0, reg.getPartitionRegs().size());
        // Add Partiton 1 again
        listener.clearCallbacks();
        reg.onGainedPartitionOwnership(1);
        assertEquals("Listener called ", listener.isAddPartitionCalled(1), true);
        assertEquals("Partition Regs size ", 1, reg.getPartitionRegs().size());
        assertEquals("Child State CHeck", reg.getPartitionRegs().values().iterator().next().getState(), RegistrationState.STARTED);
        // Pausing
        reg.pause();
        assertEquals("State CHeck", reg.getState(), RegistrationState.PAUSED);
        assertEquals("Child State CHeck", reg.getPartitionRegs().values().iterator().next().getState(), RegistrationState.PAUSED);
        // Resume
        reg.resume();
        assertEquals("State CHeck", reg.getState(), RegistrationState.RESUMED);
        assertEquals("Child State CHeck", reg.getPartitionRegs().values().iterator().next().getState(), RegistrationState.RESUMED);
        // Suspended
        reg.suspendOnError(null);
        assertEquals("State CHeck", reg.getState(), RegistrationState.SUSPENDED_ON_ERROR);
        assertEquals("Child State CHeck", reg.getPartitionRegs().values().iterator().next().getState(), RegistrationState.SUSPENDED_ON_ERROR);
        // resume
        reg.resume();
        assertEquals("State CHeck", reg.getState(), RegistrationState.RESUMED);
        assertEquals("Child State CHeck", reg.getPartitionRegs().values().iterator().next().getState(), RegistrationState.RESUMED);
        // Active node change notification
        List<String> newActiveNodes = new ArrayList<String>();
        newActiveNodes.add("localhost:7070");
        newActiveNodes.add("localhost:8080");
        newActiveNodes.add("localhost:9090");
        reg.onInstanceChange(newActiveNodes);
        assertEquals("Active Nodes", newActiveNodes, reg.getCurrentActiveNodes());
        newActiveNodes.remove(2);
        reg.onInstanceChange(newActiveNodes);
        assertEquals("Active Nodes", newActiveNodes, reg.getCurrentActiveNodes());
        newActiveNodes.add("localhost:1010");
        reg.onInstanceChange(newActiveNodes);
        assertEquals("Active Nodes", newActiveNodes, reg.getCurrentActiveNodes());
        // Partition Mapping change notification
        Map<Integer, String> activePartitionMap = new HashMap<Integer, String>();
        for (int i = 0; i < 10; i++) {
            String node = null;
            if (i % 2 == 0)
                node = "localhost:8080";
            else
                node = "localhost:7070";
            activePartitionMap.put(i, node);
        }
        reg.onPartitionMappingChange(activePartitionMap);
        assertEquals("Partition Mapping Check", activePartitionMap, reg.getActivePartitionMap());
        activePartitionMap.remove(9);
        reg.onPartitionMappingChange(activePartitionMap);
        assertEquals("Partition Mapping Check", activePartitionMap, reg.getActivePartitionMap());
        activePartitionMap.put(9, "localhost:8708");
        reg.onPartitionMappingChange(activePartitionMap);
        assertEquals("Partition Mapping Check", activePartitionMap, reg.getActivePartitionMap());
        // shutdown
        reg.shutdown();
        assertEquals("State Check", reg.getState(), RegistrationState.SHUTDOWN);
        assertEquals("Child State CHeck", reg.getPartitionRegs().values().iterator().next().getState(), RegistrationState.SHUTDOWN);
        // Operations during shutdown state
        boolean gotException = false;
        try {
            reg.onGainedPartitionOwnership(1);
        } catch (IllegalStateException ex) {
            gotException = true;
        }
        assertEquals("Exception", true, gotException);
        gotException = false;
        try {
            reg.onLostPartitionOwnership(1);
        } catch (IllegalStateException ex) {
            gotException = true;
        }
        assertEquals("Exception", true, gotException);
        gotException = false;
        try {
            reg.pause();
        } catch (IllegalStateException ex) {
            gotException = true;
        }
        assertEquals("Exception", true, gotException);
        gotException = false;
        try {
            reg.suspendOnError(null);
        } catch (IllegalStateException ex) {
            gotException = true;
        }
        assertEquals("Exception", true, gotException);
        gotException = false;
        try {
            reg.resume();
        } catch (IllegalStateException ex) {
            gotException = true;
        }
        assertEquals("Exception", true, gotException);
        // deregister
        reg.deregister();
        assertEquals("State Check", reg.getState(), RegistrationState.DEREGISTERED);
        assertEquals("Child State CHeck", 0, reg.getPartitionRegs().size());
    } finally {
        if (null != client)
            client.shutdown();
    }
}

Example 5 with DatabusHttpClientImpl

use of com.linkedin.databus.client.DatabusHttpClientImpl in project databus by linkedin.

the class TestDatabusV2RegistrationImpl method testOneConsumerRegistrationOps.

@Test
public void testOneConsumerRegistrationOps() throws Exception {
    DatabusHttpClientImpl client = null;
    try {
        DatabusHttpClientImpl.Config clientConfig = new DatabusHttpClientImpl.Config();
        clientConfig.getContainer().getJmx().setRmiEnabled(false);
        clientConfig.getContainer().setHttpPort(12003);
        client = new DatabusHttpClientImpl(clientConfig);
        registerRelay(1, "relay1", new InetSocketAddress("localhost", 8888), "S1,S2", client);
        registerRelay(2, "relay2", new InetSocketAddress("localhost", 7777), "S1,S3", client);
        registerRelay(3, "relay1.1", new InetSocketAddress("localhost", 8887), "S1,S2", client);
        registerRelay(4, "relay3", new InetSocketAddress("localhost", 6666), "S3,S4,S5", client);
        TestConsumer listener1 = new TestConsumer();
        DatabusRegistration reg = client.register(listener1, "S1", "S3");
        assertEquals("Registered State", RegistrationState.REGISTERED, reg.getState());
        assertEquals("Component Name", "Status_TestConsumer_a62d57a7", reg.getStatus().getComponentName());
        assertEquals("Component Status", Status.INITIALIZING, reg.getStatus().getStatus());
        // Start
        boolean started = reg.start();
        assertEquals("Started", true, started);
        assertEquals("Registered State", RegistrationState.STARTED, reg.getState());
        assertEquals("Component Status", Status.RUNNING, reg.getStatus().getStatus());
        //Start again
        started = reg.start();
        assertEquals("Started", false, started);
        assertEquals("Registered State", RegistrationState.STARTED, reg.getState());
        // Pause
        reg.pause();
        assertEquals("Registered State", RegistrationState.PAUSED, reg.getState());
        assertEquals("Component Status", Status.PAUSED, reg.getStatus().getStatus());
        // resume
        reg.resume();
        assertEquals("Registered State", RegistrationState.RESUMED, reg.getState());
        assertEquals("Component Status", Status.RUNNING, reg.getStatus().getStatus());
        // suspend due to error
        reg.suspendOnError(new Exception("dummy"));
        assertEquals("Registered State", RegistrationState.SUSPENDED_ON_ERROR, reg.getState());
        assertEquals("Component Status", Status.SUSPENDED_ON_ERROR, reg.getStatus().getStatus());
        // SHutdown
        reg.shutdown();
        assertEquals("Registered State", RegistrationState.SHUTDOWN, reg.getState());
        assertEquals("Component Status", Status.SHUTDOWN, reg.getStatus().getStatus());
        // Duplicate regId
        DatabusRegistration reg2 = client.register(listener1, "S1", "S3");
        boolean isException = false;
        try {
            reg2.withRegId(reg.getRegistrationId());
        } catch (DatabusClientException ex) {
            isException = true;
        }
        assertEquals("Exception expected", true, isException);
        reg2.deregister();
        reg.deregister();
    } finally {
        if (null != client)
            client.shutdown();
    }
}