Examples with Checkpoint com.linkedin.databus.core.Checkpoint used on opensource projects

Example 31 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class MockBootstrapConnection method testTransition_RequestStreamToStreamRequestSuccess.

@Test
public /**
 * Test bootstrap transition: Request_Stream to Stream_request_success
 */
void testTransition_RequestStreamToStreamRequestSuccess() throws Exception {
    BootstrapPullThread bsPuller = createBootstrapPullThread(false, false, false, 12000, 1);
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 0L);
    // TODO remove
    // cp.setSnapshotSource("source1");
    // cp.setCatchupSource("source1");
    // cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    cp.setBootstrapServerInfo(_serverInfoName);
    bsPuller.getComponentStatus().start();
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.REQUEST_START_SCN, StateId.START_SCN_RESPONSE_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    Map<Long, List<RegisterResponseEntry>> entries = new HashMap<Long, List<RegisterResponseEntry>>();
    entries.put(1L, new ArrayList<RegisterResponseEntry>());
    connState.setSourcesSchemas(entries);
    testTransitionCase(bsPuller, StateId.START_SCN_RESPONSE_SUCCESS, StateId.REQUEST_STREAM, null);
    bsPuller.getMessageQueue().clear();
    connState.getSourcesNameMap().put("source1", new IdNamePair(1L, "source1"));
    connState.getSourceIdMap().put(1L, new IdNamePair(1L, "source1"));
    testTransitionCase(bsPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS, null);
}

Example 32 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class MockBootstrapConnection method testTransition_RequestStartScnToStartScnSent.

@Test
public /**
 * Test bootstrap transition: Request_Start_Scn to Start_Scn_Sent
 */
void testTransition_RequestStartScnToStartScnSent() throws Exception {
    BootstrapPullThread bsPuller = createBootstrapPullThread(false, false, true);
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 0L);
    bsPuller.getComponentStatus().start();
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.REQUEST_START_SCN, StateId.START_SCN_REQUEST_SENT, "", null);
}

Example 33 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class MockBootstrapConnection method testTransition_HappyPathWithStartScn.

@Test
public /**
 * Test BOOTSTRAP transitions - Happy Path with startScn
 */
void testTransition_HappyPathWithStartScn() throws Exception {
    BootstrapPullThread bsPuller = createBootstrapPullThread(false, false, false);
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 50L);
    // TODO remove
    // cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    cp.setBootstrapServerInfo(_serverInfoName);
    cp.setBootstrapStartScn(100L);
    bsPuller.getComponentStatus().start();
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    // cp.setBootstrapSinceScn(50L);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_STREAM, cp);
    ServerInfo bstServerInfo = ServerInfo.buildServerInfoFromHostPort(cp.getBootstrapServerInfo(), DbusConstants.HOSTPORT_DELIMITER);
    Assert.assertEquals(bsPuller.getConnectionState().getCurrentBSServerInfo(), bstServerInfo);
    Assert.assertEquals(cp.getBootstrapStartScn().longValue(), 100L, "Cleared Bootstrap StartSCN");
    Assert.assertEquals(cp.getBootstrapSinceScn().longValue(), 50L, "Cleared Bootstrap SinceSCN");
    String actualHost = bsPuller.getCurentServer().getAddress().getHostName();
    int actualPort = bsPuller.getCurentServer().getAddress().getPort();
    Assert.assertEquals(actualHost, _host, "Current Server Host Check");
    Assert.assertEquals(actualPort, _port, "Server Port Check");
    int numRetries = bsPuller.getRetriesBeforeCkptCleanup().getRemainingRetriesNum();
    Assert.assertEquals(numRetries, 1000, "NumRetries Check");
}

Example 34 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class MockBootstrapConnection method testTransition_RestartDueToMalformedServerInfo.

@Test
public /**
 * Test BOOTSTRAP transition - Bootstrap Restart since malformed serverInfo
 */
void testTransition_RestartDueToMalformedServerInfo() throws Exception {
    final String dummyHost = "NonExistantHost";
    // no delim
    final String malformedServerInfoName = dummyHost + _port;
    BootstrapPullThread bsPuller = createBootstrapPullThread(false, false, false);
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 50L);
    // TODO remove
    // cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    // cp.setBootstrapSinceScn(900L);
    cp.setBootstrapStartScn(1111L);
    cp.setBootstrapServerInfo(malformedServerInfoName);
    bsPuller.getComponentStatus().start();
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    // cp.setBootstrapSinceScn(50L);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    Assert.assertEquals(cp.getBootstrapStartScn().longValue(), Checkpoint.UNSET_BOOTSTRAP_START_SCN, "Cleared Bootstrap StartSCN");
    Assert.assertEquals(cp.getBootstrapSinceScn().longValue(), 50L, "Cleared Bootstrap SinceSCN");
    int numRetries = bsPuller.getRetriesBeforeCkptCleanup().getRemainingRetriesNum();
    Assert.assertEquals(numRetries, 1000, "NumRetries Check");
}

Example 35 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class MockBootstrapConnection method testBootstrapPendingEvent.

// Make sure that we suspend on error when we get the x-dbus-pending-size header with a size that is
// larger than our dbusevent size.
@Test
public void testBootstrapPendingEvent() throws Exception {
    List<String> sources = Arrays.asList("source1");
    Properties clientProps = new Properties();
    clientProps.setProperty("client.container.httpPort", "0");
    clientProps.setProperty("client.container.jmx.rmiEnabled", "false");
    clientProps.setProperty("client.runtime.bootstrap.enabled", "true");
    clientProps.setProperty("client.runtime.bootstrap.service(1).name", "bs1");
    clientProps.setProperty("client.runtime.bootstrap.service(1).host", "localhost");
    clientProps.setProperty("client.runtime.bootstrap.service(1).port", "10001");
    clientProps.setProperty("client.runtime.bootstrap.service(1).sources", "source1");
    clientProps.setProperty("client.runtime.relay(1).name", "relay1");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "3");
    DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
    ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("client.", clientConfBuilder);
    configLoader.loadConfig(clientProps);
    DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
    DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConf);
    client.registerDatabusBootstrapListener(new LoggingConsumer(), null, "source1");
    Assert.assertNotNull(client, "client instantiation failed");
    DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
    // we keep the index of the next server we expect to see
    AtomicInteger serverIdx = new AtomicInteger(-1);
    List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
    sourcesResponse.add(new IdNamePair(1L, "source1"));
    Map<Long, List<RegisterResponseEntry>> registerResponse = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> regResponse = new ArrayList<RegisterResponseEntry>();
    regResponse.add(new RegisterResponseEntry(1L, (short) 1, SCHEMA$.toString()));
    registerResponse.put(1L, regResponse);
    ChunkedBodyReadableByteChannel channel = EasyMock.createMock(ChunkedBodyReadableByteChannel.class);
    // getting the pending-event-size header is called twice, once for checking and once for logging.
    EasyMock.expect(channel.getMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE)).andReturn("1000000").times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error-cause")).andReturn(null).times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error")).andReturn(null).times(2);
    EasyMock.replay(channel);
    DbusEventBuffer dbusBuffer = EasyMock.createMock(DbusEventBuffer.class);
    dbusBuffer.endEvents(false, -1, false, false, null);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(dbusBuffer.injectEvent(EasyMock.<DbusEventInternalReadable>notNull())).andReturn(true).anyTimes();
    EasyMock.expect(dbusBuffer.getEventSerializationVersion()).andReturn(DbusEventFactory.DBUS_EVENT_V1).anyTimes();
    EasyMock.expect(dbusBuffer.getMaxReadBufferCapacity()).andReturn(600).times(2);
    EasyMock.expect(dbusBuffer.getBufferFreeReadSpace()).andReturn(600000).times(2);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull())).andReturn(1).times(1);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull(), EasyMock.<List<InternalDatabusEventsListener>>notNull(), EasyMock.<DbusEventsStatisticsCollector>isNull())).andReturn(0).times(1);
    EasyMock.replay(dbusBuffer);
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
    // This guy succeeds on /sources but fails on /register
    MockBootstrapConnection mockSuccessConn = new MockBootstrapConnection(10, 10, channel, serverIdx, false);
    DatabusBootstrapConnectionFactory mockConnFactory = org.easymock.EasyMock.createMock("mockRelayFactory", DatabusBootstrapConnectionFactory.class);
    // each server should be tried MAX_RETRIES time until all retries are exhausted
    EasyMock.expect(mockConnFactory.createConnection(EasyMock.<ServerInfo>notNull(), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockSuccessConn).anyTimes();
    List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
    DatabusSourcesConnection sourcesConn2 = EasyMock.createMock(DatabusSourcesConnection.class);
    EasyMock.expect(sourcesConn2.getSourcesNames()).andReturn(Arrays.asList("source1")).anyTimes();
    EasyMock.expect(sourcesConn2.getSubscriptions()).andReturn(sourcesSubList).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionConfig()).andReturn(srcConnConf).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionStatus()).andReturn(new DatabusComponentStatus("dummy")).anyTimes();
    EasyMock.expect(sourcesConn2.getLocalRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getUnifiedClientStats()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapConnFactory()).andReturn(mockConnFactory).anyTimes();
    EasyMock.expect(sourcesConn2.loadPersistentCheckpoint()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getDataEventsBuffer()).andReturn(dbusBuffer).anyTimes();
    EasyMock.expect(sourcesConn2.isBootstrapEnabled()).andReturn(true).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapRegistrations()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapServices()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapEventsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.makeThreadSafe(mockConnFactory, true);
    EasyMock.makeThreadSafe(sourcesConn2, true);
    EasyMock.replay(mockConnFactory);
    EasyMock.replay(sourcesConn2);
    BootstrapPullThread bsPuller = new BootstrapPullThread("RelayPuller", sourcesConn2, dbusBuffer, connStateFactory, clientRtConf.getBootstrap().getServicesSet(), new ArrayList<DbusKeyCompositeFilterConfig>(), clientConf.getPullerBufferUtilizationPct(), ManagementFactory.getPlatformMBeanServer(), new DbusEventV2Factory(), null, null);
    mockSuccessConn.setCallback(bsPuller);
    bsPuller.getComponentStatus().start();
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 0L);
    // TODO remove
    // cp.setSnapshotSource("source1");
    // cp.setCatchupSource("source1");
    // cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.REQUEST_START_SCN, StateId.START_SCN_RESPONSE_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    Map<Long, List<RegisterResponseEntry>> entries = new HashMap<Long, List<RegisterResponseEntry>>();
    entries.put(1L, new ArrayList<RegisterResponseEntry>());
    connState.setSourcesSchemas(entries);
    connState.setCurrentBSServerInfo(bsPuller.getCurentServer());
    testTransitionCase(bsPuller, StateId.START_SCN_RESPONSE_SUCCESS, StateId.REQUEST_STREAM, null);
    bsPuller.getMessageQueue().clear();
    connState.getSourcesNameMap().put("source1", new IdNamePair(1L, "source1"));
    connState.getSourceIdMap().put(1L, new IdNamePair(1L, "source1"));
    testTransitionCase(bsPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_REQUEST_SUCCESS, "SUSPEND_ON_ERROR", null);
    EasyMock.verify(channel);
    EasyMock.verify(sourcesConn2);
    EasyMock.verify(dbusBuffer);
    EasyMock.verify(channel);
    EasyMock.verify(mockConnFactory);
}