Examples with LLCSegmentName com.linkedin.pinot.common.utils.LLCSegmentName used on opensource projects

Example 21 with LLCSegmentName

use of com.linkedin.pinot.common.utils.LLCSegmentName in project pinot by linkedin.

the class SegmentCompletionTest method testStoppedConsumeBeforeHold.

@Test
public void testStoppedConsumeBeforeHold() throws Exception {
    SegmentCompletionProtocol.Response response;
    Request.Params params;
    final String reason = "IAmLazy";
    // s1 stops consuming at t = 5;
    segmentCompletionMgr._secconds = 5;
    params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr).withReason(reason);
    response = segmentCompletionMgr.segmentStoppedConsuming(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED);
    Assert.assertEquals(new LLCSegmentName(segmentNameStr), segmentManager._stoppedSegmentName);
    Assert.assertEquals(s1, segmentManager._stoppedInstance);
    segmentManager._stoppedSegmentName = null;
    segmentManager._stoppedInstance = null;
    // s2 sends offset of 40, gets HOLD
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD);
    // s3 sends offset of 30, gets catchup to 40, s2 should have been decided as the winner now
    // since we are never expecting to hear back from s1
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s3).withOffset(s3Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.CATCH_UP);
    Assert.assertEquals(response.getOffset(), s2Offset);
    // s3 happens to come back (after catchup to s2offset) before s2 should get a hold since s2 has been decided as
    // the winner.
    // TODO Can optimize here since s2 is not notified yet.
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s3).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD);
    Assert.assertEquals(response.getOffset(), s2Offset);
    // s2 is asked to commit.
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT);
    // s2 executes a succesful commit
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentCommitStart(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_CONTINUE);
    segmentCompletionMgr._secconds += 5;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentCommitEnd(params, true);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_SUCCESS);
    // Now the FSM should have disappeared from the map
    Assert.assertFalse(fsmMap.containsKey(segmentNameStr));
    // Now if s3 or s1 come back, they are asked to keep the segment they have.
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.KEEP);
    // And the FSM should be removed.
    Assert.assertFalse(fsmMap.containsKey(segmentNameStr));
}

Example 22 with LLCSegmentName

use of com.linkedin.pinot.common.utils.LLCSegmentName in project pinot by linkedin.

the class SegmentCompletionTest method testFailedSlowCommit.

@Test
public void testFailedSlowCommit() throws Exception {
    SegmentCompletionProtocol.Response response;
    Request.Params params;
    final String tableName = new LLCSegmentName(segmentNameStr).getTableName();
    // s1 sends offset of 20, gets HOLD at t = 5s;
    final long startTime = 5;
    segmentCompletionMgr._secconds = startTime;
    params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD);
    // s2 sends offset of 40, gets HOLD
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD);
    // s3 sends offset of 30, gets catchup to 40
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s3).withOffset(s3Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.CATCH_UP);
    Assert.assertEquals(response.getOffset(), s2Offset);
    // Now s1 comes back, and is asked to catchup.
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.CATCH_UP);
    // s2 is asked to commit.
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT);
    long commitTimeSec = response.getBuildTimeSeconds();
    Assert.assertTrue(commitTimeSec > 0);
    // Fast forward to one second before commit time, and send a lease renewal request for 20s
    segmentCompletionMgr._secconds = startTime + commitTimeSec - 1;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr).withExtTimeSec(20);
    response = segmentCompletionMgr.extendBuildTime(params);
    Assert.assertEquals(response.getStatus(), ControllerResponseStatus.PROCESSED);
    Assert.assertTrue((fsmMap.containsKey(segmentNameStr)));
    // Come back too late.
    segmentCompletionMgr._secconds += 25;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentCommitStart(params);
    Assert.assertEquals(response.getStatus(), ControllerResponseStatus.HOLD);
    // now FSM should be out of the map.
    Assert.assertFalse((fsmMap.containsKey(segmentNameStr)));
    Assert.assertFalse(commitTimeMap.containsKey(tableName));
}

Example 23 with LLCSegmentName

use of com.linkedin.pinot.common.utils.LLCSegmentName in project pinot by linkedin.

the class SegmentCompletionTest method testLeaseTooLong.

@Test
public void testLeaseTooLong() throws Exception {
    SegmentCompletionProtocol.Response response;
    Request.Params params;
    final String tableName = new LLCSegmentName(segmentNameStr).getTableName();
    // s1 sends offset of 20, gets HOLD at t = 5s;
    final long startTime = 5;
    segmentCompletionMgr._secconds = startTime;
    params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD);
    // s2 sends offset of 40, gets HOLD
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD);
    // s3 sends offset of 30, gets catchup to 40
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s3).withOffset(s3Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.CATCH_UP);
    Assert.assertEquals(response.getOffset(), s2Offset);
    // Now s1 comes back, and is asked to catchup.
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.CATCH_UP);
    // s2 is asked to commit.
    segmentCompletionMgr._secconds += 1;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr);
    response = segmentCompletionMgr.segmentConsumed(params);
    Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT);
    long commitTimeSec = response.getBuildTimeSeconds();
    Assert.assertTrue(commitTimeSec > 0);
    // Fast forward to one second before commit time, and send a lease renewal request for 20s
    segmentCompletionMgr._secconds = startTime + commitTimeSec - 1;
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr).withExtTimeSec(20);
    response = segmentCompletionMgr.extendBuildTime(params);
    Assert.assertEquals(response.getStatus(), ControllerResponseStatus.PROCESSED);
    Assert.assertTrue((fsmMap.containsKey(segmentNameStr)));
    final int leaseTimeSec = 20;
    // Lease will not be granted if the time taken so far plus lease time exceeds the max allowabale.
    while (segmentCompletionMgr._secconds + leaseTimeSec <= startTime + SegmentCompletionManager.getMaxCommitTimeForAllSegmentsSeconds()) {
        params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr).withExtTimeSec(leaseTimeSec);
        response = segmentCompletionMgr.extendBuildTime(params);
        Assert.assertEquals(response.getStatus(), ControllerResponseStatus.PROCESSED);
        Assert.assertTrue((fsmMap.containsKey(segmentNameStr)));
        segmentCompletionMgr._secconds += leaseTimeSec;
    }
    // Now the lease request should fail.
    params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr).withExtTimeSec(leaseTimeSec);
    response = segmentCompletionMgr.extendBuildTime(params);
    Assert.assertEquals(response.getStatus(), ControllerResponseStatus.FAILED);
    Assert.assertFalse((fsmMap.containsKey(segmentNameStr)));
}

Example 24 with LLCSegmentName

use of com.linkedin.pinot.common.utils.LLCSegmentName in project pinot by linkedin.

the class ValidationManagerTest method testTotalDocumentCountRealTime.

@Test
public void testTotalDocumentCountRealTime() throws Exception {
    // Create a bunch of dummy segments
    String testTableName = "TestTableTotalDocCountTest";
    final String group1 = testTableName + "_REALTIME_1466446700000_34";
    final String group2 = testTableName + "_REALTIME_1466446700000_17";
    String segmentName1 = new HLCSegmentName(group1, "0", "1").getSegmentName();
    String segmentName2 = new HLCSegmentName(group1, "0", "2").getSegmentName();
    String segmentName3 = new HLCSegmentName(group1, "0", "3").getSegmentName();
    String segmentName4 = new HLCSegmentName(group2, "0", "3").getSegmentName();
    DummyMetadata metadata1 = new DummyMetadata(testTableName, segmentName1, 10);
    DummyMetadata metadata2 = new DummyMetadata(testTableName, segmentName2, 20);
    DummyMetadata metadata3 = new DummyMetadata(testTableName, segmentName3, 30);
    // This should get ignored in the count as it belongs to a different group id
    DummyMetadata metadata4 = new DummyMetadata(testTableName, segmentName4, 20);
    // Add them to a list
    List<SegmentMetadata> segmentMetadataList = new ArrayList<SegmentMetadata>();
    segmentMetadataList.add(metadata1);
    segmentMetadataList.add(metadata2);
    segmentMetadataList.add(metadata3);
    segmentMetadataList.add(metadata4);
    Assert.assertEquals(ValidationManager.computeRealtimeTotalDocumentInSegments(segmentMetadataList, true), 60);
    // Now add some low level segment names
    String segmentName5 = new LLCSegmentName(testTableName, 1, 0, 1000).getSegmentName();
    String segmentName6 = new LLCSegmentName(testTableName, 2, 27, 10000).getSegmentName();
    DummyMetadata metadata5 = new DummyMetadata(testTableName, segmentName5, 10);
    DummyMetadata metadata6 = new DummyMetadata(testTableName, segmentName6, 5);
    segmentMetadataList.add(metadata5);
    segmentMetadataList.add(metadata6);
    // Only the LLC segments should get counted.
    Assert.assertEquals(ValidationManager.computeRealtimeTotalDocumentInSegments(segmentMetadataList, false), 15);
}

Example 25 with LLCSegmentName

use of com.linkedin.pinot.common.utils.LLCSegmentName in project pinot by linkedin.

the class SegmentCompletionManager method segmentConsumed.

/**
   * This method is to be called when a server calls in with the segmentConsumed() API, reporting an offset in kafka
   * that it currently has (i.e. next offset that it will consume, if it continues to consume).
   */
public SegmentCompletionProtocol.Response segmentConsumed(SegmentCompletionProtocol.Request.Params reqParams) {
    if (!_helixManager.isLeader()) {
        return SegmentCompletionProtocol.RESP_NOT_LEADER;
    }
    final String segmentNameStr = reqParams.getSegmentName();
    final String instanceId = reqParams.getInstanceId();
    final long offset = reqParams.getOffset();
    LLCSegmentName segmentName = new LLCSegmentName(segmentNameStr);
    SegmentCompletionFSM fsm = lookupOrCreateFsm(segmentName, SegmentCompletionProtocol.MSG_TYPE_CONSUMED);
    SegmentCompletionProtocol.Response response = fsm.segmentConsumed(instanceId, offset);
    if (fsm.isDone()) {
        LOGGER.info("Removing FSM (if present):{}", fsm.toString());
        _fsmMap.remove(segmentNameStr);
    }
    return response;
}