Examples with LLCRealtimeSegmentZKMetadata com.linkedin.pinot.common.metadata.segment.LLCRealtimeSegmentZKMetadata used on opensource projects

Example 6 with LLCRealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.LLCRealtimeSegmentZKMetadata in project pinot by linkedin.

the class RetentionManagerTest method setupRealtimeTable.

// The most recent will be in
private Set<String> setupRealtimeTable(final int nSegments, final long now) throws Exception {
    final int replicaCount = 1;
    createRealtimeTableConfig(_realtimeTableName, replicaCount);
    Set<String> remainingSegments = new HashSet<>();
    IdealState idealState = PinotTableIdealStateBuilder.buildEmptyKafkaConsumerRealtimeIdealStateFor(_realtimeTableName, replicaCount);
    final int kafkaPartition = 5;
    final long millisInDays = TimeUnit.DAYS.toMillis(1);
    final String serverName = "Server_localhost_0";
    // If we set the segment creation time to a certain value and compare it as being X ms old,
    // then we could get unpredictable results depending on whether it takes more or less than
    // one millisecond to get to RetentionManager time comparison code. To be safe, set the
    // milliseconds off by 1/2 day.
    long segmentCreationTime = now - (nSegments + 1) * millisInDays + millisInDays / 2;
    List<LLCRealtimeSegmentZKMetadata> segmentZKMetadatas = new ArrayList<>();
    for (int seq = 1; seq <= nSegments; seq++) {
        segmentCreationTime += millisInDays;
        LLCRealtimeSegmentZKMetadata segmentMetadata = createSegmentMetadata(replicaCount, segmentCreationTime);
        LLCSegmentName llcSegmentName = new LLCSegmentName(_testTableName, kafkaPartition, seq, segmentCreationTime);
        final String segName = llcSegmentName.getSegmentName();
        segmentMetadata.setSegmentName(segName);
        if (seq == nSegments) {
            // create consuming segment
            segmentMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
            idealState.setPartitionState(segName, serverName, "CONSUMING");
            remainingSegments.add(segName);
        } else if (seq % 2 == 0) {
            // create ONLINE segment
            segmentMetadata.setStatus(CommonConstants.Segment.Realtime.Status.DONE);
            idealState.setPartitionState(segName, serverName, "ONLINE");
            remainingSegments.add(segName);
        } else {
            segmentMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
            idealState.setPartitionState(segName, serverName, "OFFLINE");
            if (now - segmentCreationTime < TimeUnit.DAYS.toMillis(RetentionManager.getRetentionTimeForOldLLCSegmentsDays())) {
                remainingSegments.add(segName);
            }
        }
        final String znodePath = ZKMetadataProvider.constructPropertyStorePathForSegment(_realtimeTableName, segName);
        _propertyStore.set(znodePath, segmentMetadata.toZNRecord(), AccessOption.PERSISTENT);
    }
    _helixAdmin.addResource(HELIX_CLUSTER_NAME, _realtimeTableName, idealState);
    return remainingSegments;
}

Example 7 with LLCRealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.LLCRealtimeSegmentZKMetadata in project pinot by linkedin.

the class PinotLLCRealtimeSegmentManagerTest method testCommittingSegment.

@Test
public void testCommittingSegment() throws Exception {
    FakePinotLLCRealtimeSegmentManager segmentManager = new FakePinotLLCRealtimeSegmentManager(true, null);
    final String topic = "someTopic";
    final String rtTableName = "table_REALTIME";
    final String rawTableName = TableNameBuilder.extractRawTableName(rtTableName);
    final int nInstances = 6;
    final int nPartitions = 16;
    final int nReplicas = 3;
    final boolean existingIS = false;
    List<String> instances = getInstanceList(nInstances);
    final String startOffset = KAFKA_OFFSET;
    IdealState idealState = PinotTableIdealStateBuilder.buildEmptyKafkaConsumerRealtimeIdealStateFor(rtTableName, nReplicas);
    segmentManager.setupHelixEntries(topic, rtTableName, nPartitions, instances, nReplicas, startOffset, DUMMY_HOST, idealState, !existingIS, 10000);
    // Now commit the first segment of partition 6.
    final int committingPartition = 6;
    final long nextOffset = 3425666L;
    LLCRealtimeSegmentZKMetadata committingSegmentMetadata = new LLCRealtimeSegmentZKMetadata(segmentManager._records.get(committingPartition));
    segmentManager._paths.clear();
    segmentManager._records.clear();
    boolean status = segmentManager.commitSegment(rawTableName, committingSegmentMetadata.getSegmentName(), nextOffset);
    segmentManager.verifyMetadataInteractions();
    Assert.assertTrue(status);
    // Get the old and new segment metadata and make sure that they are correct.
    Assert.assertEquals(segmentManager._paths.size(), 2);
    ZNRecord oldZnRec = segmentManager._records.get(0);
    ZNRecord newZnRec = segmentManager._records.get(1);
    LLCRealtimeSegmentZKMetadata oldMetadata = new LLCRealtimeSegmentZKMetadata(oldZnRec);
    LLCRealtimeSegmentZKMetadata newMetadata = new LLCRealtimeSegmentZKMetadata(newZnRec);
    LLCSegmentName oldSegmentName = new LLCSegmentName(oldMetadata.getSegmentName());
    LLCSegmentName newSegmentName = new LLCSegmentName(newMetadata.getSegmentName());
    // Assert on propertystore entries
    Assert.assertEquals(oldSegmentName.getSegmentName(), committingSegmentMetadata.getSegmentName());
    Assert.assertEquals(newSegmentName.getPartitionId(), oldSegmentName.getPartitionId());
    Assert.assertEquals(newSegmentName.getSequenceNumber(), oldSegmentName.getSequenceNumber() + 1);
    Assert.assertEquals(oldMetadata.getStatus(), CommonConstants.Segment.Realtime.Status.DONE);
    Assert.assertEquals(newMetadata.getStatus(), CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
    Assert.assertNotNull(oldMetadata.getDownloadUrl());
    Assert.assertEquals(Long.valueOf(oldMetadata.getCrc()), Long.valueOf(FakePinotLLCRealtimeSegmentManager.CRC));
    Assert.assertEquals(oldMetadata.getStartTime(), FakePinotLLCRealtimeSegmentManager.INTERVAL.getStartMillis());
    Assert.assertEquals(oldMetadata.getEndTime(), FakePinotLLCRealtimeSegmentManager.INTERVAL.getEndMillis());
    Assert.assertEquals(oldMetadata.getTotalRawDocs(), FakePinotLLCRealtimeSegmentManager.NUM_DOCS);
    Assert.assertEquals(oldMetadata.getIndexVersion(), FakePinotLLCRealtimeSegmentManager.SEGMENT_VERSION);
}

Example 8 with LLCRealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.LLCRealtimeSegmentZKMetadata in project pinot by linkedin.

the class PinotLLCRealtimeSegmentManagerTest method testUpdateFlushThresholdForSegmentMetadata.

@Test
public void testUpdateFlushThresholdForSegmentMetadata() {
    PinotLLCRealtimeSegmentManager realtimeSegmentManager = new FakePinotLLCRealtimeSegmentManager(false, Collections.<String>emptyList());
    ZNRecord partitionAssignment = new ZNRecord("fakeTable_REALTIME");
    // 4 partitions assigned to 4 servers, 4 replicas => the segments should have 250k rows each (1M / 4)
    for (int segmentId = 1; segmentId <= 4; ++segmentId) {
        List<String> instances = new ArrayList<>();
        for (int replicaId = 1; replicaId <= 4; ++replicaId) {
            instances.add("Server_1.2.3.4_123" + replicaId);
        }
        partitionAssignment.setListField(Integer.toString(segmentId), instances);
    }
    // Check that each segment has 250k rows each
    for (int segmentId = 1; segmentId <= 4; ++segmentId) {
        LLCRealtimeSegmentZKMetadata metadata = new LLCRealtimeSegmentZKMetadata();
        metadata.setSegmentName(makeFakeSegmentName(segmentId));
        realtimeSegmentManager.updateFlushThresholdForSegmentMetadata(metadata, partitionAssignment, 1000000);
        Assert.assertEquals(metadata.getSizeThresholdToFlushSegment(), 250000);
    }
    // 4 partitions assigned to 4 servers, 2 partitions/server => the segments should have 500k rows each (1M / 2)
    partitionAssignment.getListFields().clear();
    for (int segmentId = 1; segmentId <= 4; ++segmentId) {
        List<String> instances = new ArrayList<>();
        for (int replicaId = 1; replicaId <= 2; ++replicaId) {
            instances.add("Server_1.2.3.4_123" + ((replicaId + segmentId) % 4));
        }
        partitionAssignment.setListField(Integer.toString(segmentId), instances);
    }
    // Check that each segment has 500k rows each
    for (int segmentId = 1; segmentId <= 4; ++segmentId) {
        LLCRealtimeSegmentZKMetadata metadata = new LLCRealtimeSegmentZKMetadata();
        metadata.setSegmentName(makeFakeSegmentName(segmentId));
        realtimeSegmentManager.updateFlushThresholdForSegmentMetadata(metadata, partitionAssignment, 1000000);
        Assert.assertEquals(metadata.getSizeThresholdToFlushSegment(), 500000);
    }
    // 4 partitions assigned to 4 servers, 1 partition/server => the segments should have 1M rows each (1M / 1)
    partitionAssignment.getListFields().clear();
    for (int segmentId = 1; segmentId <= 4; ++segmentId) {
        List<String> instances = new ArrayList<>();
        instances.add("Server_1.2.3.4_123" + segmentId);
        partitionAssignment.setListField(Integer.toString(segmentId), instances);
    }
    // Check that each segment has 1M rows each
    for (int segmentId = 1; segmentId <= 4; ++segmentId) {
        LLCRealtimeSegmentZKMetadata metadata = new LLCRealtimeSegmentZKMetadata();
        metadata.setSegmentName(makeFakeSegmentName(segmentId));
        realtimeSegmentManager.updateFlushThresholdForSegmentMetadata(metadata, partitionAssignment, 1000000);
        Assert.assertEquals(metadata.getSizeThresholdToFlushSegment(), 1000000);
    }
    // Assign another partition to all servers => the servers should have 500k rows each (1M / 2)
    List<String> instances = new ArrayList<>();
    for (int replicaId = 1; replicaId <= 4; ++replicaId) {
        instances.add("Server_1.2.3.4_123" + replicaId);
    }
    partitionAssignment.setListField("5", instances);
    // Check that each segment has 500k rows each
    for (int segmentId = 1; segmentId <= 4; ++segmentId) {
        LLCRealtimeSegmentZKMetadata metadata = new LLCRealtimeSegmentZKMetadata();
        metadata.setSegmentName(makeFakeSegmentName(segmentId));
        realtimeSegmentManager.updateFlushThresholdForSegmentMetadata(metadata, partitionAssignment, 1000000);
        Assert.assertEquals(metadata.getSizeThresholdToFlushSegment(), 500000);
    }
}

Example 9 with LLCRealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.LLCRealtimeSegmentZKMetadata in project pinot by linkedin.

the class PinotLLCRealtimeSegmentManagerTest method testCompleteCommittingSegments.

@Test
public void testCompleteCommittingSegments() throws Exception {
    // Run multiple times randomizing the situation.
    for (int i = 0; i < 100; i++) {
        final List<ZNRecord> existingSegmentMetadata = new ArrayList<>(64);
        final int nPartitions = 16;
        final long seed = new Random().nextLong();
        Random random = new Random(seed);
        final int maxSeq = 10;
        final long now = System.currentTimeMillis();
        final String tableName = "table";
        final String realtimeTableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(tableName);
        final IdealState idealState = PinotTableIdealStateBuilder.buildEmptyKafkaConsumerRealtimeIdealStateFor(realtimeTableName, 19);
        int nIncompleteCommits = 0;
        final String topic = "someTopic";
        final int nInstances = 5;
        final int nReplicas = 3;
        List<String> instances = getInstanceList(nInstances);
        final String startOffset = KAFKA_OFFSET;
        FakePinotLLCRealtimeSegmentManager segmentManager = new FakePinotLLCRealtimeSegmentManager(false, null);
        segmentManager.setupHelixEntries(topic, realtimeTableName, nPartitions, instances, nReplicas, startOffset, DUMMY_HOST, idealState, false, 10000);
        ZNRecord partitionAssignment = segmentManager.getKafkaPartitionAssignment(realtimeTableName);
        for (int p = 0; p < nPartitions; p++) {
            // Current segment sequence ID for that partition
            int curSeq = random.nextInt(maxSeq);
            if (curSeq == 0) {
                curSeq++;
            }
            boolean incomplete = false;
            if (random.nextBoolean()) {
                incomplete = true;
            }
            for (int s = 0; s < curSeq; s++) {
                LLCSegmentName segmentName = new LLCSegmentName(tableName, p, s, now);
                String segNameStr = segmentName.getSegmentName();
                String state = PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE;
                CommonConstants.Segment.Realtime.Status status = CommonConstants.Segment.Realtime.Status.DONE;
                if (s == curSeq - 1) {
                    state = PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE;
                    if (!incomplete) {
                        status = CommonConstants.Segment.Realtime.Status.IN_PROGRESS;
                    }
                }
                List<String> instancesForThisSeg = partitionAssignment.getListField(Integer.toString(p));
                for (String instance : instancesForThisSeg) {
                    idealState.setPartitionState(segNameStr, instance, state);
                }
                LLCRealtimeSegmentZKMetadata metadata = new LLCRealtimeSegmentZKMetadata();
                metadata.setSegmentName(segNameStr);
                metadata.setStatus(status);
                existingSegmentMetadata.add(metadata.toZNRecord());
            }
            // Add an incomplete commit to some of them
            if (incomplete) {
                nIncompleteCommits++;
                LLCSegmentName segmentName = new LLCSegmentName(tableName, p, curSeq, now);
                LLCRealtimeSegmentZKMetadata metadata = new LLCRealtimeSegmentZKMetadata();
                metadata.setSegmentName(segmentName.getSegmentName());
                metadata.setStatus(CommonConstants.Segment.Realtime.Status.DONE);
                existingSegmentMetadata.add(metadata.toZNRecord());
            }
        }
        segmentManager._tableIdealState = idealState;
        segmentManager._existingSegmentMetadata = existingSegmentMetadata;
        segmentManager.completeCommittingSegments(TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(tableName));
        Assert.assertEquals(segmentManager._nCallsToUpdateHelix, nIncompleteCommits, "Failed with seed " + seed);
    }
}

Example 10 with LLCRealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.LLCRealtimeSegmentZKMetadata in project pinot by linkedin.

the class RetentionManagerTest method createSegmentMetadata.

private LLCRealtimeSegmentZKMetadata createSegmentMetadata(int replicaCount, long segmentCreationTime) {
    LLCRealtimeSegmentZKMetadata segmentMetadata = new LLCRealtimeSegmentZKMetadata();
    segmentMetadata.setCreationTime(segmentCreationTime);
    segmentMetadata.setStartOffset(0L);
    segmentMetadata.setEndOffset(-1L);
    segmentMetadata.setNumReplicas(replicaCount);
    segmentMetadata.setTableName(_testTableName);
    return segmentMetadata;
}