Examples with LLCSegmentName com.linkedin.pinot.common.utils.LLCSegmentName used on opensource projects

Example 26 with LLCSegmentName

use of com.linkedin.pinot.common.utils.LLCSegmentName in project pinot by linkedin.

the class SegmentCompletionManager method segmentCommitStart.

/**
   * This method is to be called when a server calls in with the segmentCommit() API. The server sends in the segment
   * along with the API, but it is the caller's responsibility to save the segment after this call (and before the
   * segmentCommitEnd() call).
   *
   * If successful, this method will return Response.COMMIT_CONTINUE, in which case, the caller should save the incoming
   * segment and then call segmentCommitEnd().
   *
   * Otherwise, this method will return a protocol response to be returned to the client right away (without saving the
   * incoming segment).
   */
public SegmentCompletionProtocol.Response segmentCommitStart(final SegmentCompletionProtocol.Request.Params reqParams) {
    if (!_helixManager.isLeader()) {
        return SegmentCompletionProtocol.RESP_NOT_LEADER;
    }
    final String segmentNameStr = reqParams.getSegmentName();
    final String instanceId = reqParams.getInstanceId();
    final long offset = reqParams.getOffset();
    LLCSegmentName segmentName = new LLCSegmentName(segmentNameStr);
    SegmentCompletionFSM fsm = lookupOrCreateFsm(segmentName, SegmentCompletionProtocol.MSG_TYPE_COMMMIT);
    SegmentCompletionProtocol.Response response = fsm.segmentCommitStart(instanceId, offset);
    if (fsm.isDone()) {
        LOGGER.info("Removing FSM (if present):{}", fsm.toString());
        _fsmMap.remove(segmentNameStr);
    }
    return response;
}

Example 27 with LLCSegmentName

use of com.linkedin.pinot.common.utils.LLCSegmentName in project pinot by linkedin.

the class SegmentCompletionManager method extendBuildTime.

public SegmentCompletionProtocol.Response extendBuildTime(final SegmentCompletionProtocol.Request.Params reqParams) {
    if (!_helixManager.isLeader()) {
        return SegmentCompletionProtocol.RESP_NOT_LEADER;
    }
    final String segmentNameStr = reqParams.getSegmentName();
    final String instanceId = reqParams.getInstanceId();
    final long offset = reqParams.getOffset();
    final int extTimeSec = reqParams.getExtTimeSec();
    LLCSegmentName segmentName = new LLCSegmentName(segmentNameStr);
    SegmentCompletionFSM fsm = lookupOrCreateFsm(segmentName, SegmentCompletionProtocol.MSG_TYPE_COMMMIT);
    SegmentCompletionProtocol.Response response = fsm.extendBuildTime(instanceId, offset, extTimeSec);
    if (fsm.isDone()) {
        LOGGER.info("Removing FSM (if present):{}", fsm.toString());
        _fsmMap.remove(segmentNameStr);
    }
    return response;
}

Example 28 with LLCSegmentName

use of com.linkedin.pinot.common.utils.LLCSegmentName in project pinot by linkedin.

the class SegmentCompletionManager method segmentStoppedConsuming.

/**
   * This method is to be called when a server reports that it has stopped consuming a real-time segment.
   *
   * @return
   */
public SegmentCompletionProtocol.Response segmentStoppedConsuming(SegmentCompletionProtocol.Request.Params reqParams) {
    if (!_helixManager.isLeader()) {
        return SegmentCompletionProtocol.RESP_NOT_LEADER;
    }
    final String segmentNameStr = reqParams.getSegmentName();
    final String instanceId = reqParams.getInstanceId();
    final long offset = reqParams.getOffset();
    final String reason = reqParams.getReason();
    LLCSegmentName segmentName = new LLCSegmentName(segmentNameStr);
    SegmentCompletionFSM fsm = lookupOrCreateFsm(segmentName, SegmentCompletionProtocol.MSG_TYPE_STOPPED_CONSUMING);
    SegmentCompletionProtocol.Response response = fsm.stoppedConsuming(instanceId, offset, reason);
    if (fsm.isDone()) {
        LOGGER.info("Removing FSM (if present):{}", fsm.toString());
        _fsmMap.remove(segmentNameStr);
    }
    return response;
}

Example 29 with LLCSegmentName

use of com.linkedin.pinot.common.utils.LLCSegmentName in project pinot by linkedin.

the class PinotLLCRealtimeSegmentManager method setupInitialSegments.

protected void setupInitialSegments(String realtimeTableName, ZNRecord partitionAssignment, String topicName, String initialOffset, String bootstrapHosts, IdealState idealState, boolean create, int nReplicas, int flushSize) {
    List<String> currentSegments = getExistingSegments(realtimeTableName);
    // Make sure that there are no low-level segments existing.
    if (currentSegments != null) {
        for (String segment : currentSegments) {
            if (!SegmentName.isHighLevelConsumerSegmentName(segment)) {
                // realtime segments for any other reason.
                throw new RuntimeException("Low-level segments already exist for table " + realtimeTableName);
            }
        }
    }
    // Map of segment names to the server-instances that hold the segment.
    final Map<String, List<String>> idealStateEntries = new HashMap<String, List<String>>(4);
    final Map<String, List<String>> partitionToServersMap = partitionAssignment.getListFields();
    final int nPartitions = partitionToServersMap.size();
    // Create one segment entry in PROPERTYSTORE for each kafka partition.
    // Any of these may already be there, so bail out clean if they are already present.
    final long now = System.currentTimeMillis();
    final int seqNum = STARTING_SEQUENCE_NUMBER;
    List<LLCRealtimeSegmentZKMetadata> segmentZKMetadatas = new ArrayList<>();
    // Create metadata for each segment
    for (int i = 0; i < nPartitions; i++) {
        final List<String> instances = partitionToServersMap.get(Integer.toString(i));
        LLCRealtimeSegmentZKMetadata metadata = new LLCRealtimeSegmentZKMetadata();
        final String rawTableName = TableNameBuilder.extractRawTableName(realtimeTableName);
        LLCSegmentName llcSegmentName = new LLCSegmentName(rawTableName, i, seqNum, now);
        final String segName = llcSegmentName.getSegmentName();
        metadata.setCreationTime(now);
        final long startOffset = getPartitionOffset(topicName, bootstrapHosts, initialOffset, i);
        LOGGER.info("Setting start offset for segment {} to {}", segName, startOffset);
        metadata.setStartOffset(startOffset);
        metadata.setEndOffset(END_OFFSET_FOR_CONSUMING_SEGMENTS);
        metadata.setNumReplicas(instances.size());
        metadata.setTableName(rawTableName);
        metadata.setSegmentName(segName);
        metadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
        segmentZKMetadatas.add(metadata);
        idealStateEntries.put(segName, instances);
    }
    // Compute the number of rows for each segment
    for (LLCRealtimeSegmentZKMetadata segmentZKMetadata : segmentZKMetadatas) {
        updateFlushThresholdForSegmentMetadata(segmentZKMetadata, partitionAssignment, flushSize);
    }
    // Write metadata for each segment to the Helix property store
    List<String> paths = new ArrayList<>(nPartitions);
    List<ZNRecord> records = new ArrayList<>(nPartitions);
    for (LLCRealtimeSegmentZKMetadata segmentZKMetadata : segmentZKMetadatas) {
        ZNRecord record = segmentZKMetadata.toZNRecord();
        final String znodePath = ZKMetadataProvider.constructPropertyStorePathForSegment(realtimeTableName, segmentZKMetadata.getSegmentName());
        paths.add(znodePath);
        records.add(record);
    }
    writeSegmentsToPropertyStore(paths, records, realtimeTableName);
    LOGGER.info("Added {} segments to propertyStore for table {}", paths.size(), realtimeTableName);
    updateIdealState(idealState, realtimeTableName, idealStateEntries, create, nReplicas);
}

Example 30 with LLCSegmentName

use of com.linkedin.pinot.common.utils.LLCSegmentName in project pinot by linkedin.

the class PinotLLCRealtimeSegmentManager method createSegment.

private void createSegment(String realtimeTableName, int numReplicas, int partitionId, int seqNum, List<String> serverInstances, long startOffset, ZNRecord partitionAssignment) {
    LOGGER.info("Attempting to auto-create a segment for partition {} of table {}", partitionId, realtimeTableName);
    final List<String> propStorePaths = new ArrayList<>(1);
    final List<ZNRecord> propStoreEntries = new ArrayList<>(1);
    long now = System.currentTimeMillis();
    final String tableName = TableNameBuilder.extractRawTableName(realtimeTableName);
    LLCSegmentName newSegmentName = new LLCSegmentName(tableName, partitionId, seqNum, now);
    final String newSegmentNameStr = newSegmentName.getSegmentName();
    ZNRecord newZnRecord = makeZnRecordForNewSegment(realtimeTableName, numReplicas, startOffset, newSegmentNameStr);
    final LLCRealtimeSegmentZKMetadata newSegmentZKMetadata = new LLCRealtimeSegmentZKMetadata(newZnRecord);
    updateFlushThresholdForSegmentMetadata(newSegmentZKMetadata, partitionAssignment, getRealtimeTableFlushSizeForTable(realtimeTableName));
    newZnRecord = newSegmentZKMetadata.toZNRecord();
    final String newZnodePath = ZKMetadataProvider.constructPropertyStorePathForSegment(realtimeTableName, newSegmentNameStr);
    propStorePaths.add(newZnodePath);
    propStoreEntries.add(newZnRecord);
    writeSegmentsToPropertyStore(propStorePaths, propStoreEntries, realtimeTableName);
    updateIdealState(realtimeTableName, serverInstances, null, newSegmentNameStr);
    LOGGER.info("Successful auto-create of CONSUMING segment {}", newSegmentNameStr);
    _controllerMetrics.addMeteredTableValue(realtimeTableName, ControllerMeter.LLC_AUTO_CREATED_PARTITIONS, 1);
}