Examples with RealtimeSegmentZKMetadata com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata used on opensource projects

Example 6 with RealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata in project pinot by linkedin.

the class PinotRealtimeSegmentManager method assignRealtimeSegmentsToServerInstancesIfNecessary.

private synchronized void assignRealtimeSegmentsToServerInstancesIfNecessary() throws JSONException, IOException {
    // Fetch current ideal state snapshot
    Map<String, IdealState> idealStateMap = new HashMap<String, IdealState>();
    for (String resource : _pinotHelixResourceManager.getAllRealtimeTables()) {
        final String tableName = TableNameBuilder.extractRawTableName(resource);
        AbstractTableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableName, TableType.REALTIME);
        KafkaStreamMetadata metadata = new KafkaStreamMetadata(tableConfig.getIndexingConfig().getStreamConfigs());
        if (metadata.hasHighLevelKafkaConsumerType()) {
            idealStateMap.put(resource, _pinotHelixResourceManager.getHelixAdmin().getResourceIdealState(_pinotHelixResourceManager.getHelixClusterName(), resource));
        } else {
            LOGGER.debug("Not considering table {} for realtime segment assignment");
        }
    }
    List<Pair<String, String>> listOfSegmentsToAddToInstances = new ArrayList<Pair<String, String>>();
    for (String resource : idealStateMap.keySet()) {
        try {
            IdealState state = idealStateMap.get(resource);
            // Are there any partitions?
            if (state.getPartitionSet().size() == 0) {
                // No, this is a brand new ideal state, so we will add one new segment to every partition and replica
                List<String> instancesInResource = new ArrayList<String>();
                try {
                    instancesInResource.addAll(_pinotHelixResourceManager.getServerInstancesForTable(resource, TableType.REALTIME));
                } catch (Exception e) {
                    LOGGER.error("Caught exception while fetching instances for resource {}", resource, e);
                    _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
                }
                // Assign a new segment to all server instances
                for (String instanceId : instancesInResource) {
                    InstanceZKMetadata instanceZKMetadata = _pinotHelixResourceManager.getInstanceZKMetadata(instanceId);
                    if (instanceZKMetadata == null) {
                        LOGGER.warn("Instance {} has no associated instance metadata in ZK, ignoring for segment assignment.", instanceId);
                        _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
                        continue;
                    }
                    String groupId = instanceZKMetadata.getGroupId(resource);
                    String partitionId = instanceZKMetadata.getPartition(resource);
                    if (groupId != null && !groupId.isEmpty() && partitionId != null && !partitionId.isEmpty()) {
                        listOfSegmentsToAddToInstances.add(new Pair<String, String>(new HLCSegmentName(groupId, partitionId, String.valueOf(System.currentTimeMillis())).getSegmentName(), instanceId));
                    } else {
                        LOGGER.warn("Instance {} has invalid groupId ({}) and/or partitionId ({}) for resource {}, ignoring for segment assignment.", instanceId, groupId, partitionId, resource);
                        _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
                    }
                }
            } else {
                // Add all server instances to the list of instances for which to assign a realtime segment
                Set<String> instancesToAssignRealtimeSegment = new HashSet<String>();
                try {
                    instancesToAssignRealtimeSegment.addAll(_pinotHelixResourceManager.getServerInstancesForTable(resource, TableType.REALTIME));
                } catch (Exception e) {
                    LOGGER.error("Caught exception while fetching instances for resource {}", resource, e);
                    _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
                }
                // Remove server instances that are currently processing a segment
                for (String partition : state.getPartitionSet()) {
                    // Helix partition is the segment name
                    if (SegmentName.isHighLevelConsumerSegmentName(partition)) {
                        HLCSegmentName segName = new HLCSegmentName(partition);
                        RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider.getRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), segName.getTableName(), partition);
                        if (realtimeSegmentZKMetadata == null) {
                            // Segment was deleted by retention manager.
                            continue;
                        }
                        if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) {
                            instancesToAssignRealtimeSegment.removeAll(state.getInstanceSet(partition));
                        }
                    }
                }
                // Assign a new segment to the server instances not currently processing this segment
                for (String instanceId : instancesToAssignRealtimeSegment) {
                    InstanceZKMetadata instanceZKMetadata = _pinotHelixResourceManager.getInstanceZKMetadata(instanceId);
                    String groupId = instanceZKMetadata.getGroupId(resource);
                    String partitionId = instanceZKMetadata.getPartition(resource);
                    listOfSegmentsToAddToInstances.add(new Pair<String, String>(new HLCSegmentName(groupId, partitionId, String.valueOf(System.currentTimeMillis())).getSegmentName(), instanceId));
                }
            }
        } catch (Exception e) {
            LOGGER.warn("Caught exception while processing resource {}, skipping.", resource, e);
            _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
        }
    }
    LOGGER.info("Computed list of new segments to add : " + Arrays.toString(listOfSegmentsToAddToInstances.toArray()));
    // Add the new segments to the server instances
    for (final Pair<String, String> segmentIdAndInstanceId : listOfSegmentsToAddToInstances) {
        final String segmentId = segmentIdAndInstanceId.getFirst();
        final String instanceName = segmentIdAndInstanceId.getSecond();
        try {
            final HLCSegmentName segName = new HLCSegmentName(segmentId);
            String resourceName = segName.getTableName();
            // Does the ideal state already contain this segment?
            if (!idealStateMap.get(resourceName).getPartitionSet().contains(segmentId)) {
                // No, add it
                // Create the realtime segment metadata
                RealtimeSegmentZKMetadata realtimeSegmentMetadataToAdd = new RealtimeSegmentZKMetadata();
                realtimeSegmentMetadataToAdd.setTableName(TableNameBuilder.extractRawTableName(resourceName));
                realtimeSegmentMetadataToAdd.setSegmentType(SegmentType.REALTIME);
                realtimeSegmentMetadataToAdd.setStatus(Status.IN_PROGRESS);
                realtimeSegmentMetadataToAdd.setSegmentName(segmentId);
                // Add the new metadata to the property store
                ZKMetadataProvider.setRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), realtimeSegmentMetadataToAdd);
                // Update the ideal state to add the new realtime segment
                HelixHelper.updateIdealState(_pinotHelixResourceManager.getHelixZkManager(), resourceName, new Function<IdealState, IdealState>() {

                    @Override
                    public IdealState apply(IdealState idealState) {
                        return PinotTableIdealStateBuilder.addNewRealtimeSegmentToIdealState(segmentId, idealState, instanceName);
                    }
                }, RetryPolicies.exponentialBackoffRetryPolicy(5, 500L, 2.0f));
            }
        } catch (Exception e) {
            LOGGER.warn("Caught exception while processing segment {} for instance {}, skipping.", segmentId, instanceName, e);
            _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
        }
    }
}

Example 7 with RealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata in project pinot by linkedin.

the class PinotRealtimeSegmentManager method refreshWatchers.

/**
   * Helper method to perform idempotent operation to refresh all watches (related to real-time segments):
   * - Data change listener for all existing real-time tables.
   * - Child creation listener for all existing real-time tables.
   * - Data change listener for all existing real-time segments
   *
   * @param path
   */
private void refreshWatchers(String path) {
    LOGGER.info("Received change notification for path: {}", path);
    List<Stat> stats = new ArrayList<>();
    List<ZNRecord> tableConfigs = _pinotHelixResourceManager.getPropertyStore().getChildren(TABLE_CONFIG, stats, 0);
    if (tableConfigs == null) {
        return;
    }
    for (ZNRecord tableConfigZnRecord : tableConfigs) {
        try {
            String znRecordId = tableConfigZnRecord.getId();
            if (TableNameBuilder.getTableTypeFromTableName(znRecordId) == TableType.REALTIME) {
                AbstractTableConfig abstractTableConfig = AbstractTableConfig.fromZnRecord(tableConfigZnRecord);
                KafkaStreamMetadata metadata = new KafkaStreamMetadata(abstractTableConfig.getIndexingConfig().getStreamConfigs());
                if (metadata.hasHighLevelKafkaConsumerType()) {
                    String realtimeTable = abstractTableConfig.getTableName();
                    String realtimeSegmentsPathForTable = _propertyStorePath + SEGMENTS_PATH + "/" + realtimeTable;
                    LOGGER.info("Setting data/child changes watch for real-time table '{}'", realtimeTable);
                    _zkClient.subscribeDataChanges(realtimeSegmentsPathForTable, this);
                    _zkClient.subscribeChildChanges(realtimeSegmentsPathForTable, this);
                    List<String> childNames = _pinotHelixResourceManager.getPropertyStore().getChildNames(SEGMENTS_PATH + "/" + realtimeTable, 0);
                    if (childNames != null && !childNames.isEmpty()) {
                        for (String segmentName : childNames) {
                            if (!SegmentName.isHighLevelConsumerSegmentName(segmentName)) {
                                continue;
                            }
                            String segmentPath = realtimeSegmentsPathForTable + "/" + segmentName;
                            RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider.getRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), abstractTableConfig.getTableName(), segmentName);
                            if (realtimeSegmentZKMetadata == null) {
                                // The segment got deleted by retention manager
                                continue;
                            }
                            if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) {
                                LOGGER.info("Setting data change watch for real-time segment currently being consumed: {}", segmentPath);
                                _zkClient.subscribeDataChanges(segmentPath, this);
                            } else {
                                _zkClient.unsubscribeDataChanges(segmentPath, this);
                            }
                        }
                    }
                }
            }
        } catch (Exception e) {
            // we want to continue setting watches for other tables for any kind of exception here so that
            // errors with one table don't impact others
            LOGGER.error("Caught exception while processing ZNRecord id: {}. Skipping node to continue setting watches", tableConfigZnRecord.getId(), e);
        }
    }
}

Example 8 with RealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata in project pinot by linkedin.

the class RetentionManager method retrieveSegmentMetadataForTable.

private List<SegmentZKMetadata> retrieveSegmentMetadataForTable(String tableName) {
    List<SegmentZKMetadata> segmentMetadataList = new ArrayList<>();
    ZkHelixPropertyStore<ZNRecord> propertyStore = _pinotHelixResourceManager.getPropertyStore();
    TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
    assert tableType != null;
    switch(tableType) {
        case OFFLINE:
            List<OfflineSegmentZKMetadata> offlineSegmentZKMetadatas = ZKMetadataProvider.getOfflineSegmentZKMetadataListForTable(propertyStore, tableName);
            for (OfflineSegmentZKMetadata offlineSegmentZKMetadata : offlineSegmentZKMetadatas) {
                segmentMetadataList.add(offlineSegmentZKMetadata);
            }
            break;
        case REALTIME:
            List<RealtimeSegmentZKMetadata> realtimeSegmentZKMetadatas = ZKMetadataProvider.getRealtimeSegmentZKMetadataListForTable(propertyStore, tableName);
            for (RealtimeSegmentZKMetadata realtimeSegmentZKMetadata : realtimeSegmentZKMetadatas) {
                segmentMetadataList.add(realtimeSegmentZKMetadata);
            }
            break;
        default:
            throw new IllegalArgumentException("No table type matches table name: " + tableName);
    }
    return segmentMetadataList;
}

Example 9 with RealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata in project pinot by linkedin.

the class RetentionManager method scanSegmentMetadataAndPurge.

private void scanSegmentMetadataAndPurge() {
    for (String tableName : _segmentMetadataMap.keySet()) {
        List<SegmentZKMetadata> segmentZKMetadataList = _segmentMetadataMap.get(tableName);
        List<String> segmentsToDelete = new ArrayList<>(128);
        IdealState idealState = null;
        try {
            if (TableNameBuilder.getTableTypeFromTableName(tableName).equals(TableType.REALTIME)) {
                idealState = HelixHelper.getTableIdealState(_pinotHelixResourceManager.getHelixZkManager(), tableName);
            }
        } catch (Exception e) {
            LOGGER.warn("Could not get idealstate for {}", tableName, e);
        // Ignore, worst case we have some old inactive segments in place.
        }
        for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
            RetentionStrategy deletionStrategy;
            deletionStrategy = _tableDeletionStrategy.get(tableName);
            if (deletionStrategy == null) {
                LOGGER.info("No Retention strategy found for segment: {}", segmentZKMetadata.getSegmentName());
                continue;
            }
            if (segmentZKMetadata instanceof RealtimeSegmentZKMetadata) {
                final RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = (RealtimeSegmentZKMetadata) segmentZKMetadata;
                if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) {
                    final String segmentId = realtimeSegmentZKMetadata.getSegmentName();
                    if (SegmentName.isHighLevelConsumerSegmentName(segmentId)) {
                        continue;
                    }
                    // auto-create LLC segments.
                    if (shouldDeleteInProgressLLCSegment(segmentId, idealState, realtimeSegmentZKMetadata)) {
                        segmentsToDelete.add(segmentId);
                    }
                    continue;
                }
            }
            if (deletionStrategy.isPurgeable(segmentZKMetadata)) {
                LOGGER.info("Marking segment to delete: {}", segmentZKMetadata.getSegmentName());
                segmentsToDelete.add(segmentZKMetadata.getSegmentName());
            }
        }
        if (segmentsToDelete.size() > 0) {
            LOGGER.info("Trying to delete {} segments for table {}", segmentsToDelete.size(), tableName);
            _pinotHelixResourceManager.deleteSegments(tableName, segmentsToDelete);
        }
    }
}

Example 10 with RealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata in project pinot by linkedin.

the class RealtimeQueriesSentinelTest method getRealtimeSegmentZKMetadata.

private RealtimeSegmentZKMetadata getRealtimeSegmentZKMetadata() {
    RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = new RealtimeSegmentZKMetadata();
    realtimeSegmentZKMetadata.setTableName("testTable");
    return realtimeSegmentZKMetadata;
}