use of org.apache.helix.model.IdealState in project pinot by linkedin.
the class PinotRealtimeSegmentManager method assignRealtimeSegmentsToServerInstancesIfNecessary.
private synchronized void assignRealtimeSegmentsToServerInstancesIfNecessary() throws JSONException, IOException {
// Fetch current ideal state snapshot
Map<String, IdealState> idealStateMap = new HashMap<String, IdealState>();
for (String resource : _pinotHelixResourceManager.getAllRealtimeTables()) {
final String tableName = TableNameBuilder.extractRawTableName(resource);
AbstractTableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableName, TableType.REALTIME);
KafkaStreamMetadata metadata = new KafkaStreamMetadata(tableConfig.getIndexingConfig().getStreamConfigs());
if (metadata.hasHighLevelKafkaConsumerType()) {
idealStateMap.put(resource, _pinotHelixResourceManager.getHelixAdmin().getResourceIdealState(_pinotHelixResourceManager.getHelixClusterName(), resource));
} else {
LOGGER.debug("Not considering table {} for realtime segment assignment");
}
}
List<Pair<String, String>> listOfSegmentsToAddToInstances = new ArrayList<Pair<String, String>>();
for (String resource : idealStateMap.keySet()) {
try {
IdealState state = idealStateMap.get(resource);
// Are there any partitions?
if (state.getPartitionSet().size() == 0) {
// No, this is a brand new ideal state, so we will add one new segment to every partition and replica
List<String> instancesInResource = new ArrayList<String>();
try {
instancesInResource.addAll(_pinotHelixResourceManager.getServerInstancesForTable(resource, TableType.REALTIME));
} catch (Exception e) {
LOGGER.error("Caught exception while fetching instances for resource {}", resource, e);
_controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
}
// Assign a new segment to all server instances
for (String instanceId : instancesInResource) {
InstanceZKMetadata instanceZKMetadata = _pinotHelixResourceManager.getInstanceZKMetadata(instanceId);
if (instanceZKMetadata == null) {
LOGGER.warn("Instance {} has no associated instance metadata in ZK, ignoring for segment assignment.", instanceId);
_controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
continue;
}
String groupId = instanceZKMetadata.getGroupId(resource);
String partitionId = instanceZKMetadata.getPartition(resource);
if (groupId != null && !groupId.isEmpty() && partitionId != null && !partitionId.isEmpty()) {
listOfSegmentsToAddToInstances.add(new Pair<String, String>(new HLCSegmentName(groupId, partitionId, String.valueOf(System.currentTimeMillis())).getSegmentName(), instanceId));
} else {
LOGGER.warn("Instance {} has invalid groupId ({}) and/or partitionId ({}) for resource {}, ignoring for segment assignment.", instanceId, groupId, partitionId, resource);
_controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
}
}
} else {
// Add all server instances to the list of instances for which to assign a realtime segment
Set<String> instancesToAssignRealtimeSegment = new HashSet<String>();
try {
instancesToAssignRealtimeSegment.addAll(_pinotHelixResourceManager.getServerInstancesForTable(resource, TableType.REALTIME));
} catch (Exception e) {
LOGGER.error("Caught exception while fetching instances for resource {}", resource, e);
_controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
}
// Remove server instances that are currently processing a segment
for (String partition : state.getPartitionSet()) {
// Helix partition is the segment name
if (SegmentName.isHighLevelConsumerSegmentName(partition)) {
HLCSegmentName segName = new HLCSegmentName(partition);
RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider.getRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), segName.getTableName(), partition);
if (realtimeSegmentZKMetadata == null) {
// Segment was deleted by retention manager.
continue;
}
if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) {
instancesToAssignRealtimeSegment.removeAll(state.getInstanceSet(partition));
}
}
}
// Assign a new segment to the server instances not currently processing this segment
for (String instanceId : instancesToAssignRealtimeSegment) {
InstanceZKMetadata instanceZKMetadata = _pinotHelixResourceManager.getInstanceZKMetadata(instanceId);
String groupId = instanceZKMetadata.getGroupId(resource);
String partitionId = instanceZKMetadata.getPartition(resource);
listOfSegmentsToAddToInstances.add(new Pair<String, String>(new HLCSegmentName(groupId, partitionId, String.valueOf(System.currentTimeMillis())).getSegmentName(), instanceId));
}
}
} catch (Exception e) {
LOGGER.warn("Caught exception while processing resource {}, skipping.", resource, e);
_controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
}
}
LOGGER.info("Computed list of new segments to add : " + Arrays.toString(listOfSegmentsToAddToInstances.toArray()));
// Add the new segments to the server instances
for (final Pair<String, String> segmentIdAndInstanceId : listOfSegmentsToAddToInstances) {
final String segmentId = segmentIdAndInstanceId.getFirst();
final String instanceName = segmentIdAndInstanceId.getSecond();
try {
final HLCSegmentName segName = new HLCSegmentName(segmentId);
String resourceName = segName.getTableName();
// Does the ideal state already contain this segment?
if (!idealStateMap.get(resourceName).getPartitionSet().contains(segmentId)) {
// No, add it
// Create the realtime segment metadata
RealtimeSegmentZKMetadata realtimeSegmentMetadataToAdd = new RealtimeSegmentZKMetadata();
realtimeSegmentMetadataToAdd.setTableName(TableNameBuilder.extractRawTableName(resourceName));
realtimeSegmentMetadataToAdd.setSegmentType(SegmentType.REALTIME);
realtimeSegmentMetadataToAdd.setStatus(Status.IN_PROGRESS);
realtimeSegmentMetadataToAdd.setSegmentName(segmentId);
// Add the new metadata to the property store
ZKMetadataProvider.setRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), realtimeSegmentMetadataToAdd);
// Update the ideal state to add the new realtime segment
HelixHelper.updateIdealState(_pinotHelixResourceManager.getHelixZkManager(), resourceName, new Function<IdealState, IdealState>() {
@Override
public IdealState apply(IdealState idealState) {
return PinotTableIdealStateBuilder.addNewRealtimeSegmentToIdealState(segmentId, idealState, instanceName);
}
}, RetryPolicies.exponentialBackoffRetryPolicy(5, 500L, 2.0f));
}
} catch (Exception e) {
LOGGER.warn("Caught exception while processing segment {} for instance {}, skipping.", segmentId, instanceName, e);
_controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
}
}
}
use of org.apache.helix.model.IdealState in project pinot by linkedin.
the class RetentionManager method scanSegmentMetadataAndPurge.
private void scanSegmentMetadataAndPurge() {
for (String tableName : _segmentMetadataMap.keySet()) {
List<SegmentZKMetadata> segmentZKMetadataList = _segmentMetadataMap.get(tableName);
List<String> segmentsToDelete = new ArrayList<>(128);
IdealState idealState = null;
try {
if (TableNameBuilder.getTableTypeFromTableName(tableName).equals(TableType.REALTIME)) {
idealState = HelixHelper.getTableIdealState(_pinotHelixResourceManager.getHelixZkManager(), tableName);
}
} catch (Exception e) {
LOGGER.warn("Could not get idealstate for {}", tableName, e);
// Ignore, worst case we have some old inactive segments in place.
}
for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
RetentionStrategy deletionStrategy;
deletionStrategy = _tableDeletionStrategy.get(tableName);
if (deletionStrategy == null) {
LOGGER.info("No Retention strategy found for segment: {}", segmentZKMetadata.getSegmentName());
continue;
}
if (segmentZKMetadata instanceof RealtimeSegmentZKMetadata) {
final RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = (RealtimeSegmentZKMetadata) segmentZKMetadata;
if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) {
final String segmentId = realtimeSegmentZKMetadata.getSegmentName();
if (SegmentName.isHighLevelConsumerSegmentName(segmentId)) {
continue;
}
// auto-create LLC segments.
if (shouldDeleteInProgressLLCSegment(segmentId, idealState, realtimeSegmentZKMetadata)) {
segmentsToDelete.add(segmentId);
}
continue;
}
}
if (deletionStrategy.isPurgeable(segmentZKMetadata)) {
LOGGER.info("Marking segment to delete: {}", segmentZKMetadata.getSegmentName());
segmentsToDelete.add(segmentZKMetadata.getSegmentName());
}
}
if (segmentsToDelete.size() > 0) {
LOGGER.info("Trying to delete {} segments for table {}", segmentsToDelete.size(), tableName);
_pinotHelixResourceManager.deleteSegments(tableName, segmentsToDelete);
}
}
}
use of org.apache.helix.model.IdealState in project pinot by linkedin.
the class PinotTableIdealStateBuilder method buildEmptyIdealStateFor.
/**
*
* Building an empty idealState for a given table.
* Used when creating a new table.
*
* @param tableName resource name
* @param numCopies is the number of replicas
* @return
*/
public static IdealState buildEmptyIdealStateFor(String tableName, int numCopies) {
final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(tableName);
final int replicas = numCopies;
customModeIdealStateBuilder.setStateModel(PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL).setNumPartitions(0).setNumReplica(replicas).setMaxPartitionsPerNode(1);
final IdealState idealState = customModeIdealStateBuilder.build();
idealState.setInstanceGroupTag(tableName);
return idealState;
}
use of org.apache.helix.model.IdealState in project pinot by linkedin.
the class PinotTableIdealStateBuilder method buildInitialHighLevelRealtimeIdealStateFor.
public static IdealState buildInitialHighLevelRealtimeIdealStateFor(String realtimeTableName, AbstractTableConfig realtimeTableConfig, HelixAdmin helixAdmin, String helixClusterName, ZkHelixPropertyStore<ZNRecord> zkHelixPropertyStore) {
String realtimeServerTenant = ControllerTenantNameBuilder.getRealtimeTenantNameForTenant(realtimeTableConfig.getTenantConfig().getServer());
final List<String> realtimeInstances = helixAdmin.getInstancesInClusterWithTag(helixClusterName, realtimeServerTenant);
IdealState idealState = buildEmptyKafkaConsumerRealtimeIdealStateFor(realtimeTableName, 1);
if (realtimeInstances.size() % Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()) != 0) {
throw new RuntimeException("Number of instance in current tenant should be an integer multiples of the number of replications");
}
setupInstanceConfigForKafkaHighLevelConsumer(realtimeTableName, realtimeInstances.size(), Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()), realtimeTableConfig.getIndexingConfig().getStreamConfigs(), zkHelixPropertyStore, realtimeInstances);
return idealState;
}
use of org.apache.helix.model.IdealState in project pinot by linkedin.
the class PinotTableIdealStateBuilder method removeBrokerResourceFromIdealStateFor.
/**
*
* @param brokerResourceName
* @param helixAdmin
* @param helixClusterName
* @return
*/
public static IdealState removeBrokerResourceFromIdealStateFor(String brokerResourceName, HelixAdmin helixAdmin, String helixClusterName) {
final IdealState currentIdealState = helixAdmin.getResourceIdealState(helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
final Set<String> currentInstanceSet = currentIdealState.getInstanceSet(brokerResourceName);
if (!currentInstanceSet.isEmpty() && currentIdealState.getPartitionSet().contains(brokerResourceName)) {
currentIdealState.getPartitionSet().remove(brokerResourceName);
} else {
throw new RuntimeException("Cannot found broker resource - " + brokerResourceName + " in broker resource ");
}
return currentIdealState;
}
Aggregations