Examples with Pair com.linkedin.pinot.core.query.utils.Pair used on opensource projects

Example 1 with Pair

use of com.linkedin.pinot.core.query.utils.Pair in project pinot by linkedin.

the class PinotRealtimeSegmentManager method assignRealtimeSegmentsToServerInstancesIfNecessary.

private synchronized void assignRealtimeSegmentsToServerInstancesIfNecessary() throws JSONException, IOException {
    // Fetch current ideal state snapshot
    Map<String, IdealState> idealStateMap = new HashMap<String, IdealState>();
    for (String resource : _pinotHelixResourceManager.getAllRealtimeTables()) {
        final String tableName = TableNameBuilder.extractRawTableName(resource);
        AbstractTableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableName, TableType.REALTIME);
        KafkaStreamMetadata metadata = new KafkaStreamMetadata(tableConfig.getIndexingConfig().getStreamConfigs());
        if (metadata.hasHighLevelKafkaConsumerType()) {
            idealStateMap.put(resource, _pinotHelixResourceManager.getHelixAdmin().getResourceIdealState(_pinotHelixResourceManager.getHelixClusterName(), resource));
        } else {
            LOGGER.debug("Not considering table {} for realtime segment assignment");
        }
    }
    List<Pair<String, String>> listOfSegmentsToAddToInstances = new ArrayList<Pair<String, String>>();
    for (String resource : idealStateMap.keySet()) {
        try {
            IdealState state = idealStateMap.get(resource);
            // Are there any partitions?
            if (state.getPartitionSet().size() == 0) {
                // No, this is a brand new ideal state, so we will add one new segment to every partition and replica
                List<String> instancesInResource = new ArrayList<String>();
                try {
                    instancesInResource.addAll(_pinotHelixResourceManager.getServerInstancesForTable(resource, TableType.REALTIME));
                } catch (Exception e) {
                    LOGGER.error("Caught exception while fetching instances for resource {}", resource, e);
                    _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
                }
                // Assign a new segment to all server instances
                for (String instanceId : instancesInResource) {
                    InstanceZKMetadata instanceZKMetadata = _pinotHelixResourceManager.getInstanceZKMetadata(instanceId);
                    if (instanceZKMetadata == null) {
                        LOGGER.warn("Instance {} has no associated instance metadata in ZK, ignoring for segment assignment.", instanceId);
                        _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
                        continue;
                    }
                    String groupId = instanceZKMetadata.getGroupId(resource);
                    String partitionId = instanceZKMetadata.getPartition(resource);
                    if (groupId != null && !groupId.isEmpty() && partitionId != null && !partitionId.isEmpty()) {
                        listOfSegmentsToAddToInstances.add(new Pair<String, String>(new HLCSegmentName(groupId, partitionId, String.valueOf(System.currentTimeMillis())).getSegmentName(), instanceId));
                    } else {
                        LOGGER.warn("Instance {} has invalid groupId ({}) and/or partitionId ({}) for resource {}, ignoring for segment assignment.", instanceId, groupId, partitionId, resource);
                        _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
                    }
                }
            } else {
                // Add all server instances to the list of instances for which to assign a realtime segment
                Set<String> instancesToAssignRealtimeSegment = new HashSet<String>();
                try {
                    instancesToAssignRealtimeSegment.addAll(_pinotHelixResourceManager.getServerInstancesForTable(resource, TableType.REALTIME));
                } catch (Exception e) {
                    LOGGER.error("Caught exception while fetching instances for resource {}", resource, e);
                    _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
                }
                // Remove server instances that are currently processing a segment
                for (String partition : state.getPartitionSet()) {
                    // Helix partition is the segment name
                    if (SegmentName.isHighLevelConsumerSegmentName(partition)) {
                        HLCSegmentName segName = new HLCSegmentName(partition);
                        RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider.getRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), segName.getTableName(), partition);
                        if (realtimeSegmentZKMetadata == null) {
                            // Segment was deleted by retention manager.
                            continue;
                        }
                        if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) {
                            instancesToAssignRealtimeSegment.removeAll(state.getInstanceSet(partition));
                        }
                    }
                }
                // Assign a new segment to the server instances not currently processing this segment
                for (String instanceId : instancesToAssignRealtimeSegment) {
                    InstanceZKMetadata instanceZKMetadata = _pinotHelixResourceManager.getInstanceZKMetadata(instanceId);
                    String groupId = instanceZKMetadata.getGroupId(resource);
                    String partitionId = instanceZKMetadata.getPartition(resource);
                    listOfSegmentsToAddToInstances.add(new Pair<String, String>(new HLCSegmentName(groupId, partitionId, String.valueOf(System.currentTimeMillis())).getSegmentName(), instanceId));
                }
            }
        } catch (Exception e) {
            LOGGER.warn("Caught exception while processing resource {}, skipping.", resource, e);
            _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
        }
    }
    LOGGER.info("Computed list of new segments to add : " + Arrays.toString(listOfSegmentsToAddToInstances.toArray()));
    // Add the new segments to the server instances
    for (final Pair<String, String> segmentIdAndInstanceId : listOfSegmentsToAddToInstances) {
        final String segmentId = segmentIdAndInstanceId.getFirst();
        final String instanceName = segmentIdAndInstanceId.getSecond();
        try {
            final HLCSegmentName segName = new HLCSegmentName(segmentId);
            String resourceName = segName.getTableName();
            // Does the ideal state already contain this segment?
            if (!idealStateMap.get(resourceName).getPartitionSet().contains(segmentId)) {
                // No, add it
                // Create the realtime segment metadata
                RealtimeSegmentZKMetadata realtimeSegmentMetadataToAdd = new RealtimeSegmentZKMetadata();
                realtimeSegmentMetadataToAdd.setTableName(TableNameBuilder.extractRawTableName(resourceName));
                realtimeSegmentMetadataToAdd.setSegmentType(SegmentType.REALTIME);
                realtimeSegmentMetadataToAdd.setStatus(Status.IN_PROGRESS);
                realtimeSegmentMetadataToAdd.setSegmentName(segmentId);
                // Add the new metadata to the property store
                ZKMetadataProvider.setRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), realtimeSegmentMetadataToAdd);
                // Update the ideal state to add the new realtime segment
                HelixHelper.updateIdealState(_pinotHelixResourceManager.getHelixZkManager(), resourceName, new Function<IdealState, IdealState>() {

                    @Override
                    public IdealState apply(IdealState idealState) {
                        return PinotTableIdealStateBuilder.addNewRealtimeSegmentToIdealState(segmentId, idealState, instanceName);
                    }
                }, RetryPolicies.exponentialBackoffRetryPolicy(5, 500L, 2.0f));
            }
        } catch (Exception e) {
            LOGGER.warn("Caught exception while processing segment {} for instance {}, skipping.", segmentId, instanceName, e);
            _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L);
        }
    }
}

Example 2 with Pair

use of com.linkedin.pinot.core.query.utils.Pair in project pinot by linkedin.

the class Projection method transformFromIdToValues.

public ResultTable transformFromIdToValues(ResultTable resultTable, Map<String, Dictionary> dictionaryMap, boolean addCountStar) {
    List<Pair> columnList = resultTable.getColumnList();
    for (ResultTable.Row row : resultTable) {
        int colId = 0;
        for (Object object : row) {
            String column = (String) columnList.get(colId).getFirst();
            Dictionary dictionary = dictionaryMap.get(column);
            if (object instanceof Object[]) {
                Object[] objArray = (Object[]) object;
                Object[] valArray = new Object[objArray.length];
                for (int i = 0; i < objArray.length; ++i) {
                    int dictId = (int) objArray[i];
                    valArray[i] = dictionary.get(dictId);
                }
                row.set(colId, valArray);
            } else {
                int dictId = (int) object;
                row.set(colId, dictionary.get(dictId));
            }
            ++colId;
        }
    }
    // Add additional column for count(*)
    if (addCountStar) {
        for (ResultTable.Row row : resultTable) {
            row.add(1);
        }
        resultTable.addCountStarColumn();
    }
    return resultTable;
}

Example 3 with Pair

use of com.linkedin.pinot.core.query.utils.Pair in project pinot by linkedin.

the class QueryResponse method buildAggregationGroupByResult.

private void buildAggregationGroupByResult(ResultTable resultTable) {
    List<Pair> columnList = resultTable.getColumnList();
    List<String> groupByColumns = new ArrayList<>();
    _aggregationResults = new ArrayList<>();
    for (Pair pair : columnList) {
        if (pair.getSecond() == null) {
            groupByColumns.add(pair.getFirst().toString());
        }
    }
    int numGroupByColumns = groupByColumns.size();
    List<List<String>> groupValueStrings = new ArrayList<>();
    for (ResultTable.Row row : resultTable) {
        int colId = 0;
        List<String> group = new ArrayList<>();
        for (Object value : row) {
            // All group by columns come first.
            if (columnList.get(colId).getSecond() != null) {
                break;
            }
            if (value instanceof Object[]) {
                Object[] array = (Object[]) value;
                for (Object obj : array) {
                    group.add(obj.toString());
                }
            } else {
                group.add(value.toString());
            }
            ++colId;
        }
        groupValueStrings.add(group);
    }
    for (int colId = numGroupByColumns; colId < columnList.size(); ++colId) {
        String function = resultTable.getFunction(colId);
        if (function.equalsIgnoreCase("count_*")) {
            function = "count_star";
        }
        int rowId = 0;
        List<GroupValue> groupValues = new ArrayList<>();
        for (ResultTable.Row row : resultTable) {
            String value = row.get(colId).toString();
            GroupValue groupValue = new GroupValue(value, groupValueStrings.get(rowId));
            groupValues.add(groupValue);
            ++rowId;
        }
        AggregationResult aggregationResult = new AggregationResult(groupValues, groupByColumns, function);
        _aggregationResults.add(aggregationResult);
    }
}

Example 4 with Pair

use of com.linkedin.pinot.core.query.utils.Pair in project pinot by linkedin.

the class ResultTable method convertSumToAvgIfNeeded.

private void convertSumToAvgIfNeeded() {
    if (isEmpty()) {
        return;
    }
    List<Integer> averageColumns = new ArrayList<>();
    int i = 0;
    for (Pair pair : _columnList) {
        String function = (String) pair.getSecond();
        if (function != null && function.equalsIgnoreCase(AVERAGE)) {
            averageColumns.add(i);
        }
        ++i;
    }
    // No average columns found.
    if (averageColumns.isEmpty()) {
        return;
    }
    for (Row row : _rows) {
        for (int colId : averageColumns) {
            double[] value = (double[]) row.get(colId);
            double average = (value[1] != 0) ? (value[0] / value[1]) : 0;
            row.set(colId, average);
        }
    }
}

Example 5 with Pair

use of com.linkedin.pinot.core.query.utils.Pair in project pinot by linkedin.

the class ResultTable method computeDistinctCount.

private void computeDistinctCount() {
    if (isEmpty()) {
        return;
    }
    List<Integer> distinctCountColumns = new ArrayList<>();
    int i = 0;
    for (Pair pair : _columnList) {
        String function = (String) pair.getSecond();
        if (function != null && function.equalsIgnoreCase(DISTINCT_COUNT)) {
            distinctCountColumns.add(i);
        }
        ++i;
    }
    // No distinctCount columns found.
    if (distinctCountColumns.isEmpty()) {
        return;
    }
    for (Row row : _rows) {
        for (int colId : distinctCountColumns) {
            Set<String> distinctCountSet = (Set<String>) row.get(colId);
            row.set(colId, distinctCountSet.size());
        }
    }
}