Examples with ServerInstance com.linkedin.pinot.common.response.ServerInstance used on opensource projects

Example 31 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class RandomRoutingTableTest method testHelixExternalViewBasedRoutingTable.

@Test
public void testHelixExternalViewBasedRoutingTable() throws Exception {
    URL resourceUrl = getClass().getClassLoader().getResource("SampleExternalView.json");
    Assert.assertNotNull(resourceUrl);
    String fileName = resourceUrl.getFile();
    String tableName = "testTable_OFFLINE";
    InputStream evInputStream = new FileInputStream(fileName);
    ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer();
    ZNRecord externalViewRecord = (ZNRecord) znRecordSerializer.deserialize(IOUtils.toByteArray(evInputStream));
    int totalRuns = 10000;
    RoutingTableBuilder routingStrategy = new BalancedRandomRoutingTableBuilder(10);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, new PercentageBasedRoutingTableSelector(), null, new BaseConfiguration());
    routingTable.setSmallClusterRoutingTableBuilder(routingStrategy);
    ExternalView externalView = new ExternalView(externalViewRecord);
    routingTable.markDataResourceOnline(tableName, externalView, getInstanceConfigs(externalView));
    double[] globalArrays = new double[9];
    for (int numRun = 0; numRun < totalRuns; ++numRun) {
        RoutingTableLookupRequest request = new RoutingTableLookupRequest(tableName, Collections.<String>emptyList());
        Map<ServerInstance, SegmentIdSet> serversMap = routingTable.findServers(request);
        TreeSet<ServerInstance> serverInstances = new TreeSet<ServerInstance>(serversMap.keySet());
        int i = 0;
        double[] arrays = new double[9];
        for (ServerInstance serverInstance : serverInstances) {
            globalArrays[i] += serversMap.get(serverInstance).getSegments().size();
            arrays[i++] = serversMap.get(serverInstance).getSegments().size();
        }
        for (int j = 0; i < arrays.length; ++j) {
            Assert.assertTrue(arrays[j] / totalRuns <= 31);
            Assert.assertTrue(arrays[j] / totalRuns >= 28);
        }
    //      System.out.println(Arrays.toString(arrays) + " : " + new StandardDeviation().evaluate(arrays) + " : " + new Mean().evaluate(arrays));
    }
    for (int i = 0; i < globalArrays.length; ++i) {
        Assert.assertTrue(globalArrays[i] / totalRuns <= 31);
        Assert.assertTrue(globalArrays[i] / totalRuns >= 28);
    }
//    System.out.println(Arrays.toString(globalArrays) + " : " + new StandardDeviation().evaluate(globalArrays) + " : "
//        + new Mean().evaluate(globalArrays));
}

Example 32 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class ReplicaSelectionTest method testRoundRobinSelection.

@Test
public void testRoundRobinSelection() {
    ReplicaSelection sel1 = new RoundRobinReplicaSelection();
    ServerInstance s1 = new ServerInstance("localhost", 8080);
    ServerInstance s2 = new ServerInstance("localhost", 8081);
    ServerInstance s3 = new ServerInstance("localhost", 8082);
    ServerInstance[] servers = { s1, s2, s3 };
    // Verify for an empty list, selectServer returns null
    List<ServerInstance> candidates = new ArrayList<ServerInstance>();
    Assert.assertNull(sel1.selectServer(new SegmentId("1"), candidates, null));
}

Example 33 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class PerTableRoutingConfig method buildRequestRoutingMap.

//
//  /**
//   * Builds a map needed for routing the partitions in the partition-group passed.
//   * There could be different set of servers for each partition in the passed partition-group.
//   *
//   * @param pg segmentSet for which the routing map needs to be built.
//   * @return
//   */
//  public Map<SegmentIdSet, List<ServerInstance>> buildRequestRoutingMap() {
//    Map<SegmentIdSet, List<ServerInstance>> resultMap = new HashMap<SegmentIdSet, List<ServerInstance>>();
//
//    /**
//     * NOTE: After we removed the concept of partition, this needed rewriting.
//     * For now, The File-based routing config maps nodeIds to Instances instead of segments to instances.
//     * This is because, it becomes difficult for configuring all segments in routing config. Instead,
//     * we configure the number of nodes that constitute a replica-set. For each node, different instances
//     * (as comma-seperated list) is provided. we pick one instance from each node.
//     *
//     */
//    for (Entry<Integer, List<ServerInstance>> e : _nodeToInstancesMap.entrySet()) {
//      SegmentId id = new SegmentId("" + e.getKey());
//      SegmentIdSet idSet = new SegmentIdSet();
//      idSet.addSegment(id);
//      resultMap.put(idSet, e.getValue());
//    }
//
//    // Add default
//    SegmentId id = new SegmentId("default");
//    SegmentIdSet idSet = new SegmentIdSet();
//    idSet.addSegment(id);
//    resultMap.put(idSet, _defaultServers);
//    return resultMap;
//  }
/**
   * Builds a map needed for routing the partitions in the partition-group passed.
   * There could be different set of servers for each partition in the passed partition-group.
   *
   * @return
   */
public Map<ServerInstance, SegmentIdSet> buildRequestRoutingMap() {
    Map<ServerInstance, SegmentIdSet> resultMap = new HashMap<ServerInstance, SegmentIdSet>();
    for (ServerInstance serverInstance : _defaultServers) {
        SegmentId id = new SegmentId("default");
        SegmentIdSet idSet = new SegmentIdSet();
        idSet.addSegment(id);
        resultMap.put(serverInstance, idSet);
    }
    return resultMap;
}

Example 34 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class BrokerReduceService method reduceOnDataTable.

@Nonnull
@Override
public BrokerResponseNative reduceOnDataTable(@Nonnull BrokerRequest brokerRequest, @Nonnull Map<ServerInstance, DataTable> dataTableMap, @Nullable BrokerMetrics brokerMetrics) {
    if (dataTableMap.size() == 0) {
        // Empty response.
        return BrokerResponseNative.empty();
    }
    BrokerResponseNative brokerResponseNative = new BrokerResponseNative();
    List<QueryProcessingException> processingExceptions = brokerResponseNative.getProcessingExceptions();
    long numDocsScanned = 0L;
    long numEntriesScannedInFilter = 0L;
    long numEntriesScannedPostFilter = 0L;
    long numTotalRawDocs = 0L;
    // Cache a data schema from data tables (try to cache one with data rows associated with it).
    DataSchema cachedDataSchema = null;
    // Process server response metadata.
    Iterator<Map.Entry<ServerInstance, DataTable>> iterator = dataTableMap.entrySet().iterator();
    while (iterator.hasNext()) {
        Map.Entry<ServerInstance, DataTable> entry = iterator.next();
        ServerInstance serverInstance = entry.getKey();
        DataTable dataTable = entry.getValue();
        Map<String, String> metadata = dataTable.getMetadata();
        // Reduce on trace info.
        if (brokerRequest.isEnableTrace()) {
            brokerResponseNative.getTraceInfo().put(serverInstance.getHostname(), metadata.get(DataTable.TRACE_INFO_METADATA_KEY));
        }
        // Reduce on exceptions.
        for (String key : metadata.keySet()) {
            if (key.startsWith(DataTable.EXCEPTION_METADATA_KEY)) {
                processingExceptions.add(new QueryProcessingException(Integer.parseInt(key.substring(9)), metadata.get(key)));
            }
        }
        // Reduce on execution statistics.
        String numDocsScannedString = metadata.get(DataTable.NUM_DOCS_SCANNED_METADATA_KEY);
        if (numDocsScannedString != null) {
            numDocsScanned += Long.parseLong(numDocsScannedString);
        }
        String numEntriesScannedInFilterString = metadata.get(DataTable.NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY);
        if (numEntriesScannedInFilterString != null) {
            numEntriesScannedInFilter += Long.parseLong(numEntriesScannedInFilterString);
        }
        String numEntriesScannedPostFilterString = metadata.get(DataTable.NUM_ENTRIES_SCANNED_POST_FILTER_METADATA_KEY);
        if (numEntriesScannedPostFilterString != null) {
            numEntriesScannedPostFilter += Long.parseLong(numEntriesScannedPostFilterString);
        }
        String numTotalRawDocsString = metadata.get(DataTable.TOTAL_DOCS_METADATA_KEY);
        if (numTotalRawDocsString != null) {
            numTotalRawDocs += Long.parseLong(numTotalRawDocsString);
        }
        // After processing the metadata, remove data tables without data rows inside.
        DataSchema dataSchema = dataTable.getDataSchema();
        if (dataSchema == null) {
            iterator.remove();
        } else {
            // Try to cache a data table with data rows inside, or cache one with data schema inside.
            if (dataTable.getNumberOfRows() == 0) {
                if (cachedDataSchema == null) {
                    cachedDataSchema = dataSchema;
                }
                iterator.remove();
            } else {
                cachedDataSchema = dataSchema;
            }
        }
    }
    // Set execution statistics.
    brokerResponseNative.setNumDocsScanned(numDocsScanned);
    brokerResponseNative.setNumEntriesScannedInFilter(numEntriesScannedInFilter);
    brokerResponseNative.setNumEntriesScannedPostFilter(numEntriesScannedPostFilter);
    brokerResponseNative.setTotalDocs(numTotalRawDocs);
    // Update broker metrics.
    String tableName = brokerRequest.getQuerySource().getTableName();
    if (brokerMetrics != null) {
        brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.DOCUMENTS_SCANNED, numDocsScanned);
        brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.ENTRIES_SCANNED_IN_FILTER, numEntriesScannedInFilter);
        brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.ENTRIES_SCANNED_POST_FILTER, numEntriesScannedPostFilter);
    }
    if (dataTableMap.isEmpty()) {
        // This will only happen to selection query.
        if (cachedDataSchema != null) {
            List<String> selectionColumns = SelectionOperatorUtils.getSelectionColumns(brokerRequest.getSelections().getSelectionColumns(), cachedDataSchema);
            brokerResponseNative.setSelectionResults(new SelectionResults(selectionColumns, new ArrayList<Serializable[]>(0)));
        }
    } else {
        // Reduce server responses data and set query results into the broker response.
        assert cachedDataSchema != null;
        if (brokerRequest.isSetSelections()) {
            // Selection query.
            // For data table map with more than one data tables, remove conflicting data tables.
            DataSchema masterDataSchema = cachedDataSchema.clone();
            if (dataTableMap.size() > 1) {
                List<String> droppedServers = removeConflictingResponses(masterDataSchema, dataTableMap);
                if (!droppedServers.isEmpty()) {
                    String errorMessage = QueryException.MERGE_RESPONSE_ERROR.getMessage() + ": responses for table: " + tableName + " from servers: " + droppedServers + " got dropped due to data schema inconsistency.";
                    LOGGER.error(errorMessage);
                    if (brokerMetrics != null) {
                        brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.RESPONSE_MERGE_EXCEPTIONS, 1);
                    }
                    brokerResponseNative.addToExceptions(new QueryProcessingException(QueryException.MERGE_RESPONSE_ERROR_CODE, errorMessage));
                }
            }
            setSelectionResults(brokerResponseNative, brokerRequest.getSelections(), dataTableMap, masterDataSchema);
        } else {
            // Aggregation query.
            AggregationFunction[] aggregationFunctions = AggregationFunctionUtils.getAggregationFunctions(brokerRequest.getAggregationsInfo());
            if (!brokerRequest.isSetGroupBy()) {
                // Aggregation only query.
                setAggregationResults(brokerResponseNative, aggregationFunctions, dataTableMap, cachedDataSchema);
            } else {
                // Aggregation group-by query.
                setGroupByResults(brokerResponseNative, aggregationFunctions, brokerRequest.getGroupBy(), dataTableMap);
            }
        }
    }
    return brokerResponseNative;
}

Example 35 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class BrokerReduceService method removeConflictingResponses.

/**
   * Given a data schema, remove data tables that are not compatible with this data schema.
   * <p>Upgrade the data schema passed in to cover all remaining data schemas.
   *
   * @param dataSchema data schema.
   * @param dataTableMap map from server to data table.
   * @return list of server names where the data table got removed.
   */
@Nonnull
private List<String> removeConflictingResponses(@Nonnull DataSchema dataSchema, @Nonnull Map<ServerInstance, DataTable> dataTableMap) {
    List<String> droppedServers = new ArrayList<>();
    Iterator<Map.Entry<ServerInstance, DataTable>> iterator = dataTableMap.entrySet().iterator();
    while (iterator.hasNext()) {
        Map.Entry<ServerInstance, DataTable> entry = iterator.next();
        DataSchema dataSchemaToCompare = entry.getValue().getDataSchema();
        assert dataSchemaToCompare != null;
        if (!dataSchema.isTypeCompatibleWith(dataSchemaToCompare)) {
            droppedServers.add(entry.getKey().toString());
            iterator.remove();
        } else {
            dataSchema.upgradeToCover(dataSchemaToCompare);
        }
    }
    return droppedServers;
}