Examples with ScanBasedQueryProcessor com.linkedin.pinot.tools.scan.query.ScanBasedQueryProcessor used on opensource projects

Example 1 with ScanBasedQueryProcessor

use of com.linkedin.pinot.tools.scan.query.ScanBasedQueryProcessor in project pinot by linkedin.

the class HybridClusterScanComparisonIntegrationTest method configureScanBasedComparator.

private void configureScanBasedComparator(List<Pair<File, File>> enabledRealtimeSegments) throws Exception {
    // Deduplicate overlapping realtime and offline segments
    Set<String> enabledAvroFiles = new HashSet<String>();
    List<File> segmentsToUnpack = new ArrayList<File>();
    LOGGER.info("Offline avro to segment map {}", _offlineAvroToSegmentMap);
    LOGGER.info("Enabled realtime segments {}", enabledRealtimeSegments);
    for (Map.Entry<File, File> avroAndSegment : _offlineAvroToSegmentMap.entrySet()) {
        if (!enabledAvroFiles.contains(avroAndSegment.getKey().getName())) {
            enabledAvroFiles.add(avroAndSegment.getKey().getName());
            segmentsToUnpack.add(avroAndSegment.getValue());
        }
    }
    for (Pair<File, File> avroAndSegment : enabledRealtimeSegments) {
        if (!enabledAvroFiles.contains(avroAndSegment.getLeft().getName())) {
            enabledAvroFiles.add(avroAndSegment.getLeft().getName());
            segmentsToUnpack.add(avroAndSegment.getRight());
        }
    }
    LOGGER.info("Enabled Avro files {}", enabledAvroFiles);
    // Unpack enabled segments
    ensureDirectoryExistsAndIsEmpty(_unpackedSegments);
    for (File file : segmentsToUnpack) {
        LOGGER.info("Unpacking file {}", file);
        TarGzCompressionUtils.unTar(file, _unpackedSegments);
    }
    _scanBasedQueryProcessor = new ScanBasedQueryProcessor(_unpackedSegments.getAbsolutePath());
}

Example 2 with ScanBasedQueryProcessor

use of com.linkedin.pinot.tools.scan.query.ScanBasedQueryProcessor in project pinot by linkedin.

the class QueryComparison method runFunctionMode.

private void runFunctionMode() throws Exception {
    BufferedReader resultReader = null;
    ScanBasedQueryProcessor scanBasedQueryProcessor = null;
    try (BufferedReader queryReader = new BufferedReader(new InputStreamReader(new FileInputStream(_queryFile), "UTF8"))) {
        if (_resultFile == null) {
            scanBasedQueryProcessor = new ScanBasedQueryProcessor(_segmentsDir.getAbsolutePath());
        } else {
            resultReader = new BufferedReader(new InputStreamReader(new FileInputStream(_resultFile), "UTF8"));
        }
        int passed = 0;
        int total = 0;
        String query;
        while ((query = queryReader.readLine()) != null) {
            if (query.isEmpty() || query.startsWith("#")) {
                continue;
            }
            JSONObject expectedJson = null;
            try {
                if (resultReader != null) {
                    expectedJson = new JSONObject(resultReader.readLine());
                } else {
                    QueryResponse expectedResponse = scanBasedQueryProcessor.processQuery(query);
                    expectedJson = new JSONObject(new ObjectMapper().writeValueAsString(expectedResponse));
                }
            } catch (Exception e) {
                LOGGER.error("Comparison FAILED: Id: {} Exception caught while getting expected response for query: '{}'", total, query, e);
            }
            JSONObject actualJson = null;
            if (expectedJson != null) {
                try {
                    actualJson = new JSONObject(_clusterStarter.query(query));
                } catch (Exception e) {
                    LOGGER.error("Comparison FAILED: Id: {} Exception caught while running query: '{}'", total, query, e);
                }
            }
            if (expectedJson != null && actualJson != null) {
                try {
                    if (compare(actualJson, expectedJson)) {
                        passed++;
                        LOGGER.info("Comparison PASSED: Id: {} actual Time: {} ms expected Time: {} ms Docs Scanned: {}", total, actualJson.get(TIME_USED_MS), expectedJson.get(TIME_USED_MS), actualJson.get(NUM_DOCS_SCANNED));
                        LOGGER.debug("actual Response: {}", actualJson);
                        LOGGER.debug("expected Response: {}", expectedJson);
                    } else {
                        LOGGER.error("Comparison FAILED: Id: {} query: {}", query);
                        LOGGER.info("actual Response: {}", actualJson);
                        LOGGER.info("expected Response: {}", expectedJson);
                    }
                } catch (Exception e) {
                    LOGGER.error("Comparison FAILED: Id: {} Exception caught while comparing query: '{}' actual response: {}, expected response: {}", total, query, actualJson, expectedJson, e);
                }
            }
            total++;
        }
        LOGGER.info("Total {} out of {} queries passed.", passed, total);
    } finally {
        if (resultReader != null) {
            resultReader.close();
        }
    }
}