Examples with ConnectorSplitSource com.facebook.presto.spi.ConnectorSplitSource used on opensource projects

Example 26 with ConnectorSplitSource

use of com.facebook.presto.spi.ConnectorSplitSource in project presto by prestodb.

the class TestPinotSplitManager method getSplitsHelper.

private List<PinotSplit> getSplitsHelper(PinotTableHandle pinotTable, int numSegmentsPerSplit, boolean forbidSegmentQueries) {
    PinotTableLayoutHandle pinotTableLayout = new PinotTableLayoutHandle(pinotTable);
    ConnectorSession session = createSessionWithNumSplits(numSegmentsPerSplit, forbidSegmentQueries, pinotConfig);
    ConnectorSplitSource splitSource = pinotSplitManager.getSplits(null, session, pinotTableLayout, null);
    List<PinotSplit> splits = new ArrayList<>();
    while (!splitSource.isFinished()) {
        splits.addAll(getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, 1000)).getSplits().stream().map(s -> (PinotSplit) s).collect(toList()));
    }
    return splits;
}

Example 27 with ConnectorSplitSource

use of com.facebook.presto.spi.ConnectorSplitSource in project presto by prestodb.

the class TestPrometheusSplit method testQueryDividedIntoSplitsFirstSplitHasRightTime.

@Test
public void testQueryDividedIntoSplitsFirstSplitHasRightTime() throws URISyntaxException {
    Instant now = LocalDateTime.of(2019, 10, 2, 7, 26, 56, 0).toInstant(UTC);
    PrometheusConnectorConfig config = getCommonConfig(prometheusHttpServer.resolve("/prometheus-data/prometheus-metrics.json"));
    PrometheusClient client = new PrometheusClient(config, METRIC_CODEC, TYPE_MANAGER);
    PrometheusTable table = client.getTable("default", "up");
    PrometheusTableHandle tableHandle = new PrometheusTableHandle("default", table.getName());
    TupleDomain<ColumnHandle> columnConstraints = TupleDomain.withColumnDomains(ImmutableMap.of(new PrometheusColumnHandle("value", BIGINT, 1), Domain.all(VARCHAR), new PrometheusColumnHandle("text", createUnboundedVarcharType(), 0), Domain.all(VARCHAR)));
    PrometheusTableLayoutHandle tableLayoutHandle = new PrometheusTableLayoutHandle(tableHandle, columnConstraints);
    PrometheusSplitManager splitManager = new PrometheusSplitManager(client, fixedClockAt(now), config);
    ConnectorSplitSource splits = splitManager.getSplits(null, null, tableLayoutHandle, null);
    PrometheusSplit split = (PrometheusSplit) splits.getNextBatch(NOT_PARTITIONED, 1).getNow(null).getSplits().get(0);
    String queryInSplit = split.getUri().getQuery();
    String timeShouldBe = decimalSecondString(now.toEpochMilli() - config.getMaxQueryRangeDuration().toMillis() + config.getQueryChunkSizeDuration().toMillis() - OFFSET_MILLIS * 20);
    assertEquals(queryInSplit, new URI("http://doesnotmatter:9090/api/v1/query?query=up[" + getQueryChunkSizeDurationAsPrometheusCompatibleDurationString(config) + "]" + "&time=" + timeShouldBe).getQuery());
}

Example 28 with ConnectorSplitSource

use of com.facebook.presto.spi.ConnectorSplitSource in project presto by prestodb.

the class TestPrometheusSplit method testQueryDividedIntoSplitsLastSplitHasRightTime.

@Test
public void testQueryDividedIntoSplitsLastSplitHasRightTime() throws URISyntaxException {
    Instant now = LocalDateTime.of(2019, 10, 2, 7, 26, 56, 0).toInstant(UTC);
    PrometheusConnectorConfig config = getCommonConfig(prometheusHttpServer.resolve("/prometheus-data/prometheus-metrics.json"));
    PrometheusClient client = new PrometheusClient(config, METRIC_CODEC, TYPE_MANAGER);
    PrometheusTable table = client.getTable("default", "up");
    PrometheusTableHandle tableHandle = new PrometheusTableHandle("default", table.getName());
    TupleDomain<ColumnHandle> columnConstraints = TupleDomain.withColumnDomains(ImmutableMap.of(new PrometheusColumnHandle("value", BIGINT, 1), Domain.all(VARCHAR), new PrometheusColumnHandle("text", createUnboundedVarcharType(), 0), Domain.all(VARCHAR)));
    PrometheusTableLayoutHandle tableLayoutHandle = new PrometheusTableLayoutHandle(tableHandle, columnConstraints);
    PrometheusSplitManager splitManager = new PrometheusSplitManager(client, fixedClockAt(now), config);
    ConnectorSplitSource splitsMaybe = splitManager.getSplits(null, null, tableLayoutHandle, null);
    List<ConnectorSplit> splits = splitsMaybe.getNextBatch(NOT_PARTITIONED, NUMBER_MORE_THAN_EXPECTED_NUMBER_SPLITS).getNow(null).getSplits();
    int lastSplitIndex = splits.size() - 1;
    PrometheusSplit lastSplit = (PrometheusSplit) splits.get(lastSplitIndex);
    String queryInSplit = lastSplit.getUri().getQuery();
    String timeShouldBe = decimalSecondString(now.toEpochMilli());
    URI uriAsFormed = new URI("http://doesnotmatter:9090/api/v1/query?query=up[" + getQueryChunkSizeDurationAsPrometheusCompatibleDurationString(config) + "]" + "&time=" + timeShouldBe);
    assertEquals(queryInSplit, uriAsFormed.getQuery());
}

Example 29 with ConnectorSplitSource

use of com.facebook.presto.spi.ConnectorSplitSource in project presto by prestodb.

the class TestPrometheusSplit method testQueryDividedIntoSplitsShouldHaveCorrectSpacingBetweenTimes.

@Test
public void testQueryDividedIntoSplitsShouldHaveCorrectSpacingBetweenTimes() {
    Instant now = LocalDateTime.of(2019, 10, 2, 7, 26, 56, 0).toInstant(UTC);
    PrometheusConnectorConfig config = getCommonConfig(prometheusHttpServer.resolve("/prometheus-data/prometheus-metrics.json"));
    PrometheusClient client = new PrometheusClient(config, METRIC_CODEC, TYPE_MANAGER);
    PrometheusTable table = client.getTable("default", "up");
    PrometheusTableHandle tableHandle = new PrometheusTableHandle("default", table.getName());
    TupleDomain<ColumnHandle> columnConstraints = TupleDomain.withColumnDomains(ImmutableMap.of(new PrometheusColumnHandle("value", BIGINT, 1), Domain.all(VARCHAR), new PrometheusColumnHandle("text", createUnboundedVarcharType(), 0), Domain.all(VARCHAR)));
    PrometheusTableLayoutHandle tableLayoutHandle = new PrometheusTableLayoutHandle(tableHandle, columnConstraints);
    PrometheusSplitManager splitManager = new PrometheusSplitManager(client, fixedClockAt(now), config);
    ConnectorSplitSource splits = splitManager.getSplits(null, null, tableLayoutHandle, null);
    PrometheusSplit split1 = (PrometheusSplit) splits.getNextBatch(NOT_PARTITIONED, 1).getNow(null).getSplits().get(0);
    Map<String, String> paramsMap1 = new HashMap<>();
    String[] splitKV1 = split1.getUri().getQuery().split("&");
    paramsMap1.put("query", splitKV1[0].split("=")[1]);
    paramsMap1.put("time", splitKV1[1].split("=")[1]);
    assertEquals(paramsMap1.get("query"), "up[1d]");
    PrometheusSplit split2 = (PrometheusSplit) splits.getNextBatch(NOT_PARTITIONED, 1).getNow(null).getSplits().get(0);
    Map<String, String> paramsMap2 = new HashMap<>();
    String[] splitKV2 = split2.getUri().getQuery().split("&");
    paramsMap2.put("query", splitKV2[0].split("=")[1]);
    paramsMap2.put("time", splitKV2[1].split("=")[1]);
    assertEquals(paramsMap2.get("query"), "up[1d]");
    long diff = Double.valueOf(paramsMap2.get("time")).longValue() - Double.valueOf(paramsMap1.get("time")).longValue();
    assertEquals(config.getQueryChunkSizeDuration().getValue(TimeUnit.SECONDS), diff, 0.0001);
}

Example 30 with ConnectorSplitSource

use of com.facebook.presto.spi.ConnectorSplitSource in project presto by prestodb.

the class TestRaptorSplitManager method testAssignRandomNodeWhenBackupAvailable.

@Test
public void testAssignRandomNodeWhenBackupAvailable() throws URISyntaxException {
    TestingNodeManager nodeManager = new TestingNodeManager();
    RaptorConnectorId connectorId = new RaptorConnectorId("raptor");
    NodeSupplier nodeSupplier = nodeManager::getWorkerNodes;
    InternalNode node = new InternalNode(UUID.randomUUID().toString(), new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(node);
    RaptorSplitManager raptorSplitManagerWithBackup = new RaptorSplitManager(connectorId, nodeSupplier, shardManager, true);
    deleteShardNodes();
    ConnectorTableLayoutResult layout = getOnlyElement(metadata.getTableLayouts(SESSION, tableHandle, Constraint.alwaysTrue(), Optional.empty()));
    ConnectorSplitSource partitionSplit = getSplits(raptorSplitManagerWithBackup, layout);
    List<ConnectorSplit> batch = getSplits(partitionSplit, 1);
    assertEquals(getOnlyElement(((RaptorSplit) getOnlyElement(batch)).getAddresses()), node.getHostAndPort());
}