Search in sources :

Example 1 with TaskSource

use of io.prestosql.execution.TaskSource in project hetu-core by openlookeng.

the class TestSystemMemoryBlocking method testTableScanSystemMemoryBlocking.

@Test
public void testTableScanSystemMemoryBlocking() {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.of(VARCHAR);
    TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(1, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).build()), TEST_TABLE_HANDLE, ImmutableList.of(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), types, false, Optional.empty(), 0, 0);
    PageConsumerOperator sink = createSinkOperator(types);
    Driver driver = Driver.createDriver(driverContext, source, sink);
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    Split testSplit = new Split(new CatalogName("test"), new TestSplit(), Lifespan.taskWide());
    driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, testSplit)), true));
    ListenableFuture<?> blocked = driver.processFor(new Duration(1, NANOSECONDS));
    // the driver shouldn't block in the first call as it will be able to move a page between source and the sink operator
    // but the operator should be blocked
    assertTrue(blocked.isDone());
    assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
    // and they should stay blocked until more memory becomes available
    for (int i = 0; i < 10; i++) {
        blocked = driver.processFor(new Duration(1, NANOSECONDS));
        assertFalse(blocked.isDone());
        assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
    }
    // free up some memory
    memoryPool.free(QUERY_ID, "test", memoryPool.getReservedBytes());
    // the operator should be unblocked
    assertTrue(source.getOperatorContext().isWaitingForMemory().isDone());
    // the driver shouldn't be blocked
    blocked = driver.processFor(new Duration(1, NANOSECONDS));
    assertTrue(blocked.isDone());
}
Also used : TableScanOperator(io.prestosql.operator.TableScanOperator) ScheduledSplit(io.prestosql.execution.ScheduledSplit) Driver(io.prestosql.operator.Driver) Duration(io.airlift.units.Duration) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) PageConsumerOperator(io.prestosql.testing.PageConsumerOperator) Type(io.prestosql.spi.type.Type) CatalogName(io.prestosql.spi.connector.CatalogName) UUID(java.util.UUID) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit) Split(io.prestosql.metadata.Split) ScheduledSplit(io.prestosql.execution.ScheduledSplit) TaskSource(io.prestosql.execution.TaskSource) Test(org.testng.annotations.Test)

Example 2 with TaskSource

use of io.prestosql.execution.TaskSource in project hetu-core by openlookeng.

the class HttpRemoteTask method processTaskUpdate.

private synchronized void processTaskUpdate(TaskInfo newValue, List<TaskSource> sources) {
    updateTaskInfo(newValue);
    // remove acknowledged splits, which frees memory
    for (TaskSource source : sources) {
        PlanNodeId planNodeId = source.getPlanNodeId();
        int removed = 0;
        for (ScheduledSplit split : source.getSplits()) {
            if (pendingSplits.remove(planNodeId, split)) {
                removed++;
            }
        }
        if (source.isNoMoreSplits()) {
            noMoreSplits.put(planNodeId, false);
        }
        for (Lifespan lifespan : source.getNoMoreSplitsForLifespan()) {
            pendingNoMoreSplitsForLifespan.remove(planNodeId, lifespan);
        }
        if (planFragment.isPartitionedSources(planNodeId)) {
            pendingSourceSplitCount -= removed;
        }
    }
    updateSplitQueueSpace();
    partitionedSplitCountTracker.setPartitionedSplitCount(getPartitionedSplitCount());
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) ScheduledSplit(io.prestosql.execution.ScheduledSplit) Lifespan(io.prestosql.execution.Lifespan) TaskSource(io.prestosql.execution.TaskSource)

Example 3 with TaskSource

use of io.prestosql.execution.TaskSource in project hetu-core by openlookeng.

the class HttpRemoteTask method sendUpdate.

private synchronized void sendUpdate() {
    if (abandoned.get()) {
        // Snapshot: Corresponding task has been canceled to resume. Stop any communication with it.
        return;
    }
    TaskStatus taskStatus = getTaskStatus();
    // don't update if the task hasn't been started yet or if it is already finished
    if (!needsUpdate.get() || taskStatus.getState().isDone()) {
        return;
    }
    // if there is a request already running, wait for it to complete
    if (this.currentRequest != null && !this.currentRequest.isDone()) {
        return;
    }
    // if throttled due to error, asynchronously wait for timeout and try again
    ListenableFuture<?> errorRateLimit = updateErrorTracker.acquireRequestPermit();
    if (!errorRateLimit.isDone()) {
        errorRateLimit.addListener(this::sendUpdate, executor);
        return;
    }
    List<TaskSource> sources = getSources();
    Optional<PlanFragment> fragment = sendPlan.get() ? Optional.of(planFragment) : Optional.empty();
    TaskUpdateRequest updateRequest = new TaskUpdateRequest(// so receiver can verify if the instance id matches
    instanceId, session.toSessionRepresentation(), session.getIdentity().getExtraCredentials(), fragment, sources, outputBuffers.get(), totalPartitions, parent);
    byte[] taskUpdateRequestJson = taskUpdateRequestCodec.toBytes(updateRequest);
    if (fragment.isPresent()) {
        stats.updateWithPlanBytes(taskUpdateRequestJson.length);
    }
    HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus);
    Request request = setContentTypeHeaders(isBinaryEncoding, preparePost()).setUri(uriBuilder.build()).setBodyGenerator(StaticBodyGenerator.createStaticBodyGenerator(taskUpdateRequestJson)).build();
    ResponseHandler responseHandler;
    if (isBinaryEncoding) {
        responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
    } else {
        responseHandler = createAdaptingJsonResponseHandler(unwrapJsonCodec(taskInfoCodec));
    }
    updateErrorTracker.startRequest();
    ListenableFuture<BaseResponse<TaskInfo>> future = httpClient.executeAsync(request, responseHandler);
    currentRequest = future;
    currentRequestStartNanos = System.nanoTime();
    // The needsUpdate flag needs to be set to false BEFORE adding the Future callback since callback might change the flag value
    // and does so without grabbing the instance lock.
    needsUpdate.set(false);
    Futures.addCallback(future, new SimpleHttpResponseHandler<>(new UpdateResponseHandler(sources), request.getUri(), stats), executor);
}
Also used : SmileCodec(io.prestosql.protocol.SmileCodec) AdaptingJsonResponseHandler.createAdaptingJsonResponseHandler(io.prestosql.protocol.AdaptingJsonResponseHandler.createAdaptingJsonResponseHandler) ResponseHandler(io.airlift.http.client.ResponseHandler) FullSmileResponseHandler.createFullSmileResponseHandler(io.prestosql.protocol.FullSmileResponseHandler.createFullSmileResponseHandler) TaskUpdateRequest(io.prestosql.server.TaskUpdateRequest) Request(io.airlift.http.client.Request) TaskUpdateRequest(io.prestosql.server.TaskUpdateRequest) TaskStatus(io.prestosql.execution.TaskStatus) PlanFragment(io.prestosql.sql.planner.PlanFragment) BaseResponse(io.prestosql.protocol.BaseResponse) HttpUriBuilder(io.airlift.http.client.HttpUriBuilder) TaskSource(io.prestosql.execution.TaskSource)

Example 4 with TaskSource

use of io.prestosql.execution.TaskSource in project hetu-core by openlookeng.

the class HttpRemoteTask method getSource.

private synchronized TaskSource getSource(PlanNodeId planNodeId) {
    Set<ScheduledSplit> splits = ImmutableSet.copyOf(pendingSplits.get(planNodeId));
    boolean pendingNoMoreSplits = Boolean.TRUE.equals(this.noMoreSplits.get(planNodeId));
    boolean tmpNoMoreSplits = this.noMoreSplits.containsKey(planNodeId);
    Set<Lifespan> noMoreSplitsForLifespan = pendingNoMoreSplitsForLifespan.get(planNodeId);
    TaskSource element = null;
    if (!splits.isEmpty() || !noMoreSplitsForLifespan.isEmpty() || pendingNoMoreSplits) {
        element = new TaskSource(planNodeId, splits, noMoreSplitsForLifespan, tmpNoMoreSplits);
    }
    return element;
}
Also used : ScheduledSplit(io.prestosql.execution.ScheduledSplit) Lifespan(io.prestosql.execution.Lifespan) TaskSource(io.prestosql.execution.TaskSource)

Example 5 with TaskSource

use of io.prestosql.execution.TaskSource in project hetu-core by openlookeng.

the class TestDriver method testBrokenOperatorAddSource.

@Test
public void testBrokenOperatorAddSource() throws Exception {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    // create a table scan operator that does not block, which will cause the driver loop to busy wait
    TableScanOperator source = new NotBlockedTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of());
    BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"));
    final Driver driver = Driver.createDriver(driverContext, source, brokenOperator);
    // block thread in operator processing
    Future<Boolean> driverProcessFor = executor.submit(new Callable<Boolean>() {

        @Override
        public Boolean call() {
            return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
        }
    });
    brokenOperator.waitForLocked();
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.close();
    assertTrue(driver.isFinished());
    try {
        driverProcessFor.get(1, TimeUnit.SECONDS);
        fail("Expected InterruptedException");
    } catch (ExecutionException e) {
        assertDriverInterrupted(e.getCause());
    }
}
Also used : ScheduledSplit(io.prestosql.execution.ScheduledSplit) Duration(io.airlift.units.Duration) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Type(io.prestosql.spi.type.Type) ExecutionException(java.util.concurrent.ExecutionException) TaskSource(io.prestosql.execution.TaskSource) Test(org.testng.annotations.Test)

Aggregations

TaskSource (io.prestosql.execution.TaskSource)9 ScheduledSplit (io.prestosql.execution.ScheduledSplit)8 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)6 Split (io.prestosql.metadata.Split)4 Duration (io.airlift.units.Duration)3 Driver (io.prestosql.operator.Driver)3 FixedPageSource (io.prestosql.spi.connector.FixedPageSource)3 Type (io.prestosql.spi.type.Type)3 Test (org.testng.annotations.Test)3 Lifespan (io.prestosql.execution.Lifespan)2 DriverFactory (io.prestosql.operator.DriverFactory)2 PageConsumerOperator (io.prestosql.testing.PageConsumerOperator)2 UUID (java.util.UUID)2 VerifyException (com.google.common.base.VerifyException)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 HttpUriBuilder (io.airlift.http.client.HttpUriBuilder)1 Request (io.airlift.http.client.Request)1 ResponseHandler (io.airlift.http.client.ResponseHandler)1 NodeInfo (io.airlift.node.NodeInfo)1 DynamicFilterCacheManager (io.prestosql.dynamicfilter.DynamicFilterCacheManager)1