use of com.facebook.presto.execution.TaskId in project presto by prestodb.
the class MemoryLocalQueryRunner method execute.
public void execute(@Language("SQL") String query) {
Session session = testSessionBuilder().setSystemProperty("optimizer.optimize-hash-generation", "true").build();
ExecutorService executor = localQueryRunner.getExecutor();
MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
MemoryPool systemMemoryPool = new MemoryPool(new MemoryPoolId("testSystem"), new DataSize(1, GIGABYTE));
TaskContext taskContext = new QueryContext(new QueryId("test"), new DataSize(256, MEGABYTE), memoryPool, systemMemoryPool, executor).addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), executor), session, false, false);
// Use NullOutputFactory to avoid coping out results to avoid affecting benchmark results
List<Driver> drivers = localQueryRunner.createDrivers(query, new NullOutputOperator.NullOutputFactory(), taskContext);
boolean done = false;
while (!done) {
boolean processed = false;
for (Driver driver : drivers) {
if (!driver.isFinished()) {
driver.process();
processed = true;
}
}
done = !processed;
}
}
use of com.facebook.presto.execution.TaskId in project presto by prestodb.
the class TaskResource method getResults.
@GET
@Path("{taskId}/results/{bufferId}/{token}")
@Produces(PRESTO_PAGES)
public void getResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @PathParam("token") final long token, @HeaderParam(PRESTO_MAX_SIZE) DataSize maxSize, @Suspended AsyncResponse asyncResponse) throws InterruptedException {
requireNonNull(taskId, "taskId is null");
requireNonNull(bufferId, "bufferId is null");
long start = System.nanoTime();
ListenableFuture<BufferResult> bufferResultFuture = taskManager.getTaskResults(taskId, bufferId, token, maxSize);
Duration waitTime = randomizeWaitTime(DEFAULT_MAX_WAIT_TIME);
bufferResultFuture = addTimeout(bufferResultFuture, () -> BufferResult.emptyResults(taskManager.getTaskInstanceId(taskId), token, false), waitTime, timeoutExecutor);
ListenableFuture<Response> responseFuture = Futures.transform(bufferResultFuture, result -> {
List<SerializedPage> serializedPages = result.getSerializedPages();
GenericEntity<?> entity = null;
Status status;
if (serializedPages.isEmpty()) {
status = Status.NO_CONTENT;
} else {
entity = new GenericEntity<>(serializedPages, new TypeToken<List<Page>>() {
}.getType());
status = Status.OK;
}
return Response.status(status).entity(entity).header(PRESTO_TASK_INSTANCE_ID, result.getTaskInstanceId()).header(PRESTO_PAGE_TOKEN, result.getToken()).header(PRESTO_PAGE_NEXT_TOKEN, result.getNextToken()).header(PRESTO_BUFFER_COMPLETE, result.isBufferComplete()).build();
});
// For hard timeout, add an additional 5 seconds to max wait for thread scheduling contention and GC
Duration timeout = new Duration(waitTime.toMillis() + 5000, MILLISECONDS);
bindAsyncResponse(asyncResponse, responseFuture, responseExecutor).withTimeout(timeout, Response.status(Status.NO_CONTENT).header(PRESTO_TASK_INSTANCE_ID, taskManager.getTaskInstanceId(taskId)).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, false).build());
responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor());
asyncResponse.register((CompletionCallback) throwable -> resultsRequestTime.add(Duration.nanosSince(start)));
}
use of com.facebook.presto.execution.TaskId in project presto by prestodb.
the class AbstractOperatorBenchmark method runOnce.
@Override
protected Map<String, Long> runOnce() {
Session session = testSessionBuilder().setSystemProperty("optimizer.optimize-hash-generation", "true").build();
ExecutorService executor = localQueryRunner.getExecutor();
MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
MemoryPool systemMemoryPool = new MemoryPool(new MemoryPoolId("testSystem"), new DataSize(1, GIGABYTE));
TaskContext taskContext = new QueryContext(new QueryId("test"), new DataSize(256, MEGABYTE), memoryPool, systemMemoryPool, executor).addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), executor), session, false, false);
CpuTimer cpuTimer = new CpuTimer();
execute(taskContext);
CpuDuration executionTime = cpuTimer.elapsedTime();
TaskStats taskStats = taskContext.getTaskStats();
long inputRows = taskStats.getRawInputPositions();
long inputBytes = taskStats.getRawInputDataSize().toBytes();
long outputRows = taskStats.getOutputPositions();
long outputBytes = taskStats.getOutputDataSize().toBytes();
double inputMegaBytes = new DataSize(inputBytes, BYTE).getValue(MEGABYTE);
return ImmutableMap.<String, Long>builder().put("elapsed_millis", executionTime.getWall().toMillis()).put("input_rows_per_second", (long) (inputRows / executionTime.getWall().getValue(SECONDS))).put("output_rows_per_second", (long) (outputRows / executionTime.getWall().getValue(SECONDS))).put("input_megabytes", (long) inputMegaBytes).put("input_megabytes_per_second", (long) (inputMegaBytes / executionTime.getWall().getValue(SECONDS))).put("wall_nanos", executionTime.getWall().roundTo(NANOSECONDS)).put("cpu_nanos", executionTime.getCpu().roundTo(NANOSECONDS)).put("user_nanos", executionTime.getUser().roundTo(NANOSECONDS)).put("input_rows", inputRows).put("input_bytes", inputBytes).put("output_rows", outputRows).put("output_bytes", outputBytes).build();
}
use of com.facebook.presto.execution.TaskId in project presto by prestodb.
the class TestHttpRemoteTask method testRemoteTaskMismatch.
// This timeout should never be reached because a daemon thread in test should fail the test and do proper cleanup.
@Test(timeOut = 30000)
public void testRemoteTaskMismatch() throws InterruptedException, ExecutionException {
Duration idleTimeout = new Duration(3, SECONDS);
Duration failTimeout = new Duration(20, SECONDS);
JsonCodec<TaskStatus> taskStatusCodec = JsonCodec.jsonCodec(TaskStatus.class);
JsonCodec<TaskInfo> taskInfoCodec = JsonCodec.jsonCodec(TaskInfo.class);
TaskManagerConfig taskManagerConfig = new TaskManagerConfig();
// Shorten status refresh wait and info update interval so that we can have a shorter test timeout
taskManagerConfig.setStatusRefreshMaxWait(new Duration(idleTimeout.roundTo(MILLISECONDS) / 100, MILLISECONDS));
taskManagerConfig.setInfoUpdateInterval(new Duration(idleTimeout.roundTo(MILLISECONDS) / 10, MILLISECONDS));
AtomicLong lastActivityNanos = new AtomicLong(System.nanoTime());
HttpProcessor httpProcessor = new HttpProcessor(taskStatusCodec, taskInfoCodec, lastActivityNanos);
TestingHttpClient testingHttpClient = new TestingHttpClient(httpProcessor);
HttpRemoteTaskFactory httpRemoteTaskFactory = new HttpRemoteTaskFactory(new QueryManagerConfig(), taskManagerConfig, testingHttpClient, new TestSqlTaskManager.MockLocationFactory(), taskStatusCodec, taskInfoCodec, JsonCodec.jsonCodec(TaskUpdateRequest.class), new RemoteTaskStats());
RemoteTask remoteTask = httpRemoteTaskFactory.createRemoteTask(TEST_SESSION, new TaskId("test", 1, 2), new PrestoNode("node-id", URI.create("http://192.0.1.2"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), createInitialEmptyOutputBuffers(OutputBuffers.BufferType.BROADCAST), new NodeTaskMap.PartitionedSplitCountTracker(i -> {
}), true);
httpProcessor.setInitialTaskInfo(remoteTask.getTaskInfo());
remoteTask.start();
CompletableFuture<Void> testComplete = new CompletableFuture<>();
asyncRun(idleTimeout.roundTo(MILLISECONDS), failTimeout.roundTo(MILLISECONDS), lastActivityNanos, () -> testComplete.complete(null), (message, cause) -> testComplete.completeExceptionally(new AssertionError(message, cause)));
testComplete.get();
httpRemoteTaskFactory.stop();
assertTrue(remoteTask.getTaskStatus().getState().isDone(), format("TaskStatus is not in a done state: %s", remoteTask.getTaskStatus()));
assertEquals(getOnlyElement(remoteTask.getTaskStatus().getFailures()).getErrorCode(), REMOTE_TASK_MISMATCH.toErrorCode());
assertTrue(remoteTask.getTaskInfo().getTaskStatus().getState().isDone(), format("TaskInfo is not in a done state: %s", remoteTask.getTaskInfo()));
}
Aggregations