use of com.facebook.presto.client.NodeVersion in project presto by prestodb.
the class TestHttpRemoteTask method testRemoteTaskMismatch.
// This timeout should never be reached because a daemon thread in test should fail the test and do proper cleanup.
@Test(timeOut = 30000)
public void testRemoteTaskMismatch() throws InterruptedException, ExecutionException {
Duration idleTimeout = new Duration(3, SECONDS);
Duration failTimeout = new Duration(20, SECONDS);
JsonCodec<TaskStatus> taskStatusCodec = JsonCodec.jsonCodec(TaskStatus.class);
JsonCodec<TaskInfo> taskInfoCodec = JsonCodec.jsonCodec(TaskInfo.class);
TaskManagerConfig taskManagerConfig = new TaskManagerConfig();
// Shorten status refresh wait and info update interval so that we can have a shorter test timeout
taskManagerConfig.setStatusRefreshMaxWait(new Duration(idleTimeout.roundTo(MILLISECONDS) / 100, MILLISECONDS));
taskManagerConfig.setInfoUpdateInterval(new Duration(idleTimeout.roundTo(MILLISECONDS) / 10, MILLISECONDS));
AtomicLong lastActivityNanos = new AtomicLong(System.nanoTime());
HttpProcessor httpProcessor = new HttpProcessor(taskStatusCodec, taskInfoCodec, lastActivityNanos);
TestingHttpClient testingHttpClient = new TestingHttpClient(httpProcessor);
HttpRemoteTaskFactory httpRemoteTaskFactory = new HttpRemoteTaskFactory(new QueryManagerConfig(), taskManagerConfig, testingHttpClient, new TestSqlTaskManager.MockLocationFactory(), taskStatusCodec, taskInfoCodec, JsonCodec.jsonCodec(TaskUpdateRequest.class), new RemoteTaskStats());
RemoteTask remoteTask = httpRemoteTaskFactory.createRemoteTask(TEST_SESSION, new TaskId("test", 1, 2), new PrestoNode("node-id", URI.create("http://192.0.1.2"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), createInitialEmptyOutputBuffers(OutputBuffers.BufferType.BROADCAST), new NodeTaskMap.PartitionedSplitCountTracker(i -> {
}), true);
httpProcessor.setInitialTaskInfo(remoteTask.getTaskInfo());
remoteTask.start();
CompletableFuture<Void> testComplete = new CompletableFuture<>();
asyncRun(idleTimeout.roundTo(MILLISECONDS), failTimeout.roundTo(MILLISECONDS), lastActivityNanos, () -> testComplete.complete(null), (message, cause) -> testComplete.completeExceptionally(new AssertionError(message, cause)));
testComplete.get();
httpRemoteTaskFactory.stop();
assertTrue(remoteTask.getTaskStatus().getState().isDone(), format("TaskStatus is not in a done state: %s", remoteTask.getTaskStatus()));
assertEquals(getOnlyElement(remoteTask.getTaskStatus().getFailures()).getErrorCode(), REMOTE_TASK_MISMATCH.toErrorCode());
assertTrue(remoteTask.getTaskInfo().getTaskStatus().getState().isDone(), format("TaskInfo is not in a done state: %s", remoteTask.getTaskInfo()));
}
use of com.facebook.presto.client.NodeVersion in project presto by prestodb.
the class TestClusterSizeMonitor method addWorker.
private void addWorker(InMemoryNodeManager nodeManager) {
String identifier = "worker/" + numWorkers.incrementAndGet();
nodeManager.addNode(CONNECTOR_ID, new InternalNode(identifier, URI.create("localhost/" + identifier), new NodeVersion("1"), false));
}
use of com.facebook.presto.client.NodeVersion in project presto by prestodb.
the class DiscoveryNodeManager method findCurrentNode.
private static InternalNode findCurrentNode(List<ServiceDescriptor> allServices, String currentNodeId, NodeVersion expectedNodeVersion, boolean httpsRequired) {
for (ServiceDescriptor service : allServices) {
URI uri = getHttpUri(service, httpsRequired);
OptionalInt thriftPort = getThriftServerPort(service);
NodeVersion nodeVersion = getNodeVersion(service);
if (uri != null && nodeVersion != null) {
InternalNode node = new InternalNode(service.getNodeId(), uri, thriftPort, nodeVersion, isCoordinator(service), isResourceManager(service), ALIVE);
if (node.getNodeIdentifier().equals(currentNodeId)) {
checkState(node.getNodeVersion().equals(expectedNodeVersion), "INVARIANT: current node version (%s) should be equal to %s", node.getNodeVersion(), expectedNodeVersion);
return node;
}
}
}
throw new IllegalStateException("INVARIANT: current node not returned from service selector");
}
use of com.facebook.presto.client.NodeVersion in project presto by prestodb.
the class LowMemoryKillerTestingUtils method toNodeMemoryInfoList.
static List<MemoryInfo> toNodeMemoryInfoList(long maxReservedPoolBytes, long maxGeneralPoolBytes, String reservedQuery, Map<String, Map<String, Long>> queries) {
Map<InternalNode, NodeReservation> nodeReservations = new HashMap<>();
for (Map.Entry<String, Map<String, Long>> entry : queries.entrySet()) {
QueryId queryId = new QueryId(entry.getKey());
Map<String, Long> reservationByNode = entry.getValue();
for (Map.Entry<String, Long> nodeEntry : reservationByNode.entrySet()) {
InternalNode node = new InternalNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false);
long bytes = nodeEntry.getValue();
if (bytes == 0) {
continue;
}
if (reservedQuery.equals(entry.getKey())) {
nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getReserved().add(queryId, bytes);
} else {
nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getGeneral().add(queryId, bytes);
}
}
}
ImmutableList.Builder<MemoryInfo> result = ImmutableList.builder();
for (Map.Entry<InternalNode, NodeReservation> entry : nodeReservations.entrySet()) {
NodeReservation nodeReservation = entry.getValue();
ImmutableMap.Builder<MemoryPoolId, MemoryPoolInfo> pools = ImmutableMap.builder();
if (nodeReservation.getGeneral().getTotalReservedBytes() > 0) {
pools.put(GENERAL_POOL, new MemoryPoolInfo(maxGeneralPoolBytes, nodeReservation.getGeneral().getTotalReservedBytes(), 0, nodeReservation.getGeneral().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
}
if (nodeReservation.getReserved().getTotalReservedBytes() > 0) {
pools.put(RESERVED_POOL, new MemoryPoolInfo(maxReservedPoolBytes, nodeReservation.getReserved().getTotalReservedBytes(), 0, nodeReservation.getReserved().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
}
result.add(new MemoryInfo(new DataSize(maxReservedPoolBytes + maxGeneralPoolBytes, BYTE), pools.build()));
}
return result.build();
}
use of com.facebook.presto.client.NodeVersion in project urban-eureka by errir503.
the class LowMemoryKillerTestingUtils method toNodeMemoryInfoList.
static List<MemoryInfo> toNodeMemoryInfoList(long maxReservedPoolBytes, long maxGeneralPoolBytes, String reservedQuery, Map<String, Map<String, Long>> queries) {
Map<InternalNode, NodeReservation> nodeReservations = new HashMap<>();
for (Map.Entry<String, Map<String, Long>> entry : queries.entrySet()) {
QueryId queryId = new QueryId(entry.getKey());
Map<String, Long> reservationByNode = entry.getValue();
for (Map.Entry<String, Long> nodeEntry : reservationByNode.entrySet()) {
InternalNode node = new InternalNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false);
long bytes = nodeEntry.getValue();
if (bytes == 0) {
continue;
}
if (reservedQuery.equals(entry.getKey())) {
nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getReserved().add(queryId, bytes);
} else {
nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getGeneral().add(queryId, bytes);
}
}
}
ImmutableList.Builder<MemoryInfo> result = ImmutableList.builder();
for (Map.Entry<InternalNode, NodeReservation> entry : nodeReservations.entrySet()) {
NodeReservation nodeReservation = entry.getValue();
ImmutableMap.Builder<MemoryPoolId, MemoryPoolInfo> pools = ImmutableMap.builder();
if (nodeReservation.getGeneral().getTotalReservedBytes() > 0) {
pools.put(GENERAL_POOL, new MemoryPoolInfo(maxGeneralPoolBytes, nodeReservation.getGeneral().getTotalReservedBytes(), 0, nodeReservation.getGeneral().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
}
if (nodeReservation.getReserved().getTotalReservedBytes() > 0) {
pools.put(RESERVED_POOL, new MemoryPoolInfo(maxReservedPoolBytes, nodeReservation.getReserved().getTotalReservedBytes(), 0, nodeReservation.getReserved().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
}
result.add(new MemoryInfo(new DataSize(maxReservedPoolBytes + maxGeneralPoolBytes, BYTE), pools.build()));
}
return result.build();
}
Aggregations