Examples with ChangeRequestsSnapshot org.apache.druid.server.coordination.ChangeRequestsSnapshot used on opensource projects

Example 1 with ChangeRequestsSnapshot

use of org.apache.druid.server.coordination.ChangeRequestsSnapshot in project druid by druid-io.

the class SegmentListerResource method getSegments.

/**
 * This endpoint is used by HttpServerInventoryView to keep an up-to-date list of segments served by
 * historical/realtime nodes.
 *
 * This endpoint lists segments served by this server and can also incrementally provide the segments added/dropped
 * since last response.
 *
 * Here is how, this is used.
 *
 * (1) Client sends first request /druid/internal/v1/segments?counter=-1&timeout=<timeout>
 * Server responds with list of segments currently served and a <counter,hash> pair.
 *
 * (2) Client sends subsequent requests /druid/internal/v1/segments?counter=<counter>&hash=<hash>&timeout=<timeout>
 * Where <counter,hash> values are used from the last response. Server responds with list of segment updates
 * since given counter.
 *
 * This endpoint makes the client wait till either there is some segment update or given timeout elapses.
 *
 * So, clients keep on sending next request immediately after receiving the response in order to keep the list
 * of segments served by this server up-to-date.
 *
 * @param counter counter received in last response.
 * @param hash hash received in last response.
 * @param timeout after which response is sent even if there are no new segment updates.
 * @param req
 * @return null to avoid "MUST return a non-void type" warning.
 * @throws IOException
 */
@GET
@Produces({ MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE })
@Consumes({ MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE })
public Void getSegments(@QueryParam("counter") long counter, @QueryParam("hash") long hash, @QueryParam("timeout") long timeout, @Context final HttpServletRequest req) throws IOException {
    if (announcer == null) {
        sendErrorResponse(req, HttpServletResponse.SC_NOT_FOUND, "announcer is not available.");
        return null;
    }
    if (timeout <= 0) {
        sendErrorResponse(req, HttpServletResponse.SC_BAD_REQUEST, "timeout must be positive.");
        return null;
    }
    final ResponseContext context = createContext(req.getHeader("Accept"));
    final ListenableFuture<ChangeRequestsSnapshot<DataSegmentChangeRequest>> future = announcer.getSegmentChangesSince(new ChangeRequestHistory.Counter(counter, hash));
    final AsyncContext asyncContext = req.startAsync();
    asyncContext.addListener(new AsyncListener() {

        @Override
        public void onComplete(AsyncEvent event) {
        }

        @Override
        public void onTimeout(AsyncEvent event) {
            // HTTP 204 NO_CONTENT is sent to the client.
            future.cancel(true);
            event.getAsyncContext().complete();
        }

        @Override
        public void onError(AsyncEvent event) {
        }

        @Override
        public void onStartAsync(AsyncEvent event) {
        }
    });
    Futures.addCallback(future, new FutureCallback<ChangeRequestsSnapshot<DataSegmentChangeRequest>>() {

        @Override
        public void onSuccess(ChangeRequestsSnapshot<DataSegmentChangeRequest> result) {
            try {
                HttpServletResponse response = (HttpServletResponse) asyncContext.getResponse();
                response.setStatus(HttpServletResponse.SC_OK);
                context.inputMapper.writerWithType(HttpServerInventoryView.SEGMENT_LIST_RESP_TYPE_REF).writeValue(asyncContext.getResponse().getOutputStream(), result);
                asyncContext.complete();
            } catch (Exception ex) {
                log.debug(ex, "Request timed out or closed already.");
            }
        }

        @Override
        public void onFailure(Throwable th) {
            try {
                HttpServletResponse response = (HttpServletResponse) asyncContext.getResponse();
                if (th instanceof IllegalArgumentException) {
                    response.sendError(HttpServletResponse.SC_BAD_REQUEST, th.getMessage());
                } else {
                    response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, th.getMessage());
                }
                asyncContext.complete();
            } catch (Exception ex) {
                log.debug(ex, "Request timed out or closed already.");
            }
        }
    });
    asyncContext.setTimeout(timeout);
    return null;
}

Example 2 with ChangeRequestsSnapshot

use of org.apache.druid.server.coordination.ChangeRequestsSnapshot in project druid by druid-io.

the class WorkerTaskManager method getChangesSince.

public ListenableFuture<ChangeRequestsSnapshot<WorkerHistoryItem>> getChangesSince(ChangeRequestHistory.Counter counter) {
    Preconditions.checkState(lifecycleLock.awaitStarted(1, TimeUnit.SECONDS), "not started");
    if (counter.getCounter() < 0) {
        synchronized (lock) {
            List<WorkerHistoryItem> items = new ArrayList<>();
            items.add(new WorkerHistoryItem.Metadata(disabled.get()));
            for (Task task : assignedTasks.values()) {
                items.add(new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.unknown())));
            }
            for (TaskDetails details : runningTasks.values()) {
                items.add(new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(details.task, details.status, details.location)));
            }
            for (TaskAnnouncement taskAnnouncement : completedTasks.values()) {
                items.add(new WorkerHistoryItem.TaskUpdate(taskAnnouncement));
            }
            SettableFuture<ChangeRequestsSnapshot<WorkerHistoryItem>> future = SettableFuture.create();
            future.set(ChangeRequestsSnapshot.success(changeHistory.getLastCounter(), Lists.newArrayList(items)));
            return future;
        }
    } else {
        return changeHistory.getRequestsSince(counter);
    }
}

Example 3 with ChangeRequestsSnapshot

use of org.apache.druid.server.coordination.ChangeRequestsSnapshot in project druid by druid-io.

the class TaskManagementResource method getWorkerState.

/**
 * This endpoint is used by HttpRemoteTaskRunner to keep an up-to-date state of the worker wrt to the tasks it is
 * running, completed etc and other metadata such as its enabled/disabled status.
 *
 * Here is how, this is used.
 *
 * (1) Client sends first request /druid/internal/v1/worker?counter=-1&timeout=<timeout>
 * Server responds with current list of running/completed tasks and metadata. And, a <counter,hash> pair.
 *
 * (2) Client sends subsequent requests /druid/internal/v1/worker?counter=<counter>&hash=<hash>&timeout=<timeout>
 * Where <counter,hash> values are used from the last response. Server responds with changes since then.
 *
 * This endpoint makes the client wait till either there is some update or given timeout elapses.
 *
 * So, clients keep on sending next request immediately after receiving the response in order to keep the state of
 * this server up-to-date.
 *
 * @param counter counter received in last response.
 * @param hash hash received in last response.
 * @param timeout after which response is sent even if there are no new segment updates.
 * @param req
 * @return null to avoid "MUST return a non-void type" warning.
 * @throws IOException
 */
@GET
@Produces({ MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE })
public Void getWorkerState(@QueryParam("counter") long counter, @QueryParam("hash") long hash, @QueryParam("timeout") long timeout, @Context final HttpServletRequest req) throws IOException {
    if (timeout <= 0) {
        sendErrorResponse(req, HttpServletResponse.SC_BAD_REQUEST, "timeout must be positive.");
        return null;
    }
    final ResponseContext context = createContext(req.getHeader("Accept"));
    final ListenableFuture<ChangeRequestsSnapshot<WorkerHistoryItem>> future = workerTaskManager.getChangesSince(new ChangeRequestHistory.Counter(counter, hash));
    final AsyncContext asyncContext = req.startAsync();
    asyncContext.addListener(new AsyncListener() {

        @Override
        public void onComplete(AsyncEvent event) {
        }

        @Override
        public void onTimeout(AsyncEvent event) {
            // HTTP 204 NO_CONTENT is sent to the client.
            future.cancel(true);
            event.getAsyncContext().complete();
        }

        @Override
        public void onError(AsyncEvent event) {
        }

        @Override
        public void onStartAsync(AsyncEvent event) {
        }
    });
    Futures.addCallback(future, new FutureCallback<ChangeRequestsSnapshot>() {

        @Override
        public void onSuccess(ChangeRequestsSnapshot result) {
            try {
                HttpServletResponse response = (HttpServletResponse) asyncContext.getResponse();
                response.setStatus(HttpServletResponse.SC_OK);
                context.inputMapper.writerWithType(WorkerHolder.WORKER_SYNC_RESP_TYPE_REF).writeValue(asyncContext.getResponse().getOutputStream(), result);
                asyncContext.complete();
            } catch (Exception ex) {
                log.debug(ex, "Request timed out or closed already.");
            }
        }

        @Override
        public void onFailure(Throwable th) {
            try {
                HttpServletResponse response = (HttpServletResponse) asyncContext.getResponse();
                if (th instanceof IllegalArgumentException) {
                    response.sendError(HttpServletResponse.SC_BAD_REQUEST, th.getMessage());
                } else {
                    response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, th.getMessage());
                }
                asyncContext.complete();
            } catch (Exception ex) {
                log.debug(ex, "Request timed out or closed already.");
            }
        }
    });
    asyncContext.setTimeout(timeout);
    return null;
}

Example 4 with ChangeRequestsSnapshot

use of org.apache.druid.server.coordination.ChangeRequestsSnapshot in project druid by druid-io.

the class HttpServerInventoryViewTest method testSimple.

@Test(timeout = 60_000L)
public void testSimple() throws Exception {
    ObjectMapper jsonMapper = TestHelper.makeJsonMapper();
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(DataNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
    EasyMock.replay(druidNodeDiscoveryProvider);
    final DataSegment segment1 = new DataSegment("test1", Intervals.of("2014/2015"), "v1", null, null, null, null, 0, 0);
    final DataSegment segment2 = new DataSegment("test2", Intervals.of("2014/2015"), "v1", null, null, null, null, 0, 0);
    final DataSegment segment3 = new DataSegment("test3", Intervals.of("2014/2015"), "v1", null, null, null, null, 0, 0);
    final DataSegment segment4 = new DataSegment("test4", Intervals.of("2014/2015"), "v1", null, null, null, null, 0, 0);
    final DataSegment segment5 = new DataSegment("non-loading-datasource", Intervals.of("2014/2015"), "v1", null, null, null, null, 0, 0);
    TestHttpClient httpClient = new TestHttpClient(ImmutableList.of(Futures.immediateFuture(new ByteArrayInputStream(jsonMapper.writerWithType(HttpServerInventoryView.SEGMENT_LIST_RESP_TYPE_REF).writeValueAsBytes(new ChangeRequestsSnapshot(false, null, ChangeRequestHistory.Counter.ZERO, ImmutableList.of(new SegmentChangeRequestLoad(segment1)))))), Futures.immediateFuture(new ByteArrayInputStream(jsonMapper.writerWithType(HttpServerInventoryView.SEGMENT_LIST_RESP_TYPE_REF).writeValueAsBytes(new ChangeRequestsSnapshot(false, null, ChangeRequestHistory.Counter.ZERO, ImmutableList.of(new SegmentChangeRequestDrop(segment1), new SegmentChangeRequestLoad(segment2), new SegmentChangeRequestLoad(segment3)))))), Futures.immediateFuture(new ByteArrayInputStream(jsonMapper.writerWithType(HttpServerInventoryView.SEGMENT_LIST_RESP_TYPE_REF).writeValueAsBytes(new ChangeRequestsSnapshot(true, "force reset counter", ChangeRequestHistory.Counter.ZERO, ImmutableList.of())))), Futures.immediateFuture(new ByteArrayInputStream(jsonMapper.writerWithType(HttpServerInventoryView.SEGMENT_LIST_RESP_TYPE_REF).writeValueAsBytes(new ChangeRequestsSnapshot(false, null, ChangeRequestHistory.Counter.ZERO, ImmutableList.of(new SegmentChangeRequestLoad(segment3), new SegmentChangeRequestLoad(segment4), new SegmentChangeRequestLoad(segment5))))))));
    DiscoveryDruidNode druidNode = new DiscoveryDruidNode(new DruidNode("service", "host", false, 8080, null, true, false), NodeRole.HISTORICAL, ImmutableMap.of(DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerType.HISTORICAL, 0)));
    HttpServerInventoryView httpServerInventoryView = new HttpServerInventoryView(jsonMapper, httpClient, druidNodeDiscoveryProvider, (pair) -> !pair.rhs.getDataSource().equals("non-loading-datasource"), new HttpServerInventoryViewConfig(null, null, null), "test");
    CountDownLatch initializeCallback1 = new CountDownLatch(1);
    Map<SegmentId, CountDownLatch> segmentAddLathces = ImmutableMap.of(segment1.getId(), new CountDownLatch(1), segment2.getId(), new CountDownLatch(1), segment3.getId(), new CountDownLatch(1), segment4.getId(), new CountDownLatch(1));
    Map<SegmentId, CountDownLatch> segmentDropLatches = ImmutableMap.of(segment1.getId(), new CountDownLatch(1), segment2.getId(), new CountDownLatch(1));
    httpServerInventoryView.registerSegmentCallback(Execs.directExecutor(), new ServerView.SegmentCallback() {

        @Override
        public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) {
            segmentAddLathces.get(segment.getId()).countDown();
            return ServerView.CallbackAction.CONTINUE;
        }

        @Override
        public ServerView.CallbackAction segmentRemoved(DruidServerMetadata server, DataSegment segment) {
            segmentDropLatches.get(segment.getId()).countDown();
            return ServerView.CallbackAction.CONTINUE;
        }

        @Override
        public ServerView.CallbackAction segmentViewInitialized() {
            initializeCallback1.countDown();
            return ServerView.CallbackAction.CONTINUE;
        }
    });
    final CountDownLatch serverRemovedCalled = new CountDownLatch(1);
    httpServerInventoryView.registerServerRemovedCallback(Execs.directExecutor(), new ServerView.ServerRemovedCallback() {

        @Override
        public ServerView.CallbackAction serverRemoved(DruidServer server) {
            if (server.getName().equals("host:8080")) {
                serverRemovedCalled.countDown();
                return ServerView.CallbackAction.CONTINUE;
            } else {
                throw new RE("Unknown server [%s]", server.getName());
            }
        }
    });
    httpServerInventoryView.start();
    druidNodeDiscovery.listener.nodesAdded(ImmutableList.of(druidNode));
    initializeCallback1.await();
    segmentAddLathces.get(segment1.getId()).await();
    segmentDropLatches.get(segment1.getId()).await();
    segmentAddLathces.get(segment2.getId()).await();
    segmentAddLathces.get(segment3.getId()).await();
    segmentAddLathces.get(segment4.getId()).await();
    segmentDropLatches.get(segment2.getId()).await();
    DruidServer druidServer = httpServerInventoryView.getInventoryValue("host:8080");
    Assert.assertEquals(ImmutableMap.of(segment3.getId(), segment3, segment4.getId(), segment4), Maps.uniqueIndex(druidServer.iterateAllSegments(), DataSegment::getId));
    druidNodeDiscovery.listener.nodesRemoved(ImmutableList.of(druidNode));
    serverRemovedCalled.await();
    Assert.assertNull(httpServerInventoryView.getInventoryValue("host:8080"));
    EasyMock.verify(druidNodeDiscoveryProvider);
    httpServerInventoryView.stop();
}