use of io.druid.java.util.common.Pair in project druid by druid-io.
the class OverlordResource method getCompleteTasks.
@GET
@Path("/completeTasks")
@Produces(MediaType.APPLICATION_JSON)
public Response getCompleteTasks(@Context final HttpServletRequest req) {
final List<TaskStatus> recentlyFinishedTasks;
if (authConfig.isEnabled()) {
// This is an experimental feature, see - https://github.com/druid-io/druid/pull/2424
final Map<Pair<Resource, Action>, Access> resourceAccessMap = new HashMap<>();
final AuthorizationInfo authorizationInfo = (AuthorizationInfo) req.getAttribute(AuthConfig.DRUID_AUTH_TOKEN);
recentlyFinishedTasks = ImmutableList.copyOf(Iterables.filter(taskStorageQueryAdapter.getRecentlyFinishedTaskStatuses(), new Predicate<TaskStatus>() {
@Override
public boolean apply(TaskStatus input) {
final String taskId = input.getId();
final Optional<Task> optionalTask = taskStorageQueryAdapter.getTask(taskId);
if (!optionalTask.isPresent()) {
throw new WebApplicationException(Response.serverError().entity(String.format("No task information found for task with id: [%s]", taskId)).build());
}
Resource resource = new Resource(optionalTask.get().getDataSource(), ResourceType.DATASOURCE);
Action action = Action.READ;
Pair<Resource, Action> key = new Pair<>(resource, action);
if (resourceAccessMap.containsKey(key)) {
return resourceAccessMap.get(key).isAllowed();
} else {
Access access = authorizationInfo.isAuthorized(key.lhs, key.rhs);
resourceAccessMap.put(key, access);
return access.isAllowed();
}
}
}));
} else {
recentlyFinishedTasks = taskStorageQueryAdapter.getRecentlyFinishedTaskStatuses();
}
final List<TaskResponseObject> completeTasks = Lists.transform(recentlyFinishedTasks, new Function<TaskStatus, TaskResponseObject>() {
@Override
public TaskResponseObject apply(TaskStatus taskStatus) {
// Would be nice to include the real created date, but the TaskStorage API doesn't yet allow it.
return new TaskResponseObject(taskStatus.getId(), new DateTime(0), new DateTime(0), Optional.of(taskStatus), TaskLocation.unknown());
}
});
return Response.ok(completeTasks).build();
}
use of io.druid.java.util.common.Pair in project druid by druid-io.
the class OverlordResource method getWaitingTasks.
@GET
@Path("/waitingTasks")
@Produces(MediaType.APPLICATION_JSON)
public Response getWaitingTasks(@Context final HttpServletRequest req) {
return workItemsResponse(new Function<TaskRunner, Collection<? extends TaskRunnerWorkItem>>() {
@Override
public Collection<? extends TaskRunnerWorkItem> apply(TaskRunner taskRunner) {
// A bit roundabout, but works as a way of figuring out what tasks haven't been handed
// off to the runner yet:
final List<Task> allActiveTasks = taskStorageQueryAdapter.getActiveTasks();
final List<Task> activeTasks;
if (authConfig.isEnabled()) {
// This is an experimental feature, see - https://github.com/druid-io/druid/pull/2424
final Map<Pair<Resource, Action>, Access> resourceAccessMap = new HashMap<>();
final AuthorizationInfo authorizationInfo = (AuthorizationInfo) req.getAttribute(AuthConfig.DRUID_AUTH_TOKEN);
activeTasks = ImmutableList.copyOf(Iterables.filter(allActiveTasks, new Predicate<Task>() {
@Override
public boolean apply(Task input) {
Resource resource = new Resource(input.getDataSource(), ResourceType.DATASOURCE);
Action action = Action.READ;
Pair<Resource, Action> key = new Pair<>(resource, action);
if (resourceAccessMap.containsKey(key)) {
return resourceAccessMap.get(key).isAllowed();
} else {
Access access = authorizationInfo.isAuthorized(key.lhs, key.rhs);
resourceAccessMap.put(key, access);
return access.isAllowed();
}
}
}));
} else {
activeTasks = allActiveTasks;
}
final Set<String> runnersKnownTasks = Sets.newHashSet(Iterables.transform(taskRunner.getKnownTasks(), new Function<TaskRunnerWorkItem, String>() {
@Override
public String apply(final TaskRunnerWorkItem workItem) {
return workItem.getTaskId();
}
}));
final List<TaskRunnerWorkItem> waitingTasks = Lists.newArrayList();
for (final Task task : activeTasks) {
if (!runnersKnownTasks.contains(task.getId())) {
waitingTasks.add(// Would be nice to include the real created date, but the TaskStorage API doesn't yet allow it.
new TaskRunnerWorkItem(task.getId(), SettableFuture.<TaskStatus>create(), new DateTime(0), new DateTime(0)) {
@Override
public TaskLocation getLocation() {
return TaskLocation.unknown();
}
});
}
}
return waitingTasks;
}
});
}
use of io.druid.java.util.common.Pair in project druid by druid-io.
the class BaseFilterTest method tearDown.
public static void tearDown(String className) throws Exception {
Map<String, Pair<StorageAdapter, Closeable>> adaptersForClass = adapterCache.get().get(className);
if (adaptersForClass != null) {
for (Map.Entry<String, Pair<StorageAdapter, Closeable>> entry : adaptersForClass.entrySet()) {
Closeable closeable = entry.getValue().rhs;
closeable.close();
}
adapterCache.get().put(className, null);
}
}
use of io.druid.java.util.common.Pair in project druid by druid-io.
the class BaseFilterTest method makeConstructors.
public static Collection<Object[]> makeConstructors() {
final List<Object[]> constructors = Lists.newArrayList();
final Map<String, BitmapSerdeFactory> bitmapSerdeFactories = ImmutableMap.<String, BitmapSerdeFactory>of("concise", new ConciseBitmapSerdeFactory(), "roaring", new RoaringBitmapSerdeFactory(true));
final Map<String, IndexMerger> indexMergers = ImmutableMap.<String, IndexMerger>of("IndexMerger", TestHelper.getTestIndexMerger(), "IndexMergerV9", TestHelper.getTestIndexMergerV9());
final Map<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>> finishers = ImmutableMap.of("incremental", new Function<IndexBuilder, Pair<StorageAdapter, Closeable>>() {
@Override
public Pair<StorageAdapter, Closeable> apply(IndexBuilder input) {
final IncrementalIndex index = input.buildIncrementalIndex();
return Pair.<StorageAdapter, Closeable>of(new IncrementalIndexStorageAdapter(index), new Closeable() {
@Override
public void close() throws IOException {
index.close();
}
});
}
}, "mmapped", new Function<IndexBuilder, Pair<StorageAdapter, Closeable>>() {
@Override
public Pair<StorageAdapter, Closeable> apply(IndexBuilder input) {
final QueryableIndex index = input.buildMMappedIndex();
return Pair.<StorageAdapter, Closeable>of(new QueryableIndexStorageAdapter(index), new Closeable() {
@Override
public void close() throws IOException {
index.close();
}
});
}
}, "mmappedMerged", new Function<IndexBuilder, Pair<StorageAdapter, Closeable>>() {
@Override
public Pair<StorageAdapter, Closeable> apply(IndexBuilder input) {
final QueryableIndex index = input.buildMMappedMergedIndex();
return Pair.<StorageAdapter, Closeable>of(new QueryableIndexStorageAdapter(index), new Closeable() {
@Override
public void close() throws IOException {
index.close();
}
});
}
});
for (Map.Entry<String, BitmapSerdeFactory> bitmapSerdeFactoryEntry : bitmapSerdeFactories.entrySet()) {
for (Map.Entry<String, IndexMerger> indexMergerEntry : indexMergers.entrySet()) {
for (Map.Entry<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>> finisherEntry : finishers.entrySet()) {
for (boolean cnf : ImmutableList.of(false, true)) {
for (boolean optimize : ImmutableList.of(false, true)) {
final String testName = String.format("bitmaps[%s], indexMerger[%s], finisher[%s], optimize[%s]", bitmapSerdeFactoryEntry.getKey(), indexMergerEntry.getKey(), finisherEntry.getKey(), optimize);
final IndexBuilder indexBuilder = IndexBuilder.create().indexSpec(new IndexSpec(bitmapSerdeFactoryEntry.getValue(), null, null, null)).indexMerger(indexMergerEntry.getValue());
constructors.add(new Object[] { testName, indexBuilder, finisherEntry.getValue(), cnf, optimize });
}
}
}
}
}
return constructors;
}
use of io.druid.java.util.common.Pair in project druid by druid-io.
the class ServerManagerTest method testReferenceCountingWhileQueryExecuting.
@Test
public void testReferenceCountingWhileQueryExecuting() throws Exception {
loadQueryable("test", "3", new Interval("2011-04-04/2011-04-05"));
Future future = assertQueryable(Granularities.DAY, "test", new Interval("2011-04-04/2011-04-06"), ImmutableList.<Pair<String, Interval>>of(new Pair<String, Interval>("3", new Interval("2011-04-04/2011-04-05"))));
queryNotifyLatch.await(1000, TimeUnit.MILLISECONDS);
Assert.assertEquals(1, factory.getSegmentReferences().size());
for (ReferenceCountingSegment referenceCountingSegment : factory.getSegmentReferences()) {
Assert.assertEquals(1, referenceCountingSegment.getNumReferences());
}
queryWaitYieldLatch.countDown();
Assert.assertEquals(1, factory.getAdapters().size());
for (SegmentForTesting segmentForTesting : factory.getAdapters()) {
Assert.assertFalse(segmentForTesting.isClosed());
}
dropQueryable("test", "3", new Interval("2011-04-04/2011-04-05"));
for (SegmentForTesting segmentForTesting : factory.getAdapters()) {
Assert.assertFalse(segmentForTesting.isClosed());
}
queryWaitLatch.countDown();
future.get();
for (SegmentForTesting segmentForTesting : factory.getAdapters()) {
Assert.assertTrue(segmentForTesting.isClosed());
}
}
Aggregations