Examples with ISE org.apache.druid.java.util.common.ISE used on opensource projects

Example 91 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class JobHelperTest method testEvilZip.

@Test
public void testEvilZip() throws IOException {
    final File tmpDir = temporaryFolder.newFolder("testEvilZip");
    final File evilResult = new File("/tmp/evil.txt");
    Files.deleteIfExists(evilResult.toPath());
    File evilZip = new File(tmpDir, "evil.zip");
    Files.deleteIfExists(evilZip.toPath());
    CompressionUtilsTest.makeEvilZip(evilZip);
    try {
        JobHelper.unzipNoGuava(new Path(evilZip.getCanonicalPath()), new Configuration(), tmpDir, new Progressable() {

            @Override
            public void progress() {
            }
        }, RetryPolicies.TRY_ONCE_THEN_FAIL);
    } catch (ISE ise) {
        Assert.assertTrue(ise.getMessage().contains("does not start with outDir"));
        Assert.assertFalse("Zip exploit triggered, /tmp/evil.txt was written.", evilResult.exists());
        return;
    }
    Assert.fail("Exception was not thrown for malicious zip file");
}

Example 92 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class LocalTaskActionClient method submit.

@Override
public <RetType> RetType submit(TaskAction<RetType> taskAction) {
    log.debug("Performing action for task[%s]: %s", task.getId(), taskAction);
    if (auditLogConfig.isEnabled() && taskAction.isAudited()) {
        // Add audit log
        try {
            final long auditLogStartTime = System.currentTimeMillis();
            storage.addAuditLog(task, taskAction);
            emitTimerMetric("task/action/log/time", System.currentTimeMillis() - auditLogStartTime);
        } catch (Exception e) {
            final String actionClass = taskAction.getClass().getName();
            log.makeAlert(e, "Failed to record action in audit log").addData("task", task.getId()).addData("actionClass", actionClass).emit();
            throw new ISE(e, "Failed to record action [%s] in audit log", actionClass);
        }
    }
    final long performStartTime = System.currentTimeMillis();
    final RetType result = taskAction.perform(task, toolbox);
    emitTimerMetric("task/action/run/time", System.currentTimeMillis() - performStartTime);
    return result;
}

Example 93 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class SingleTaskBackgroundRunner method run.

@Override
public ListenableFuture<TaskStatus> run(final Task task) {
    if (runningItem == null) {
        final TaskToolbox toolbox = toolboxFactory.build(task);
        final Object taskPriorityObj = task.getContextValue(TaskThreadPriority.CONTEXT_KEY);
        int taskPriority = 0;
        try {
            taskPriority = taskPriorityObj == null ? 0 : Numbers.parseInt(taskPriorityObj);
        } catch (NumberFormatException e) {
            log.error(e, "Error parsing task priority [%s] for task [%s]", taskPriorityObj, task.getId());
        }
        // Ensure an executor for that priority exists
        executorService = buildExecutorService(taskPriority);
        final ListenableFuture<TaskStatus> statusFuture = executorService.submit(new SingleTaskBackgroundRunnerCallable(task, location, toolbox));
        runningItem = new SingleTaskBackgroundRunnerWorkItem(task, location, statusFuture);
        return statusFuture;
    } else {
        throw new ISE("Already running task[%s]", runningItem.getTask().getId());
    }
}

Example 94 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class TaskLockbox method syncFromStorage.

/**
 * Wipe out our current in-memory state and resync it from our bundled {@link TaskStorage}.
 */
public void syncFromStorage() {
    giant.lock();
    try {
        // Load stuff from taskStorage first. If this fails, we don't want to lose all our locks.
        final Set<String> storedActiveTasks = new HashSet<>();
        final List<Pair<Task, TaskLock>> storedLocks = new ArrayList<>();
        for (final Task task : taskStorage.getActiveTasks()) {
            storedActiveTasks.add(task.getId());
            for (final TaskLock taskLock : taskStorage.getLocks(task.getId())) {
                storedLocks.add(Pair.of(task, taskLock));
            }
        }
        // Sort locks by version, so we add them back in the order they were acquired.
        final Ordering<Pair<Task, TaskLock>> byVersionOrdering = new Ordering<Pair<Task, TaskLock>>() {

            @Override
            public int compare(Pair<Task, TaskLock> left, Pair<Task, TaskLock> right) {
                // The second compare shouldn't be necessary, but, whatever.
                return ComparisonChain.start().compare(left.rhs.getVersion(), right.rhs.getVersion()).compare(left.lhs.getId(), right.lhs.getId()).result();
            }
        };
        running.clear();
        activeTasks.clear();
        activeTasks.addAll(storedActiveTasks);
        // Bookkeeping for a log message at the end
        int taskLockCount = 0;
        for (final Pair<Task, TaskLock> taskAndLock : byVersionOrdering.sortedCopy(storedLocks)) {
            final Task task = Preconditions.checkNotNull(taskAndLock.lhs, "task");
            final TaskLock savedTaskLock = Preconditions.checkNotNull(taskAndLock.rhs, "savedTaskLock");
            if (savedTaskLock.getInterval().toDurationMillis() <= 0) {
                // "Impossible", but you never know what crazy stuff can be restored from storage.
                log.warn("Ignoring lock[%s] with empty interval for task: %s", savedTaskLock, task.getId());
                continue;
            }
            // Create a new taskLock if it doesn't have a proper priority,
            // so that every taskLock in memory has the priority.
            final TaskLock savedTaskLockWithPriority = savedTaskLock.getPriority() == null ? savedTaskLock.withPriority(task.getPriority()) : savedTaskLock;
            final TaskLockPosse taskLockPosse = verifyAndCreateOrFindLockPosse(task, savedTaskLockWithPriority);
            if (taskLockPosse != null) {
                taskLockPosse.addTask(task);
                final TaskLock taskLock = taskLockPosse.getTaskLock();
                if (savedTaskLockWithPriority.getVersion().equals(taskLock.getVersion())) {
                    taskLockCount++;
                    log.info("Reacquired lock[%s] for task: %s", taskLock, task.getId());
                } else {
                    taskLockCount++;
                    log.info("Could not reacquire lock on interval[%s] version[%s] (got version[%s] instead) for task: %s", savedTaskLockWithPriority.getInterval(), savedTaskLockWithPriority.getVersion(), taskLock.getVersion(), task.getId());
                }
            } else {
                throw new ISE("Could not reacquire lock on interval[%s] version[%s] for task: %s", savedTaskLockWithPriority.getInterval(), savedTaskLockWithPriority.getVersion(), task.getId());
            }
        }
        log.info("Synced %,d locks for %,d activeTasks from storage (%,d locks ignored).", taskLockCount, activeTasks.size(), storedLocks.size() - taskLockCount);
    } finally {
        giant.unlock();
    }
}

Example 95 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class TaskLockbox method tryLock.

/**
 * Attempt to acquire a lock for a task, without removing it from the queue. Can safely be called multiple times on
 * the same task until the lock is preempted.
 *
 * @return {@link LockResult} containing a new or an existing lock if succeeded. Otherwise, {@link LockResult} with a
 * {@link LockResult#revoked} flag.
 *
 * @throws IllegalStateException if the task is not a valid active task
 */
public LockResult tryLock(final Task task, final LockRequest request) {
    giant.lock();
    try {
        if (!activeTasks.contains(task.getId())) {
            throw new ISE("Unable to grant lock to inactive Task [%s]", task.getId());
        }
        Preconditions.checkArgument(request.getInterval().toDurationMillis() > 0, "interval empty");
        SegmentIdWithShardSpec newSegmentId = null;
        final LockRequest convertedRequest;
        if (request instanceof LockRequestForNewSegment) {
            final LockRequestForNewSegment lockRequestForNewSegment = (LockRequestForNewSegment) request;
            if (lockRequestForNewSegment.getGranularity() == LockGranularity.SEGMENT) {
                newSegmentId = allocateSegmentId(lockRequestForNewSegment, request.getVersion());
                if (newSegmentId == null) {
                    return LockResult.fail();
                }
                convertedRequest = new SpecificSegmentLockRequest(lockRequestForNewSegment, newSegmentId);
            } else {
                convertedRequest = new TimeChunkLockRequest(lockRequestForNewSegment);
            }
        } else {
            convertedRequest = request;
        }
        final TaskLockPosse posseToUse = createOrFindLockPosse(convertedRequest);
        if (posseToUse != null && !posseToUse.getTaskLock().isRevoked()) {
            if (request instanceof LockRequestForNewSegment) {
                final LockRequestForNewSegment lockRequestForNewSegment = (LockRequestForNewSegment) request;
                if (lockRequestForNewSegment.getGranularity() == LockGranularity.TIME_CHUNK) {
                    if (newSegmentId != null) {
                        throw new ISE("SegmentId must be allocated after getting a timeChunk lock," + " but we already have [%s] before getting the lock?", newSegmentId);
                    }
                    newSegmentId = allocateSegmentId(lockRequestForNewSegment, posseToUse.getTaskLock().getVersion());
                }
            }
            // Add to existing TaskLockPosse, if necessary
            if (posseToUse.addTask(task)) {
                log.info("Added task[%s] to TaskLock[%s]", task.getId(), posseToUse.getTaskLock());
                // Update task storage facility. If it fails, revoke the lock.
                try {
                    taskStorage.addLock(task.getId(), posseToUse.getTaskLock());
                    return LockResult.ok(posseToUse.getTaskLock(), newSegmentId);
                } catch (Exception e) {
                    log.makeAlert("Failed to persist lock in storage").addData("task", task.getId()).addData("dataSource", posseToUse.getTaskLock().getDataSource()).addData("interval", posseToUse.getTaskLock().getInterval()).addData("version", posseToUse.getTaskLock().getVersion()).emit();
                    unlock(task, convertedRequest.getInterval(), posseToUse.getTaskLock().getGranularity() == LockGranularity.SEGMENT ? ((SegmentLock) posseToUse.taskLock).getPartitionId() : null);
                    return LockResult.fail();
                }
            } else {
                log.info("Task[%s] already present in TaskLock[%s]", task.getId(), posseToUse.getTaskLock().getGroupId());
                return LockResult.ok(posseToUse.getTaskLock(), newSegmentId);
            }
        } else {
            final boolean lockRevoked = posseToUse != null && posseToUse.getTaskLock().isRevoked();
            if (lockRevoked) {
                return LockResult.revoked(posseToUse.getTaskLock());
            }
            return LockResult.fail();
        }
    } finally {
        giant.unlock();
    }
}