Examples with Ref org.apache.hive.common.util.Ref used on opensource projects

Example 1 with Ref

use of org.apache.hive.common.util.Ref in project hive by apache.

the class LlapTaskSchedulerService method schedulePendingTasks.

@VisibleForTesting
protected void schedulePendingTasks() throws InterruptedException {
    Ref<TaskInfo> downgradedTask = new Ref<>(null);
    writeLock.lock();
    try {
        if (LOG.isDebugEnabled()) {
            LOG.debug("ScheduleRun: {}", constructPendingTaskCountsLogMessage());
        }
        Iterator<Entry<Priority, List<TaskInfo>>> pendingIterator = pendingTasks.entrySet().iterator();
        Resource totalResource = getTotalResources();
        while (pendingIterator.hasNext()) {
            Entry<Priority, List<TaskInfo>> entry = pendingIterator.next();
            List<TaskInfo> taskListAtPriority = entry.getValue();
            Iterator<TaskInfo> taskIter = taskListAtPriority.iterator();
            boolean scheduledAllAtPriority = true;
            while (taskIter.hasNext()) {
                // TODO Optimization: Add a check to see if there's any capacity available. No point in
                // walking through all active nodes, if they don't have potential capacity.
                TaskInfo taskInfo = taskIter.next();
                if (taskInfo.getNumPreviousAssignAttempts() == 1) {
                    dagStats.registerDelayedAllocation();
                }
                taskInfo.triedAssigningTask();
                ScheduleResult scheduleResult = scheduleTask(taskInfo, totalResource, downgradedTask);
                // Note: we must handle downgradedTask after this. We do it at the end, outside the lock.
                if (LOG.isDebugEnabled()) {
                    LOG.debug("ScheduleResult for Task: {} = {}", taskInfo, scheduleResult);
                }
                if (scheduleResult == ScheduleResult.SCHEDULED) {
                    taskIter.remove();
                } else {
                    if (scheduleResult == ScheduleResult.INADEQUATE_TOTAL_RESOURCES) {
                        LOG.info("Inadequate total resources before scheduling pending tasks." + " Signalling scheduler timeout monitor thread to start timer.");
                        startTimeoutMonitor();
                    // TODO Nothing else should be done for this task. Move on.
                    }
                    // Try pre-empting a task so that a higher priority task can take it's place.
                    // Preempt only if there's no pending preemptions to avoid preempting twice for a task.
                    String[] potentialHosts;
                    if (scheduleResult == ScheduleResult.DELAYED_LOCALITY) {
                        // Add the task to the delayed task queue if it does not already exist.
                        maybeAddToDelayedTaskQueue(taskInfo);
                        // Try preempting a lower priority task in any case.
                        // preempt only on specific hosts, if no preemptions already exist on those.
                        potentialHosts = taskInfo.requestedHosts;
                        // Protect against a bad location being requested.
                        if (potentialHosts == null || potentialHosts.length == 0) {
                            potentialHosts = null;
                        }
                    } else {
                        // preempt on any host.
                        potentialHosts = null;
                    }
                    // At this point we're dealing with all return types, except ScheduleResult.SCHEDULED.
                    if (potentialHosts != null) {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Attempting to preempt on requested host for task={}, potentialHosts={}", taskInfo, Arrays.toString(potentialHosts));
                        }
                        // Preempt on specific host
                        boolean shouldPreempt = true;
                        for (String host : potentialHosts) {
                            // Preempt only if there are no pending preemptions on the same host
                            // When the premption registers, the request at the highest priority will be given the slot,
                            // even if the initial preemption was caused by some other task.
                            // TODO Maybe register which task the preemption was for, to avoid a bad non-local allocation.
                            MutableInt pendingHostPreemptions = pendingPreemptionsPerHost.get(host);
                            if (pendingHostPreemptions != null && pendingHostPreemptions.intValue() > 0) {
                                shouldPreempt = false;
                                LOG.debug("Not preempting for task={}. Found an existing preemption request on host={}, pendingPreemptionCount={}", taskInfo.task, host, pendingHostPreemptions.intValue());
                                break;
                            }
                        }
                        if (shouldPreempt) {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Attempting to preempt for {} on potential hosts={}. TotalPendingPreemptions={}", taskInfo.task, Arrays.toString(potentialHosts), pendingPreemptions.get());
                            }
                            preemptTasks(entry.getKey().getPriority(), vertexNum(taskInfo), 1, potentialHosts);
                        } else {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Not preempting for {} on potential hosts={}. An existing preemption request exists", taskInfo.task, Arrays.toString(potentialHosts));
                            }
                        }
                    } else {
                        // Either DELAYED_RESOURCES or DELAYED_LOCALITY with an unknown requested host.
                        // Request for a preemption if there's none pending. If a single preemption is pending,
                        // and this is the next task to be assigned, it will be assigned once that slot becomes available.
                        LOG.debug("Attempting to preempt on any host for task={}, pendingPreemptions={}", taskInfo.task, pendingPreemptions.get());
                        if (pendingPreemptions.get() == 0) {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Attempting to preempt for task={}, priority={} on any available host", taskInfo.task, taskInfo.priority);
                            }
                            preemptTasks(entry.getKey().getPriority(), vertexNum(taskInfo), 1, null);
                        } else {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Skipping preemption since there are {} pending preemption request. For task={}", pendingPreemptions.get(), taskInfo);
                            }
                        }
                    }
                    // Since there was an allocation failure - don't try assigning tasks at the next priority.
                    scheduledAllAtPriority = false;
                    // Don't break if this allocation failure was a result of a LOCALITY_DELAY. Others could still be allocated.
                    if (scheduleResult != ScheduleResult.DELAYED_LOCALITY) {
                        break;
                    }
                }
            // end of else - i.e. could not allocate
            }
            // end of loop over pending tasks
            if (taskListAtPriority.isEmpty()) {
                // Remove the entry, if there's nothing left at the specific priority level
                pendingIterator.remove();
            }
            if (!scheduledAllAtPriority) {
                LOG.debug("Unable to schedule all requests at priority={}. Skipping subsequent priority levels", entry.getKey());
                // Don't attempt scheduling for additional priorities
                break;
            }
        }
    } finally {
        writeLock.unlock();
    }
    if (downgradedTask.value != null) {
        WM_LOG.info("Downgrading " + downgradedTask.value.attemptId);
        checkAndSendGuaranteedStateUpdate(downgradedTask.value);
    }
}

Example 2 with Ref

use of org.apache.hive.common.util.Ref in project hive by apache.

the class TestSparkUtilities method testCreateMoveTaskDoesntCreateCascadeTempDirs.

@Test
public void testCreateMoveTaskDoesntCreateCascadeTempDirs() throws Exception {
    FileSinkOperator fsOp = mock(FileSinkOperator.class);
    ParseContext pctx = mock(ParseContext.class);
    Configuration conf = new Configuration();
    conf.set("_hive.hdfs.session.path", "hdfs:/dummypath");
    conf.set("_hive.local.session.path", "hdfs:/dummypath");
    Context ctx = new Context(conf);
    String executionId = ctx.getExecutionId();
    Context ctxSpy = spy(ctx);
    FileSinkDesc fileSinkDesc = mock(FileSinkDesc.class);
    Path mrPath = new Path("hdfs:/tmp/.staging/" + executionId + "/-mr-10001");
    Path mrPath2 = new Path("hdfs:/tmp/.staging/" + executionId + "/-mr-10002");
    Path extPath = new Path("hdfs:/tmp/.staging/" + executionId + "/-ext-10001");
    Path extPath2 = new Path("hdfs:/tmp/.staging/" + executionId + "/-ext-10002");
    final Ref<Path> expectedPathRef = new Ref<>(mrPath);
    final Ref<Path> testPathRef = new Ref<>(extPath);
    doAnswer(invocationOnMock -> {
        return ctxSpy;
    }).when(pctx).getContext();
    doAnswer(invocationOnMock -> {
        return mrPath2;
    }).when(ctxSpy).getMRTmpPath();
    doAnswer(invocationOnMock -> {
        return extPath2;
    }).when(ctxSpy).getExternalTmpPath(any(Path.class));
    doAnswer(invocationOnMock -> {
        return testPathRef.value;
    }).when(fileSinkDesc).getFinalDirName();
    doAnswer(invocationOnMock -> {
        return null;
    }).when(fileSinkDesc).getLinkedFileSinkDesc();
    doAnswer(invocationOnMock -> {
        return fileSinkDesc;
    }).when(fsOp).getConf();
    doAnswer(invocationOnMock -> {
        assertEquals(expectedPathRef.value, invocationOnMock.getArgument(0, Path.class));
        return null;
    }).when(fileSinkDesc).setDirName(any(Path.class));
    testPathRef.value = mrPath;
    expectedPathRef.value = mrPath2;
    GenSparkUtils.createMoveTask(null, true, fsOp, pctx, null, null, null);
    testPathRef.value = extPath;
    expectedPathRef.value = extPath2;
    GenSparkUtils.createMoveTask(null, true, fsOp, pctx, null, null, null);
}

Example 3 with Ref

use of org.apache.hive.common.util.Ref in project hive by apache.

the class WorkloadManager method handleUpdateErrorOnMasterThread.

private void handleUpdateErrorOnMasterThread(WmTezSession session, int failedEndpointVersion, IdentityHashMap<WmTezSession, GetRequest> toReuse, WmThreadSyncWork syncWork, HashSet<String> poolsToRedistribute) {
    // First, check if the registry has been updated since the error, and skip the error if
    // we have received new, valid registry info (TODO: externally, add a grace period for this?).
    Ref<Integer> endpointVersion = new Ref<>(-1);
    AmPluginInfo info = session.getAmPluginInfo(endpointVersion);
    if (info != null && endpointVersion.value > failedEndpointVersion) {
        LOG.info("Ignoring an update error; endpoint information has been updated to {}", info);
        return;
    }
    GetRequest reuseRequest = toReuse.remove(session);
    if (reuseRequest != null) {
        // This session is bad, so don't allow reuse; just convert it to normal get.
        reuseRequest.sessionToReuse = null;
    }
    // We are assuming the update-error AM is bad and just try to kill it.
    RemoveSessionResult rr = checkAndRemoveSessionFromItsPool(session, poolsToRedistribute, null, true);
    switch(rr) {
        case OK:
        case NOT_FOUND:
            // Regardless whether it was removed successfully or after failing to remove, restart it.
            // Since we just restart this from under the user, mark it so we handle it properly when
            // the user tries to actually use this session and fails, proceeding to return/destroy it.
            session.setIsIrrelevantForWm("Failed to update resource allocation");
            // We assume AM might be bad so we will not try to kill the query here; just scrap the AM.
            // TODO: propagate this error to TezJobMonitor somehow? Without using killQuery
            syncWork.toRestartInUse.add(session);
            break;
        case IGNORE:
            // An update error for some session that was actually already killed by us.
            return;
        default:
            throw new AssertionError("Unknown state " + rr);
    }
}