use of org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor in project tez by apache.
the class TestExtServicesWithLocalMode method test1.
@Test(timeout = 30000)
public void test1() throws Exception {
UserPayload userPayload = TezUtils.createUserPayloadFromConf(confForJobs);
TaskSchedulerDescriptor[] taskSchedulerDescriptors = new TaskSchedulerDescriptor[] { TaskSchedulerDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskSchedulerService.class.getName()).setUserPayload(userPayload) };
ContainerLauncherDescriptor[] containerLauncherDescriptors = new ContainerLauncherDescriptor[] { ContainerLauncherDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceNoOpContainerLauncher.class.getName()).setUserPayload(userPayload) };
TaskCommunicatorDescriptor[] taskCommunicatorDescriptors = new TaskCommunicatorDescriptor[] { TaskCommunicatorDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskCommunicatorImpl.class.getName()).setUserPayload(userPayload) };
ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true, false, taskSchedulerDescriptors, containerLauncherDescriptors, taskCommunicatorDescriptors);
TezConfiguration tezConf = new TezConfiguration(confForJobs);
TezClient tezClient = TezClient.newBuilder("test1", tezConf).setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();
try {
tezClient.start();
Path dataPath1 = new Path(SRC_DATA_DIR, "inPath1");
Path dataPath2 = new Path(SRC_DATA_DIR, "inPath2");
Path expectedResultPath = new Path(SRC_DATA_DIR, "expectedOutputPath");
JoinDataGen dataGen = new JoinDataGen();
String[] dataGenArgs = new String[] { dataPath1.toString(), "1048576", dataPath2.toString(), "524288", expectedResultPath.toString(), "2" };
assertEquals(0, dataGen.run(tezConf, dataGenArgs, tezClient));
Path outputPath = new Path(SRC_DATA_DIR, "outPath");
HashJoinExample joinExample = new HashJoinExample();
String[] args = new String[] { dataPath1.toString(), dataPath2.toString(), "2", outputPath.toString() };
assertEquals(0, joinExample.run(tezConf, args, tezClient));
LOG.info("Completed generating Data - Expected Hash Result and Actual Join Result");
assertEquals(0, tezTestServiceCluster.getNumSubmissions());
// ext can consume from ext.
runJoinValidate(tezClient, "allInExt", 7, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH);
LOG.info("Completed allInExt");
// uber can consume from uber.
runJoinValidate(tezClient, "noneInExt", 0, null, null, null);
LOG.info("Completed noneInExt");
// uber can consume from ext
runJoinValidate(tezClient, "lhsInExt", 2, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, null, null);
LOG.info("Completed lhsInExt");
// ext cannot consume from uber in this mode since there's no shuffle handler working,
// and the local data transfer semantics may not match.
} finally {
tezClient.stop();
}
}
use of org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor in project tez by apache.
the class TestExternalTezServicesErrors method setup.
@BeforeClass
public static void setup() throws Exception {
extServiceTestHelper = new ExternalTezServiceTestHelper(TEST_ROOT_DIR);
UserPayload userPayload = TezUtils.createUserPayloadFromConf(extServiceTestHelper.getConfForJobs());
UserPayload userPayloadThrowError = ErrorPluginConfiguration.toUserPayload(ErrorPluginConfiguration.createThrowErrorConf());
UserPayload userPayloadReportFatalErrorLauncher = ErrorPluginConfiguration.toUserPayload(ErrorPluginConfiguration.createReportFatalErrorConf(SUFFIX_LAUNCHER));
UserPayload userPayloadReportFatalErrorTaskComm = ErrorPluginConfiguration.toUserPayload(ErrorPluginConfiguration.createReportFatalErrorConf(SUFFIX_TASKCOMM));
UserPayload userPayloadReportFatalErrorScheduler = ErrorPluginConfiguration.toUserPayload(ErrorPluginConfiguration.createReportFatalErrorConf(SUFFIX_SCHEDULER));
UserPayload userPayloadReportNonFatalErrorLauncher = ErrorPluginConfiguration.toUserPayload(ErrorPluginConfiguration.createReportNonFatalErrorConf(SUFFIX_LAUNCHER));
UserPayload userPayloadReportNonFatalErrorTaskComm = ErrorPluginConfiguration.toUserPayload(ErrorPluginConfiguration.createReportNonFatalErrorConf(SUFFIX_TASKCOMM));
UserPayload userPayloadReportNonFatalErrorScheduler = ErrorPluginConfiguration.toUserPayload(ErrorPluginConfiguration.createReportNonFatalErrorConf(SUFFIX_SCHEDULER));
TaskSchedulerDescriptor[] taskSchedulerDescriptors = new TaskSchedulerDescriptor[] { TaskSchedulerDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskSchedulerService.class.getName()).setUserPayload(userPayload), TaskSchedulerDescriptor.create(EXT_THROW_ERROR_ENTITY_NAME, TezTestServiceTaskSchedulerServiceWithErrors.class.getName()).setUserPayload(userPayloadThrowError), TaskSchedulerDescriptor.create(EXT_REPORT_FATAL_ERROR_ENTITY_NAME, TezTestServiceTaskSchedulerServiceWithErrors.class.getName()).setUserPayload(userPayloadReportFatalErrorScheduler), TaskSchedulerDescriptor.create(EXT_REPORT_NON_FATAL_ERROR_ENTITY_NAME, TezTestServiceTaskSchedulerServiceWithErrors.class.getName()).setUserPayload(userPayloadReportNonFatalErrorScheduler) };
ContainerLauncherDescriptor[] containerLauncherDescriptors = new ContainerLauncherDescriptor[] { ContainerLauncherDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceNoOpContainerLauncher.class.getName()).setUserPayload(userPayload), ContainerLauncherDescriptor.create(EXT_THROW_ERROR_ENTITY_NAME, TezTestServiceContainerLauncherWithErrors.class.getName()).setUserPayload(userPayloadThrowError), ContainerLauncherDescriptor.create(EXT_REPORT_FATAL_ERROR_ENTITY_NAME, TezTestServiceContainerLauncherWithErrors.class.getName()).setUserPayload(userPayloadReportFatalErrorLauncher), ContainerLauncherDescriptor.create(EXT_REPORT_NON_FATAL_ERROR_ENTITY_NAME, TezTestServiceContainerLauncherWithErrors.class.getName()).setUserPayload(userPayloadReportNonFatalErrorLauncher) };
TaskCommunicatorDescriptor[] taskCommunicatorDescriptors = new TaskCommunicatorDescriptor[] { TaskCommunicatorDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskCommunicatorImpl.class.getName()).setUserPayload(userPayload), TaskCommunicatorDescriptor.create(EXT_THROW_ERROR_ENTITY_NAME, TezTestServiceTaskCommunicatorWithErrors.class.getName()).setUserPayload(userPayloadThrowError), TaskCommunicatorDescriptor.create(EXT_REPORT_FATAL_ERROR_ENTITY_NAME, TezTestServiceTaskCommunicatorWithErrors.class.getName()).setUserPayload(userPayloadReportFatalErrorTaskComm), TaskCommunicatorDescriptor.create(EXT_REPORT_NON_FATAL_ERROR_ENTITY_NAME, TezTestServiceTaskCommunicatorWithErrors.class.getName()).setUserPayload(userPayloadReportNonFatalErrorTaskComm) };
servicePluginsDescriptor = ServicePluginsDescriptor.create(true, true, taskSchedulerDescriptors, containerLauncherDescriptors, taskCommunicatorDescriptors);
extServiceTestHelper.setupSharedTezClient(servicePluginsDescriptor);
// Generate the join data set used for each run.
// Can a timeout be enforced here ?
Path dataPath1 = new Path(SRC_DATA_DIR, "inPath1");
Path dataPath2 = new Path(SRC_DATA_DIR, "inPath2");
extServiceTestHelper.setupHashJoinData(SRC_DATA_DIR, dataPath1, dataPath2, HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH);
extServiceTestHelper.shutdownSharedTezClient();
}
use of org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor in project tez by apache.
the class TestExternalTezServices method setup.
@BeforeClass
public static void setup() throws Exception {
extServiceTestHelper = new ExternalTezServiceTestHelper(TEST_ROOT_DIR);
UserPayload userPayload = TezUtils.createUserPayloadFromConf(extServiceTestHelper.getConfForJobs());
TaskSchedulerDescriptor[] taskSchedulerDescriptors = new TaskSchedulerDescriptor[] { TaskSchedulerDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskSchedulerService.class.getName()).setUserPayload(userPayload) };
ContainerLauncherDescriptor[] containerLauncherDescriptors = new ContainerLauncherDescriptor[] { ContainerLauncherDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceNoOpContainerLauncher.class.getName()).setUserPayload(userPayload) };
TaskCommunicatorDescriptor[] taskCommunicatorDescriptors = new TaskCommunicatorDescriptor[] { TaskCommunicatorDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskCommunicatorImpl.class.getName()).setUserPayload(userPayload) };
ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true, true, taskSchedulerDescriptors, containerLauncherDescriptors, taskCommunicatorDescriptors);
extServiceTestHelper.setupSharedTezClient(servicePluginsDescriptor);
// Generate the join data set used for each run.
// Can a timeout be enforced here ?
Path dataPath1 = new Path(SRC_DATA_DIR, "inPath1");
Path dataPath2 = new Path(SRC_DATA_DIR, "inPath2");
extServiceTestHelper.setupHashJoinData(SRC_DATA_DIR, dataPath1, dataPath2, HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH);
}
use of org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor in project hive by apache.
the class TezSessionState method openInternal.
protected void openInternal(final HiveConf conf, Collection<String> additionalFiles, boolean isAsync, LogHelper console, Path scratchDir) throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException {
this.conf = conf;
// TODO Why is the queue name set again. It has already been setup via setQueueName. Do only one of the two.
String confQueueName = conf.get(TezConfiguration.TEZ_QUEUE_NAME);
if (queueName != null && !queueName.equals(confQueueName)) {
LOG.warn("Resetting a queue name that was already set: was " + queueName + ", now " + confQueueName);
}
this.queueName = confQueueName;
this.doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS);
final boolean llapMode = "llap".equalsIgnoreCase(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE));
// TODO This - at least for the session pool - will always be the hive user. How does doAs above this affect things ?
UserGroupInformation ugi = Utils.getUGI();
user = ugi.getShortUserName();
LOG.info("User of session id " + sessionId + " is " + user);
// create the tez tmp dir
tezScratchDir = scratchDir == null ? createTezDir(sessionId) : scratchDir;
additionalFilesNotFromConf.clear();
if (additionalFiles != null) {
additionalFilesNotFromConf.addAll(additionalFiles);
}
refreshLocalResourcesFromConf(conf);
// unless already installed on all the cluster nodes, we'll have to
// localize hive-exec.jar as well.
appJarLr = createJarLocalResource(utils.getExecJarPathLocal());
// configuration for the application master
final Map<String, LocalResource> commonLocalResources = new HashMap<String, LocalResource>();
commonLocalResources.put(utils.getBaseName(appJarLr), appJarLr);
for (LocalResource lr : localizedResources) {
commonLocalResources.put(utils.getBaseName(lr), lr);
}
if (llapMode) {
// localize llap client jars
addJarLRByClass(LlapTaskSchedulerService.class, commonLocalResources);
addJarLRByClass(LlapProtocolClientImpl.class, commonLocalResources);
addJarLRByClass(LlapProtocolClientProxy.class, commonLocalResources);
addJarLRByClassName("org.apache.hadoop.registry.client.api.RegistryOperations", commonLocalResources);
}
// Create environment for AM.
Map<String, String> amEnv = new HashMap<String, String>();
MRHelpers.updateEnvBasedOnMRAMEnv(conf, amEnv);
// and finally we're ready to create and start the session
// generate basic tez config
final TezConfiguration tezConfig = new TezConfiguration(conf);
// set up the staging directory to use
tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString());
conf.stripHiddenConfigurations(tezConfig);
ServicePluginsDescriptor servicePluginsDescriptor;
Credentials llapCredentials = null;
if (llapMode) {
if (UserGroupInformation.isSecurityEnabled()) {
llapCredentials = new Credentials();
llapCredentials.addToken(LlapTokenIdentifier.KIND_NAME, getLlapToken(user, tezConfig));
}
// TODO Change this to not serialize the entire Configuration - minor.
UserPayload servicePluginPayload = TezUtils.createUserPayloadFromConf(tezConfig);
// we need plugins to handle llap and uber mode
servicePluginsDescriptor = ServicePluginsDescriptor.create(true, new TaskSchedulerDescriptor[] { TaskSchedulerDescriptor.create(LLAP_SERVICE, LLAP_SCHEDULER).setUserPayload(servicePluginPayload) }, new ContainerLauncherDescriptor[] { ContainerLauncherDescriptor.create(LLAP_SERVICE, LLAP_LAUNCHER) }, new TaskCommunicatorDescriptor[] { TaskCommunicatorDescriptor.create(LLAP_SERVICE, LLAP_TASK_COMMUNICATOR).setUserPayload(servicePluginPayload) });
} else {
servicePluginsDescriptor = ServicePluginsDescriptor.create(true);
}
// container prewarming. tell the am how many containers we need
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS);
n = Math.max(tezConfig.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT), n);
tezConfig.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, n);
}
setupSessionAcls(tezConfig, conf);
final TezClient session = TezClient.newBuilder("HIVE-" + sessionId, tezConfig).setIsSession(true).setLocalResources(commonLocalResources).setCredentials(llapCredentials).setServicePluginDescriptor(servicePluginsDescriptor).build();
LOG.info("Opening new Tez Session (id: " + sessionId + ", scratch dir: " + tezScratchDir + ")");
TezJobMonitor.initShutdownHook();
if (!isAsync) {
startSessionAndContainers(session, conf, commonLocalResources, tezConfig, false);
this.session = session;
} else {
FutureTask<TezClient> sessionFuture = new FutureTask<>(new Callable<TezClient>() {
@Override
public TezClient call() throws Exception {
try {
return startSessionAndContainers(session, conf, commonLocalResources, tezConfig, true);
} catch (Throwable t) {
LOG.error("Failed to start Tez session", t);
throw (t instanceof Exception) ? (Exception) t : new Exception(t);
}
}
});
new Thread(sessionFuture, "Tez session start thread").start();
// We assume here nobody will try to get session before open() returns.
this.console = console;
this.sessionFuture = sessionFuture;
}
}
use of org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor in project tez by apache.
the class TestTaskSchedulerManager method testReportFailureFromTaskScheduler.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskScheduler() {
String dagName = DAG_NAME;
Configuration conf = new TezConfiguration();
String taskSchedulerName = "testTaskScheduler";
String expIdentifier = "[0:" + taskSchedulerName + "]";
EventHandler eventHandler = mock(EventHandler.class);
AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
doReturn(taskSchedulerName).when(appContext).getTaskSchedulerName(0);
doReturn(eventHandler).when(appContext).getEventHandler();
doReturn(conf).when(appContext).getAMConf();
InetSocketAddress address = new InetSocketAddress("host", 55000);
DAGClientServer dagClientServer = mock(DAGClientServer.class);
doReturn(address).when(dagClientServer).getBindAddress();
DAG dag = mock(DAG.class);
TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
doReturn(dagName).when(dag).getName();
doReturn(dagId).when(dag).getID();
doReturn(dag).when(appContext).getCurrentDAG();
NamedEntityDescriptor<TaskSchedulerDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>(taskSchedulerName, TaskSchedulerForFailureTest.class.getName());
List<NamedEntityDescriptor> list = new LinkedList<>();
list.add(namedEntityDescriptor);
TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(appContext, dagClientServer, eventHandler, mock(ContainerSignatureMatcher.class), mock(WebUIService.class), list, false, new HadoopShimsLoader(appContext.getAMConf()).getHadoopShim()) {
@Override
TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
// Avoid wrapping in threads
return rawContext;
}
};
try {
taskSchedulerManager.init(new TezConfiguration());
taskSchedulerManager.start();
taskSchedulerManager.getTotalResources(0);
ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
verify(eventHandler, times(1)).handle(argumentCaptor.capture());
Event rawEvent = argumentCaptor.getValue();
assertTrue(rawEvent instanceof DAGEventTerminateDag);
DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
assertTrue(killEvent.getDiagnosticInfo().contains(expIdentifier));
reset(eventHandler);
taskSchedulerManager.getAvailableResources(0);
argumentCaptor = ArgumentCaptor.forClass(Event.class);
verify(eventHandler, times(1)).handle(argumentCaptor.capture());
rawEvent = argumentCaptor.getValue();
assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
assertTrue(event.getDiagnosticInfo().contains(expIdentifier));
} finally {
taskSchedulerManager.stop();
}
}
Aggregations