Examples with JobTokenIdentifier org.apache.tez.common.security.JobTokenIdentifier used on opensource projects

Example 11 with JobTokenIdentifier

use of org.apache.tez.common.security.JobTokenIdentifier in project tez by apache.

the class ShuffleUtils method getJobTokenSecretFromTokenBytes.

public static SecretKey getJobTokenSecretFromTokenBytes(ByteBuffer meta) throws IOException {
    DataInputByteBuffer in = new DataInputByteBuffer();
    in.reset(meta);
    Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>();
    jt.readFields(in);
    SecretKey sk = JobTokenSecretManager.createSecretKey(jt.getPassword());
    return sk;
}

Example 12 with JobTokenIdentifier

use of org.apache.tez.common.security.JobTokenIdentifier in project tez by apache.

the class TestTezClientUtils method testSessionTokenInAmClc.

@Test(timeout = 5000)
public void testSessionTokenInAmClc() throws IOException, YarnException {
    TezConfiguration tezConf = new TezConfiguration();
    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    DAG dag = DAG.create("testdag");
    dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1).setTaskLaunchCmdOpts("initialLaunchOpts"));
    Credentials credentials = new Credentials();
    JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager();
    TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials);
    Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
    assertNotNull(jobToken);
    AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials);
    ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf, new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(), null, null);
    ContainerLaunchContext amClc = appSubmissionContext.getAMContainerSpec();
    Map<String, ByteBuffer> amServiceData = amClc.getServiceData();
    assertNotNull(amServiceData);
    assertEquals(1, amServiceData.size());
    DataInputByteBuffer dibb = new DataInputByteBuffer();
    dibb.reset(amServiceData.values().iterator().next());
    Token<JobTokenIdentifier> jtSent = new Token<JobTokenIdentifier>();
    jtSent.readFields(dibb);
    assertTrue(Arrays.equals(jobToken.getIdentifier(), jtSent.getIdentifier()));
}

Example 13 with JobTokenIdentifier

use of org.apache.tez.common.security.JobTokenIdentifier in project hive by apache.

the class GenericUDTFGetSplits method getSplits.

// generateLightWeightSplits - if true then
// 1) schema and planBytes[] in each LlapInputSplit are not populated
// 2) schemaSplit(contains only schema) and planSplit(contains only planBytes[]) are populated in SplitResult
private SplitResult getSplits(JobConf job, TezWork work, Schema schema, ApplicationId extClientAppId, boolean generateLightWeightSplits) throws IOException {
    SplitResult splitResult = new SplitResult();
    splitResult.schemaSplit = new LlapInputSplit(0, new byte[0], new byte[0], new byte[0], new SplitLocationInfo[0], new LlapDaemonInfo[0], schema, "", new byte[0], "");
    if (schemaSplitOnly) {
        // schema only
        return splitResult;
    }
    DAG dag = DAG.create(work.getName());
    dag.setCredentials(job.getCredentials());
    DagUtils utils = DagUtils.getInstance();
    Context ctx = new Context(job);
    MapWork mapWork = (MapWork) work.getAllWork().get(0);
    // bunch of things get setup in the context based on conf but we need only the MR tmp directory
    // for the following method.
    JobConf wxConf = utils.initializeVertexConf(job, ctx, mapWork);
    // TODO: should we also whitelist input formats here? from mapred.input.format.class
    Path scratchDir = utils.createTezDir(ctx.getMRScratchDir(), job);
    try {
        LocalResource appJarLr = createJarLocalResource(utils.getExecJarPathLocal(ctx.getConf()), utils, job);
        LlapCoordinator coordinator = LlapCoordinator.getInstance();
        if (coordinator == null) {
            throw new IOException("LLAP coordinator is not initialized; must be running in HS2 with " + ConfVars.LLAP_HS2_ENABLE_COORDINATOR.varname + " enabled");
        }
        // Update the queryId to use the generated extClientAppId. See comment below about
        // why this is done.
        HiveConf.setVar(wxConf, HiveConf.ConfVars.HIVEQUERYID, extClientAppId.toString());
        Vertex wx = utils.createVertex(wxConf, mapWork, scratchDir, work, DagUtils.createTezLrMap(appJarLr, null));
        String vertexName = wx.getName();
        dag.addVertex(wx);
        utils.addCredentials(mapWork, dag, job);
        // we have the dag now proceed to get the splits:
        Preconditions.checkState(HiveConf.getBoolVar(wxConf, ConfVars.HIVE_TEZ_GENERATE_CONSISTENT_SPLITS));
        Preconditions.checkState(HiveConf.getBoolVar(wxConf, ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS));
        HiveSplitGenerator splitGenerator = new HiveSplitGenerator(wxConf, mapWork, false, inputArgNumSplits);
        List<Event> eventList = splitGenerator.initialize();
        int numGroupedSplitsGenerated = eventList.size() - 1;
        InputSplit[] result = new InputSplit[numGroupedSplitsGenerated];
        InputConfigureVertexTasksEvent configureEvent = (InputConfigureVertexTasksEvent) eventList.get(0);
        List<TaskLocationHint> hints = configureEvent.getLocationHint().getTaskLocationHints();
        Preconditions.checkState(hints.size() == numGroupedSplitsGenerated);
        if (LOG.isDebugEnabled()) {
            LOG.debug("NumEvents=" + eventList.size() + ", NumSplits=" + result.length);
        }
        // This assumes LLAP cluster owner is always the HS2 user.
        String llapUser = LlapRegistryService.currentUser();
        String queryUser = null;
        byte[] tokenBytes = null;
        LlapSigner signer = null;
        if (UserGroupInformation.isSecurityEnabled()) {
            signer = coordinator.getLlapSigner(job);
            // 1. Generate the token for query user (applies to all splits).
            queryUser = SessionState.getUserFromAuthenticator();
            if (queryUser == null) {
                queryUser = UserGroupInformation.getCurrentUser().getUserName();
                LOG.warn("Cannot determine the session user; using " + queryUser + " instead");
            }
            LlapTokenLocalClient tokenClient = coordinator.getLocalTokenClient(job, llapUser);
            // We put the query user, not LLAP user, into the message and token.
            Token<LlapTokenIdentifier> token = tokenClient.createToken(extClientAppId.toString(), queryUser, true);
            LOG.info("Created the token for remote user: {}", token);
            bos.reset();
            token.write(dos);
            tokenBytes = bos.toByteArray();
        } else {
            queryUser = UserGroupInformation.getCurrentUser().getUserName();
        }
        // Generate umbilical token (applies to all splits)
        Token<JobTokenIdentifier> umbilicalToken = JobTokenCreator.createJobToken(extClientAppId);
        LOG.info("Number of splits: " + numGroupedSplitsGenerated);
        SignedMessage signedSvs = null;
        byte[] submitWorkBytes = null;
        final byte[] emptySubmitWorkBytes = new byte[0];
        final Schema emptySchema = new Schema();
        for (int i = 0; i < numGroupedSplitsGenerated; i++) {
            TaskSpec taskSpec = new TaskSpecBuilder().constructTaskSpec(dag, vertexName, numGroupedSplitsGenerated, extClientAppId, i);
            // 2. Generate the vertex/submit information for all events.
            if (i == 0) {
                // The queryId could either be picked up from the current request being processed, or
                // generated. The current request isn't exactly correct since the query is 'done' once we
                // return the results. Generating a new one has the added benefit of working once this
                // is moved out of a UDTF into a proper API.
                // Setting this to the generated AppId which is unique.
                // Despite the differences in TaskSpec, the vertex spec should be the same.
                signedSvs = createSignedVertexSpec(signer, taskSpec, extClientAppId, queryUser, extClientAppId.toString());
                SubmitWorkInfo submitWorkInfo = new SubmitWorkInfo(extClientAppId, System.currentTimeMillis(), numGroupedSplitsGenerated, signedSvs.message, signedSvs.signature, umbilicalToken);
                submitWorkBytes = SubmitWorkInfo.toBytes(submitWorkInfo);
                if (generateLightWeightSplits) {
                    splitResult.planSplit = new LlapInputSplit(0, submitWorkBytes, new byte[0], new byte[0], new SplitLocationInfo[0], new LlapDaemonInfo[0], new Schema(), "", new byte[0], "");
                }
            }
            // 3. Generate input event.
            SignedMessage eventBytes = makeEventBytes(wx, vertexName, eventList.get(i + 1), signer);
            // 4. Make location hints.
            SplitLocationInfo[] locations = makeLocationHints(hints.get(i));
            // 5. populate info about llap daemons(to help client submit request and read data)
            LlapDaemonInfo[] llapDaemonInfos = populateLlapDaemonInfos(job, locations);
            // 6. Generate JWT for external clients if it's a cloud deployment
            // we inject extClientAppId in JWT which is same as what fragment contains.
            // extClientAppId in JWT and in fragment are compared on LLAP when a fragment is submitted.
            // see method ContainerRunnerImpl#verifyJwtForExternalClient
            String jwt = "";
            if (LlapUtil.isCloudDeployment(job)) {
                LlapExtClientJwtHelper llapExtClientJwtHelper = new LlapExtClientJwtHelper(job);
                jwt = llapExtClientJwtHelper.buildJwtForLlap(extClientAppId);
            }
            if (generateLightWeightSplits) {
                result[i] = new LlapInputSplit(i, emptySubmitWorkBytes, eventBytes.message, eventBytes.signature, locations, llapDaemonInfos, emptySchema, llapUser, tokenBytes, jwt);
            } else {
                result[i] = new LlapInputSplit(i, submitWorkBytes, eventBytes.message, eventBytes.signature, locations, llapDaemonInfos, schema, llapUser, tokenBytes, jwt);
            }
        }
        splitResult.actualSplits = result;
        return splitResult;
    } catch (Exception e) {
        throw new IOException(e);
    }
}

Example 14 with JobTokenIdentifier

use of org.apache.tez.common.security.JobTokenIdentifier in project hive by apache.

the class SubmitWorkInfo method createJobToken.

private Token<JobTokenIdentifier> createJobToken() {
    String tokenIdentifier = fakeAppId.toString();
    JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(tokenIdentifier));
    Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, new JobTokenSecretManager());
    sessionToken.setService(identifier.getJobId());
    return sessionToken;
}

Example 15 with JobTokenIdentifier

use of org.apache.tez.common.security.JobTokenIdentifier in project tez by apache.

the class TestTaskCommunicatorManager1 method testPortRange.

private boolean testPortRange(int port) {
    boolean succeedToAllocate = true;
    try {
        Configuration conf = new Configuration();
        JobTokenIdentifier identifier = new JobTokenIdentifier(new Text("fakeIdentifier"));
        Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, new JobTokenSecretManager());
        sessionToken.setService(identifier.getJobId());
        TokenCache.setSessionToken(sessionToken, credentials);
        conf.set(TezConfiguration.TEZ_AM_TASK_AM_PORT_RANGE, port + "-" + port);
        UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
        taskAttemptListener = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload)));
        taskAttemptListener.init(conf);
        taskAttemptListener.start();
        int resultedPort = taskAttemptListener.getTaskCommunicator(0).getAddress().getPort();
        assertEquals(port, resultedPort);
    } catch (Exception e) {
        succeedToAllocate = false;
    } finally {
        if (taskAttemptListener != null) {
            try {
                taskAttemptListener.close();
            } catch (IOException e) {
                e.printStackTrace();
                fail("fail to stop TaskAttemptListener");
            }
        }
    }
    return succeedToAllocate;
}