Examples with SignedMessage org.apache.hadoop.hive.llap.security.LlapSigner.SignedMessage used on opensource projects

Example 1 with SignedMessage

use of org.apache.hadoop.hive.llap.security.LlapSigner.SignedMessage in project hive by apache.

the class GenericUDTFGetSplits method getSplits.

public InputSplit[] getSplits(JobConf job, int numSplits, TezWork work, Schema schema) throws IOException {
    DAG dag = DAG.create(work.getName());
    dag.setCredentials(job.getCredentials());
    DagUtils utils = DagUtils.getInstance();
    Context ctx = new Context(job);
    MapWork mapWork = (MapWork) work.getAllWork().get(0);
    // bunch of things get setup in the context based on conf but we need only the MR tmp directory
    // for the following method.
    JobConf wxConf = utils.initializeVertexConf(job, ctx, mapWork);
    // TODO: should we also whitelist input formats here? from mapred.input.format.class
    Path scratchDir = utils.createTezDir(ctx.getMRScratchDir(), job);
    FileSystem fs = scratchDir.getFileSystem(job);
    try {
        LocalResource appJarLr = createJarLocalResource(utils.getExecJarPathLocal(), utils, job);
        Vertex wx = utils.createVertex(wxConf, mapWork, scratchDir, appJarLr, new ArrayList<LocalResource>(), fs, ctx, false, work, work.getVertexType(mapWork));
        String vertexName = wx.getName();
        dag.addVertex(wx);
        utils.addCredentials(mapWork, dag);
        // we have the dag now proceed to get the splits:
        Preconditions.checkState(HiveConf.getBoolVar(wxConf, ConfVars.HIVE_TEZ_GENERATE_CONSISTENT_SPLITS));
        Preconditions.checkState(HiveConf.getBoolVar(wxConf, ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS));
        HiveSplitGenerator splitGenerator = new HiveSplitGenerator(wxConf, mapWork);
        List<Event> eventList = splitGenerator.initialize();
        InputSplit[] result = new InputSplit[eventList.size() - 1];
        InputConfigureVertexTasksEvent configureEvent = (InputConfigureVertexTasksEvent) eventList.get(0);
        List<TaskLocationHint> hints = configureEvent.getLocationHint().getTaskLocationHints();
        Preconditions.checkState(hints.size() == eventList.size() - 1);
        if (LOG.isDebugEnabled()) {
            LOG.debug("NumEvents=" + eventList.size() + ", NumSplits=" + result.length);
        }
        LlapCoordinator coordinator = LlapCoordinator.getInstance();
        if (coordinator == null) {
            throw new IOException("LLAP coordinator is not initialized; must be running in HS2 with " + ConfVars.LLAP_HS2_ENABLE_COORDINATOR.varname + " enabled");
        }
        // See the discussion in the implementation as to why we generate app ID.
        ApplicationId applicationId = coordinator.createExtClientAppId();
        // This assumes LLAP cluster owner is always the HS2 user.
        String llapUser = UserGroupInformation.getLoginUser().getShortUserName();
        String queryUser = null;
        byte[] tokenBytes = null;
        LlapSigner signer = null;
        if (UserGroupInformation.isSecurityEnabled()) {
            signer = coordinator.getLlapSigner(job);
            // 1. Generate the token for query user (applies to all splits).
            queryUser = SessionState.getUserFromAuthenticator();
            if (queryUser == null) {
                queryUser = UserGroupInformation.getCurrentUser().getUserName();
                LOG.warn("Cannot determine the session user; using " + queryUser + " instead");
            }
            LlapTokenLocalClient tokenClient = coordinator.getLocalTokenClient(job, llapUser);
            // We put the query user, not LLAP user, into the message and token.
            Token<LlapTokenIdentifier> token = tokenClient.createToken(applicationId.toString(), queryUser, true);
            LOG.info("Created the token for remote user: {}", token);
            bos.reset();
            token.write(dos);
            tokenBytes = bos.toByteArray();
        } else {
            queryUser = UserGroupInformation.getCurrentUser().getUserName();
        }
        LOG.info("Number of splits: " + (eventList.size() - 1));
        SignedMessage signedSvs = null;
        for (int i = 0; i < eventList.size() - 1; i++) {
            TaskSpec taskSpec = new TaskSpecBuilder().constructTaskSpec(dag, vertexName, eventList.size() - 1, applicationId, i);
            // 2. Generate the vertex/submit information for all events.
            if (i == 0) {
                // The queryId could either be picked up from the current request being processed, or
                // generated. The current request isn't exactly correct since the query is 'done' once we
                // return the results. Generating a new one has the added benefit of working once this
                // is moved out of a UDTF into a proper API.
                // Setting this to the generated AppId which is unique.
                // Despite the differences in TaskSpec, the vertex spec should be the same.
                signedSvs = createSignedVertexSpec(signer, taskSpec, applicationId, queryUser, applicationId.toString());
            }
            SubmitWorkInfo submitWorkInfo = new SubmitWorkInfo(applicationId, System.currentTimeMillis(), taskSpec.getVertexParallelism(), signedSvs.message, signedSvs.signature);
            byte[] submitWorkBytes = SubmitWorkInfo.toBytes(submitWorkInfo);
            // 3. Generate input event.
            SignedMessage eventBytes = makeEventBytes(wx, vertexName, eventList.get(i + 1), signer);
            // 4. Make location hints.
            SplitLocationInfo[] locations = makeLocationHints(hints.get(i));
            result[i] = new LlapInputSplit(i, submitWorkBytes, eventBytes.message, eventBytes.signature, locations, schema, llapUser, tokenBytes);
        }
        return result;
    } catch (Exception e) {
        throw new IOException(e);
    }
}

Example 2 with SignedMessage

use of org.apache.hadoop.hive.llap.security.LlapSigner.SignedMessage in project hive by apache.

the class TestLlapSignerImpl method testSigning.

@Test(timeout = 10000)
public void testSigning() throws Exception {
    FakeSecretManager fsm = new FakeSecretManager();
    fsm.startThreads();
    // Make sure the signature works.
    LlapSignerImpl signer = new LlapSignerImpl(fsm);
    byte theByte = 1;
    TestSignable in = new TestSignable(theByte);
    TestSignable in2 = new TestSignable(++theByte);
    SignedMessage sm = signer.serializeAndSign(in);
    SignedMessage sm2 = signer.serializeAndSign(in2);
    TestSignable out = TestSignable.deserialize(sm.message);
    TestSignable out2 = TestSignable.deserialize(sm2.message);
    assertEquals(in, out);
    assertEquals(in2, out2);
    signer.checkSignature(sm.message, sm.signature, out.masterKeyId);
    signer.checkSignature(sm2.message, sm2.signature, out2.masterKeyId);
    // Make sure the broken signature doesn't work.
    try {
        signer.checkSignature(sm.message, sm2.signature, out.masterKeyId);
        fail("Didn't throw");
    } catch (SecurityException ex) {
    // Expected.
    }
    int index = sm.signature.length / 2;
    sm.signature[index] = (byte) (sm.signature[index] + 1);
    try {
        signer.checkSignature(sm.message, sm.signature, out.masterKeyId);
        fail("Didn't throw");
    } catch (SecurityException ex) {
    // Expected.
    }
    sm.signature[index] = (byte) (sm.signature[index] - 1);
    fsm = rollKey(fsm, out.masterKeyId);
    signer = new LlapSignerImpl(fsm);
    // Sign in2 with a different key.
    sm2 = signer.serializeAndSign(in2);
    out2 = TestSignable.deserialize(sm2.message);
    assertNotEquals(out.masterKeyId, out2.masterKeyId);
    assertEquals(in2, out2);
    signer.checkSignature(sm2.message, sm2.signature, out2.masterKeyId);
    signer.checkSignature(sm.message, sm.signature, out.masterKeyId);
    // Make sure the key ID mismatch causes error.
    try {
        signer.checkSignature(sm2.message, sm2.signature, out.masterKeyId);
        fail("Didn't throw");
    } catch (SecurityException ex) {
    // Expected.
    }
    // The same for rolling the key; re-create the fsm with only the key #2.
    fsm = rollKey(fsm, out2.masterKeyId);
    signer = new LlapSignerImpl(fsm);
    signer.checkSignature(sm2.message, sm2.signature, out2.masterKeyId);
    // The key is missing - shouldn't be able to verify.
    try {
        signer.checkSignature(sm.message, sm.signature, out.masterKeyId);
        fail("Didn't throw");
    } catch (SecurityException ex) {
    // Expected.
    }
    fsm.stopThreads();
}

Example 3 with SignedMessage

use of org.apache.hadoop.hive.llap.security.LlapSigner.SignedMessage in project hive by apache.

the class GenericUDTFGetSplits method makeEventBytes.

private SignedMessage makeEventBytes(Vertex wx, String vertexName, Event event, LlapSigner signer) throws IOException {
    assert event instanceof InputDataInformationEvent;
    List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = TaskSpecBuilder.getVertexInputs(wx);
    Preconditions.checkState(inputs.size() == 1);
    Signable signableNte = NotTezEventHelper.createSignableNotTezEvent((InputDataInformationEvent) event, vertexName, inputs.get(0).getName());
    if (signer != null) {
        return signer.serializeAndSign(signableNte);
    } else {
        SignedMessage sm = new SignedMessage();
        sm.message = signableNte.serialize();
        return sm;
    }
}

Example 4 with SignedMessage

use of org.apache.hadoop.hive.llap.security.LlapSigner.SignedMessage in project hive by apache.

the class GenericUDTFGetSplits method createSignedVertexSpec.

private SignedMessage createSignedVertexSpec(LlapSigner signer, TaskSpec taskSpec, ApplicationId applicationId, String queryUser, String queryIdString) throws IOException {
    QueryIdentifierProto queryIdentifierProto = QueryIdentifierProto.newBuilder().setApplicationIdString(applicationId.toString()).setDagIndex(taskSpec.getDagIdentifier()).setAppAttemptNumber(0).build();
    final SignableVertexSpec.Builder svsb = Converters.constructSignableVertexSpec(taskSpec, queryIdentifierProto, applicationId.toString(), queryUser, queryIdString);
    if (signer == null) {
        SignedMessage result = new SignedMessage();
        result.message = serializeVertexSpec(svsb);
        return result;
    }
    return signer.serializeAndSign(new Signable() {

        @Override
        public void setSignInfo(int masterKeyId) {
            svsb.setSignatureKeyId(masterKeyId);
        }

        @Override
        public byte[] serialize() throws IOException {
            return serializeVertexSpec(svsb);
        }
    });
}