Examples with BlockTokenIdentifier org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier used on opensource projects

Example 11 with BlockTokenIdentifier

use of org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier in project hadoop by apache.

the class NamenodeFsck method copyBlock.

/*
   * XXX (ab) Bulk of this method is copied verbatim from {@link DFSClient}, which is
   * bad. Both places should be refactored to provide a method to copy blocks
   * around.
   */
private void copyBlock(final DFSClient dfs, LocatedBlock lblock, OutputStream fos) throws Exception {
    int failures = 0;
    InetSocketAddress targetAddr = null;
    TreeSet<DatanodeInfo> deadNodes = new TreeSet<DatanodeInfo>();
    BlockReader blockReader = null;
    ExtendedBlock block = lblock.getBlock();
    while (blockReader == null) {
        DatanodeInfo chosenNode;
        try {
            chosenNode = bestNode(dfs, lblock.getLocations(), deadNodes);
            targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
        } catch (IOException ie) {
            if (failures >= HdfsClientConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT) {
                throw new IOException("Could not obtain block " + lblock, ie);
            }
            LOG.info("Could not obtain block from any node:  " + ie);
            try {
                Thread.sleep(10000);
            } catch (InterruptedException iex) {
            }
            deadNodes.clear();
            failures++;
            continue;
        }
        try {
            String file = BlockReaderFactory.getFileName(targetAddr, block.getBlockPoolId(), block.getBlockId());
            blockReader = new BlockReaderFactory(dfs.getConf()).setFileName(file).setBlock(block).setBlockToken(lblock.getBlockToken()).setStartOffset(0).setLength(block.getNumBytes()).setVerifyChecksum(true).setClientName("fsck").setDatanodeInfo(chosenNode).setInetSocketAddress(targetAddr).setCachingStrategy(CachingStrategy.newDropBehind()).setClientCacheContext(dfs.getClientContext()).setConfiguration(namenode.getConf()).setTracer(tracer).setRemotePeerFactory(new RemotePeerFactory() {

                @Override
                public Peer newConnectedPeer(InetSocketAddress addr, Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId) throws IOException {
                    Peer peer = null;
                    Socket s = NetUtils.getDefaultSocketFactory(conf).createSocket();
                    try {
                        s.connect(addr, HdfsConstants.READ_TIMEOUT);
                        s.setSoTimeout(HdfsConstants.READ_TIMEOUT);
                        peer = DFSUtilClient.peerFromSocketAndKey(dfs.getSaslDataTransferClient(), s, NamenodeFsck.this, blockToken, datanodeId, HdfsConstants.READ_TIMEOUT);
                    } finally {
                        if (peer == null) {
                            IOUtils.closeQuietly(s);
                        }
                    }
                    return peer;
                }
            }).build();
        } catch (IOException ex) {
            // Put chosen node into dead list, continue
            LOG.info("Failed to connect to " + targetAddr + ":" + ex);
            deadNodes.add(chosenNode);
        }
    }
    byte[] buf = new byte[1024];
    int cnt = 0;
    boolean success = true;
    long bytesRead = 0;
    try {
        while ((cnt = blockReader.read(buf, 0, buf.length)) > 0) {
            fos.write(buf, 0, cnt);
            bytesRead += cnt;
        }
        if (bytesRead != block.getNumBytes()) {
            throw new IOException("Recorded block size is " + block.getNumBytes() + ", but datanode returned " + bytesRead + " bytes");
        }
    } catch (Exception e) {
        LOG.error("Error reading block", e);
        success = false;
    } finally {
        blockReader.close();
    }
    if (!success) {
        throw new Exception("Could not copy block data for " + lblock.getBlock());
    }
}

Example 12 with BlockTokenIdentifier

use of org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier in project hadoop by apache.

the class TestDecommissionWithStriped method prepareBlockIndexAndTokenList.

private void prepareBlockIndexAndTokenList(List<LocatedBlock> lbs, List<HashMap<DatanodeInfo, Byte>> locToIndexList, List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList) {
    for (LocatedBlock lb : lbs) {
        HashMap<DatanodeInfo, Byte> locToIndex = new HashMap<DatanodeInfo, Byte>();
        locToIndexList.add(locToIndex);
        HashMap<DatanodeInfo, Token<BlockTokenIdentifier>> locToToken = new HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>();
        locToTokenList.add(locToToken);
        DatanodeInfo[] di = lb.getLocations();
        LocatedStripedBlock stripedBlk = (LocatedStripedBlock) lb;
        for (int i = 0; i < di.length; i++) {
            locToIndex.put(di[i], stripedBlk.getBlockIndices()[i]);
            locToToken.put(di[i], stripedBlk.getBlockTokens()[i]);
        }
    }
}

Example 13 with BlockTokenIdentifier

use of org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier in project hadoop by apache.

the class TestSortLocatedStripedBlock method prepareBlockIndexAndTokenList.

private void prepareBlockIndexAndTokenList(List<LocatedBlock> lbs, List<HashMap<DatanodeInfo, Byte>> locToIndexList, List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList) {
    for (LocatedBlock lb : lbs) {
        HashMap<DatanodeInfo, Byte> locToIndex = new HashMap<DatanodeInfo, Byte>();
        locToIndexList.add(locToIndex);
        HashMap<DatanodeInfo, Token<BlockTokenIdentifier>> locToToken = new HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>();
        locToTokenList.add(locToToken);
        DatanodeInfo[] di = lb.getLocations();
        LocatedStripedBlock stripedBlk = (LocatedStripedBlock) lb;
        for (int i = 0; i < di.length; i++) {
            locToIndex.put(di[i], stripedBlk.getBlockIndices()[i]);
            locToToken.put(di[i], stripedBlk.getBlockTokens()[i]);
        }
    }
}

Example 14 with BlockTokenIdentifier

use of org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier in project hadoop by apache.

the class TestSortLocatedStripedBlock method testWithMultipleInServiceAndDecommnDatanodes.

/**
   * Test to verify sorting with multiple in-service and decommissioned
   * datanodes exists in storage lists.
   *
   * We have storage list, marked decommissioned internal blocks with a '
   * d0, d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12, d13
   * mapping to indices
   * 0', 1', 2, 3, 4, 5, 6, 7', 8', 0, 1, 7, 8, 1
   *
   * Decommissioned node indices: 0', 1', 7', 8'
   *
   * Additional In-Service node d13 at the end, block index: 1
   *
   * So in the original list nodes d0, d1, d7, d8 are decommissioned state.
   *
   * After sorting the expected block indices list will be,
   * 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 0', 1', 7', 8'
   *
   * After sorting the expected storage list will be,
   * d9, d10, d2, d3, d4, d5, d6, d11, d12, d13, d0, d1, d7, d8.
   *
   * Note: after sorting block indices will not be in ascending order.
   */
@Test(timeout = 10000)
public void testWithMultipleInServiceAndDecommnDatanodes() {
    LOG.info("Starting test testWithMultipleInServiceAndDecommnDatanodes");
    // two located block groups
    int lbsCount = 2;
    List<Integer> decommnNodeIndices = new ArrayList<>();
    decommnNodeIndices.add(0);
    decommnNodeIndices.add(1);
    decommnNodeIndices.add(7);
    decommnNodeIndices.add(8);
    List<Integer> targetNodeIndices = new ArrayList<>();
    targetNodeIndices.addAll(decommnNodeIndices);
    // at the end add an additional In-Service node to blockIndex=1
    targetNodeIndices.add(1);
    // map contains decommissioned node details in each located strip block
    // which will be used for assertions
    HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(lbsCount * decommnNodeIndices.size());
    List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount, dataBlocks, parityBlocks, decommnNodeIndices, targetNodeIndices, decommissionedNodes);
    List<DatanodeInfo> staleDns = new ArrayList<>();
    for (LocatedBlock lb : lbs) {
        DatanodeInfo[] locations = lb.getLocations();
        DatanodeInfo staleDn = locations[locations.length - 1];
        staleDn.setLastUpdateMonotonic(Time.monotonicNow() - (STALE_INTERVAL * 2));
        staleDns.add(staleDn);
    }
    // prepare expected block index and token list.
    List<HashMap<DatanodeInfo, Byte>> locToIndexList = new ArrayList<>();
    List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList = new ArrayList<>();
    prepareBlockIndexAndTokenList(lbs, locToIndexList, locToTokenList);
    dm.sortLocatedBlocks(null, lbs);
    assertDecommnNodePosition(groupSize + 1, decommissionedNodes, lbs);
    assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
    for (LocatedBlock lb : lbs) {
        byte[] blockIndices = ((LocatedStripedBlock) lb).getBlockIndices();
        // after sorting stale block index will be placed after normal nodes.
        Assert.assertEquals("Failed to move stale node to bottom!", 1, blockIndices[9]);
        DatanodeInfo[] locations = lb.getLocations();
        // After sorting stale node d13 will be placed after normal nodes
        Assert.assertEquals("Failed to move stale dn after normal one!", staleDns.remove(0), locations[9]);
    }
}

Example 15 with BlockTokenIdentifier

use of org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier in project hadoop by apache.

the class SaslDataTransferServer method buildServerPassword.

/**
   * Calculates the expected correct password on the server side for the
   * general-purpose handshake.  The password consists of the block access
   * token's password (known to the DataNode via its secret manager).  This
   * expects that the client has supplied a user name consisting of its
   * serialized block access token identifier.
   *
   * @param userName SASL user name containing serialized block access token
   *   identifier
   * @return expected correct SASL password
   * @throws IOException for any error
   */
private char[] buildServerPassword(String userName) throws IOException {
    BlockTokenIdentifier identifier = deserializeIdentifier(userName);
    byte[] tokenPassword = blockPoolTokenSecretManager.retrievePassword(identifier);
    return (new String(Base64.encodeBase64(tokenPassword, false), Charsets.UTF_8)).toCharArray();
}