Examples with Slot org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot used on opensource projects

Example 1 with Slot

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot in project hadoop by apache.

the class ShortCircuitRegistry method processBlockMunlockRequest.

/**
   * Mark any slots associated with this blockId as unanchorable.
   *
   * @param blockId        The block ID.
   * @return               True if we should allow the munlock request.
   */
public synchronized boolean processBlockMunlockRequest(ExtendedBlockId blockId) {
    if (!enabled)
        return true;
    boolean allowMunlock = true;
    Set<Slot> affectedSlots = slots.get(blockId);
    for (Slot slot : affectedSlots) {
        slot.makeUnanchorable();
        if (slot.isAnchored()) {
            allowMunlock = false;
        }
    }
    return allowMunlock;
}

Example 2 with Slot

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot in project hadoop by apache.

the class ShortCircuitRegistry method registerSlot.

public synchronized void registerSlot(ExtendedBlockId blockId, SlotId slotId, boolean isCached) throws InvalidRequestException {
    if (!enabled) {
        if (LOG.isTraceEnabled()) {
            LOG.trace(this + " can't register a slot because the " + "ShortCircuitRegistry is not enabled.");
        }
        throw new UnsupportedOperationException();
    }
    ShmId shmId = slotId.getShmId();
    RegisteredShm shm = segments.get(shmId);
    if (shm == null) {
        throw new InvalidRequestException("there is no shared memory segment " + "registered with shmId " + shmId);
    }
    Slot slot = shm.registerSlot(slotId.getSlotIdx(), blockId);
    if (isCached) {
        slot.makeAnchorable();
    } else {
        slot.makeUnanchorable();
    }
    boolean added = slots.put(blockId, slot);
    Preconditions.checkState(added);
    if (LOG.isTraceEnabled()) {
        LOG.trace(this + ": registered " + blockId + " with slot " + slotId + " (isCached=" + isCached + ")");
    }
}

Example 3 with Slot

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot in project hadoop by apache.

the class ShortCircuitRegistry method unregisterSlot.

public synchronized void unregisterSlot(SlotId slotId) throws InvalidRequestException {
    if (!enabled) {
        if (LOG.isTraceEnabled()) {
            LOG.trace("unregisterSlot: ShortCircuitRegistry is " + "not enabled.");
        }
        throw new UnsupportedOperationException();
    }
    ShmId shmId = slotId.getShmId();
    RegisteredShm shm = segments.get(shmId);
    if (shm == null) {
        throw new InvalidRequestException("there is no shared memory segment " + "registered with shmId " + shmId);
    }
    Slot slot = shm.getSlot(slotId.getSlotIdx());
    slot.makeInvalid();
    shm.unregisterSlot(slotId.getSlotIdx());
    slots.remove(slot.getBlockId(), slot);
}

Example 4 with Slot

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot in project hadoop by apache.

the class TestShortCircuitShm method testAllocateSlots.

@Test(timeout = 60000)
public void testAllocateSlots() throws Exception {
    File path = new File(TEST_BASE, "testAllocateSlots");
    path.mkdirs();
    SharedFileDescriptorFactory factory = SharedFileDescriptorFactory.create("shm_", new String[] { path.getAbsolutePath() });
    FileInputStream stream = factory.createDescriptor("testAllocateSlots", 4096);
    ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
    int numSlots = 0;
    ArrayList<Slot> slots = new ArrayList<Slot>();
    while (!shm.isFull()) {
        Slot slot = shm.allocAndRegisterSlot(new ExtendedBlockId(123L, "test_bp1"));
        slots.add(slot);
        numSlots++;
    }
    LOG.info("allocated " + numSlots + " slots before running out.");
    int slotIdx = 0;
    for (Iterator<Slot> iter = shm.slotIterator(); iter.hasNext(); ) {
        Assert.assertTrue(slots.contains(iter.next()));
    }
    for (Slot slot : slots) {
        Assert.assertFalse(slot.addAnchor());
        Assert.assertEquals(slotIdx++, slot.getSlotIdx());
    }
    for (Slot slot : slots) {
        slot.makeAnchorable();
    }
    for (Slot slot : slots) {
        Assert.assertTrue(slot.addAnchor());
    }
    for (Slot slot : slots) {
        slot.removeAnchor();
    }
    for (Slot slot : slots) {
        shm.unregisterSlot(slot.getSlotIdx());
        slot.makeInvalid();
    }
    shm.free();
    stream.close();
    FileUtil.fullyDelete(path);
}

Example 5 with Slot

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot in project hadoop by apache.

the class TestShortCircuitCache method testAllocShm.

@Test(timeout = 60000)
public void testAllocShm() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testAllocShm", sockDir);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty
            Assert.assertEquals(0, info.size());
        }
    });
    DomainPeer peer = getDomainPeerToDn(conf);
    MutableBoolean usedPeer = new MutableBoolean(false);
    ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
    final DatanodeInfo datanode = new DatanodeInfoBuilder().setNodeID(cluster.getDataNodes().get(0).getDatanodeId()).build();
    // Allocating the first shm slot requires using up a peer.
    Slot slot = cache.allocShmSlot(datanode, peer, usedPeer, blockId, "testAllocShm_client");
    Assert.assertNotNull(slot);
    Assert.assertTrue(usedPeer.booleanValue());
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty
            Assert.assertEquals(1, info.size());
            PerDatanodeVisitorInfo vinfo = info.get(datanode);
            Assert.assertFalse(vinfo.disabled);
            Assert.assertEquals(0, vinfo.full.size());
            Assert.assertEquals(1, vinfo.notFull.size());
        }
    });
    cache.scheduleSlotReleaser(slot);
    // Wait for the slot to be released, and the shared memory area to be
    // closed.  Since we didn't register this shared memory segment on the
    // server, it will also be a test of how well the server deals with
    // bogus client behavior.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            final MutableBoolean done = new MutableBoolean(false);
            try {
                cache.getDfsClientShmManager().visit(new Visitor() {

                    @Override
                    public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
                        done.setValue(info.get(datanode).full.isEmpty() && info.get(datanode).notFull.isEmpty());
                    }
                });
            } catch (IOException e) {
                LOG.error("error running visitor", e);
            }
            return done.booleanValue();
        }
    }, 10, 60000);
    cluster.shutdown();
    sockDir.close();
}