Examples with AlreadyBeingCreatedException org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException used on opensource projects

Example 1 with AlreadyBeingCreatedException

use of org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException in project hadoop by apache.

the class FSNamesystem method internalReleaseLease.

/**
   * Move a file that is being written to be immutable.
   * @param src The filename
   * @param lease The lease for the client creating the file
   * @param recoveryLeaseHolder reassign lease to this holder if the last block
   *        needs recovery; keep current holder if null.
   * @throws AlreadyBeingCreatedException if file is waiting to achieve minimal
   *         replication;<br>
   *         RecoveryInProgressException if lease recovery is in progress.<br>
   *         IOException in case of an error.
   * @return true  if file has been successfully finalized and closed or 
   *         false if block recovery has been initiated. Since the lease owner
   *         has been changed and logged, caller should call logSync().
   */
boolean internalReleaseLease(Lease lease, String src, INodesInPath iip, String recoveryLeaseHolder) throws IOException {
    LOG.info("Recovering " + lease + ", src=" + src);
    assert !isInSafeMode();
    assert hasWriteLock();
    final INodeFile pendingFile = iip.getLastINode().asFile();
    int nrBlocks = pendingFile.numBlocks();
    BlockInfo[] blocks = pendingFile.getBlocks();
    int nrCompleteBlocks;
    BlockInfo curBlock = null;
    for (nrCompleteBlocks = 0; nrCompleteBlocks < nrBlocks; nrCompleteBlocks++) {
        curBlock = blocks[nrCompleteBlocks];
        if (!curBlock.isComplete())
            break;
        assert blockManager.hasMinStorage(curBlock) : "A COMPLETE block is not minimally replicated in " + src;
    }
    // then reap lease immediately and close the file.
    if (nrCompleteBlocks == nrBlocks) {
        finalizeINodeFileUnderConstruction(src, pendingFile, iip.getLatestSnapshotId(), false);
        NameNode.stateChangeLog.warn("BLOCK*" + " internalReleaseLease: All existing blocks are COMPLETE," + " lease removed, file " + src + " closed.");
        // closed!
        return true;
    }
    // If the penultimate block is not COMPLETE, then it must be COMMITTED.
    if (nrCompleteBlocks < nrBlocks - 2 || nrCompleteBlocks == nrBlocks - 2 && curBlock != null && curBlock.getBlockUCState() != BlockUCState.COMMITTED) {
        final String message = "DIR* NameSystem.internalReleaseLease: " + "attempt to release a create lock on " + src + " but file is already closed.";
        NameNode.stateChangeLog.warn(message);
        throw new IOException(message);
    }
    // The last block is not COMPLETE, and
    // that the penultimate block if exists is either COMPLETE or COMMITTED
    final BlockInfo lastBlock = pendingFile.getLastBlock();
    BlockUCState lastBlockState = lastBlock.getBlockUCState();
    BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
    // If penultimate block doesn't exist then its minReplication is met
    boolean penultimateBlockMinStorage = penultimateBlock == null || blockManager.hasMinStorage(penultimateBlock);
    switch(lastBlockState) {
        case COMPLETE:
            assert false : "Already checked that the last block is incomplete";
            break;
        case COMMITTED:
            // Close file if committed blocks are minimally replicated
            if (penultimateBlockMinStorage && blockManager.hasMinStorage(lastBlock)) {
                finalizeINodeFileUnderConstruction(src, pendingFile, iip.getLatestSnapshotId(), false);
                NameNode.stateChangeLog.warn("BLOCK*" + " internalReleaseLease: Committed blocks are minimally" + " replicated, lease removed, file" + src + " closed.");
                // closed!
                return true;
            }
            // Cannot close file right now, since some blocks 
            // are not yet minimally replicated.
            // This may potentially cause infinite loop in lease recovery
            // if there are no valid replicas on data-nodes.
            String message = "DIR* NameSystem.internalReleaseLease: " + "Failed to release lease for file " + src + ". Committed blocks are waiting to be minimally replicated." + " Try again later.";
            NameNode.stateChangeLog.warn(message);
            throw new AlreadyBeingCreatedException(message);
        case UNDER_CONSTRUCTION:
        case UNDER_RECOVERY:
            BlockUnderConstructionFeature uc = lastBlock.getUnderConstructionFeature();
            // determine if last block was intended to be truncated
            Block recoveryBlock = uc.getTruncateBlock();
            boolean truncateRecovery = recoveryBlock != null;
            boolean copyOnTruncate = truncateRecovery && recoveryBlock.getBlockId() != lastBlock.getBlockId();
            assert !copyOnTruncate || recoveryBlock.getBlockId() < lastBlock.getBlockId() && recoveryBlock.getGenerationStamp() < lastBlock.getGenerationStamp() && recoveryBlock.getNumBytes() > lastBlock.getNumBytes() : "wrong recoveryBlock";
            // setup the last block locations from the blockManager if not known
            if (uc.getNumExpectedLocations() == 0) {
                uc.setExpectedLocations(lastBlock, blockManager.getStorages(lastBlock), lastBlock.getBlockType());
            }
            if (uc.getNumExpectedLocations() == 0 && lastBlock.getNumBytes() == 0) {
                // There is no datanode reported to this block.
                // may be client have crashed before writing data to pipeline.
                // This blocks doesn't need any recovery.
                // We can remove this block and close the file.
                pendingFile.removeLastBlock(lastBlock);
                finalizeINodeFileUnderConstruction(src, pendingFile, iip.getLatestSnapshotId(), false);
                NameNode.stateChangeLog.warn("BLOCK* internalReleaseLease: " + "Removed empty last block and closed file " + src);
                return true;
            }
            // start recovery of the last block for this file
            long blockRecoveryId = nextGenerationStamp(blockManager.isLegacyBlock(lastBlock));
            lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
            if (copyOnTruncate) {
                lastBlock.setGenerationStamp(blockRecoveryId);
            } else if (truncateRecovery) {
                recoveryBlock.setGenerationStamp(blockRecoveryId);
            }
            uc.initializeBlockRecovery(lastBlock, blockRecoveryId);
            leaseManager.renewLease(lease);
            // Cannot close file right now, since the last block requires recovery.
            // This may potentially cause infinite loop in lease recovery
            // if there are no valid replicas on data-nodes.
            NameNode.stateChangeLog.warn("DIR* NameSystem.internalReleaseLease: " + "File " + src + " has not been closed." + " Lease recovery is in progress. " + "RecoveryId = " + blockRecoveryId + " for block " + lastBlock);
            break;
    }
    return false;
}

Example 2 with AlreadyBeingCreatedException

use of org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException in project hadoop by apache.

the class FSNamesystem method recoverLeaseInternal.

boolean recoverLeaseInternal(RecoverLeaseOp op, INodesInPath iip, String src, String holder, String clientMachine, boolean force) throws IOException {
    assert hasWriteLock();
    INodeFile file = iip.getLastINode().asFile();
    if (file.isUnderConstruction()) {
        //
        // If the file is under construction , then it must be in our
        // leases. Find the appropriate lease record.
        //
        Lease lease = leaseManager.getLease(holder);
        if (!force && lease != null) {
            Lease leaseFile = leaseManager.getLease(file);
            if (leaseFile != null && leaseFile.equals(lease)) {
                // holder is trying to obtain it again.
                throw new AlreadyBeingCreatedException(op.getExceptionMessage(src, holder, clientMachine, holder + " is already the current lease holder."));
            }
        }
        //
        // Find the original holder.
        //
        FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature();
        String clientName = uc.getClientName();
        lease = leaseManager.getLease(clientName);
        if (lease == null) {
            throw new AlreadyBeingCreatedException(op.getExceptionMessage(src, holder, clientMachine, "the file is under construction but no leases found."));
        }
        if (force) {
            // close now: no need to wait for soft lease expiration and 
            // close only the file src
            LOG.info("recoverLease: " + lease + ", src=" + src + " from client " + clientName);
            return internalReleaseLease(lease, src, iip, holder);
        } else {
            assert lease.getHolder().equals(clientName) : "Current lease holder " + lease.getHolder() + " does not match file creator " + clientName;
            //
            if (lease.expiredSoftLimit()) {
                LOG.info("startFile: recover " + lease + ", src=" + src + " client " + clientName);
                if (internalReleaseLease(lease, src, iip, null)) {
                    return true;
                } else {
                    throw new RecoveryInProgressException(op.getExceptionMessage(src, holder, clientMachine, "lease recovery is in progress. Try again later."));
                }
            } else {
                final BlockInfo lastBlock = file.getLastBlock();
                if (lastBlock != null && lastBlock.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
                    throw new RecoveryInProgressException(op.getExceptionMessage(src, holder, clientMachine, "another recovery is in progress by " + clientName + " on " + uc.getClientMachine()));
                } else {
                    throw new AlreadyBeingCreatedException(op.getExceptionMessage(src, holder, clientMachine, "this file lease is currently owned by " + clientName + " on " + uc.getClientMachine()));
                }
            }
        }
    } else {
        return true;
    }
}

Example 3 with AlreadyBeingCreatedException

use of org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException in project hadoop by apache.

the class WriteManager method handleWrite.

void handleWrite(DFSClient dfsClient, WRITE3Request request, Channel channel, int xid, Nfs3FileAttributes preOpAttr) throws IOException {
    int count = request.getCount();
    byte[] data = request.getData().array();
    if (data.length < count) {
        WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_INVAL);
        Nfs3Utils.writeChannel(channel, response.serialize(new XDR(), xid, new VerifierNone()), xid);
        return;
    }
    FileHandle handle = request.getHandle();
    if (LOG.isDebugEnabled()) {
        LOG.debug("handleWrite " + request);
    }
    // Check if there is a stream to write
    FileHandle fileHandle = request.getHandle();
    OpenFileCtx openFileCtx = fileContextCache.get(fileHandle);
    if (openFileCtx == null) {
        LOG.info("No opened stream for fileId: " + fileHandle.getFileId());
        String fileIdPath = Nfs3Utils.getFileIdPath(fileHandle.getFileId());
        HdfsDataOutputStream fos = null;
        Nfs3FileAttributes latestAttr = null;
        try {
            int bufferSize = config.getInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
            fos = dfsClient.append(fileIdPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null);
            latestAttr = Nfs3Utils.getFileAttr(dfsClient, fileIdPath, iug);
        } catch (RemoteException e) {
            IOException io = e.unwrapRemoteException();
            if (io instanceof AlreadyBeingCreatedException) {
                LOG.warn("Can't append file: " + fileIdPath + ". Possibly the file is being closed. Drop the request: " + request + ", wait for the client to retry...");
                return;
            }
            throw e;
        } catch (IOException e) {
            LOG.error("Can't append to file: " + fileIdPath, e);
            if (fos != null) {
                fos.close();
            }
            WccData fileWcc = new WccData(Nfs3Utils.getWccAttr(preOpAttr), preOpAttr);
            WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO, fileWcc, count, request.getStableHow(), Nfs3Constant.WRITE_COMMIT_VERF);
            Nfs3Utils.writeChannel(channel, response.serialize(new XDR(), xid, new VerifierNone()), xid);
            return;
        }
        // Add open stream
        String writeDumpDir = config.get(NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_KEY, NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_DEFAULT);
        openFileCtx = new OpenFileCtx(fos, latestAttr, writeDumpDir + "/" + fileHandle.getFileId(), dfsClient, iug, aixCompatMode, config);
        if (!addOpenFileStream(fileHandle, openFileCtx)) {
            LOG.info("Can't add new stream. Close it. Tell client to retry.");
            try {
                fos.close();
            } catch (IOException e) {
                LOG.error("Can't close stream for fileId: " + handle.getFileId(), e);
            }
            // Notify client to retry
            WccData fileWcc = new WccData(latestAttr.getWccAttr(), latestAttr);
            WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_JUKEBOX, fileWcc, 0, request.getStableHow(), Nfs3Constant.WRITE_COMMIT_VERF);
            Nfs3Utils.writeChannel(channel, response.serialize(new XDR(), xid, new VerifierNone()), xid);
            return;
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Opened stream for appending file: " + fileHandle.getFileId());
        }
    }
    // Add write into the async job queue
    openFileCtx.receivedNewWrite(dfsClient, request, channel, xid, asyncDataService, iug);
    return;
}

Example 4 with AlreadyBeingCreatedException

use of org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException in project hadoop by apache.

the class TestLeaseRecovery2 method recoverLeaseUsingCreate.

private void recoverLeaseUsingCreate(Path filepath) throws IOException, InterruptedException {
    FileSystem dfs2 = getFSAsAnotherUser(conf);
    for (int i = 0; i < 10; i++) {
        AppendTestUtil.LOG.info("i=" + i);
        try {
            dfs2.create(filepath, false, BUF_SIZE, (short) 1, BLOCK_SIZE);
            fail("Creation of an existing file should never succeed.");
        } catch (FileAlreadyExistsException e) {
            // expected
            return;
        } catch (AlreadyBeingCreatedException e) {
            // expected
            return;
        } catch (IOException ioe) {
            AppendTestUtil.LOG.warn("UNEXPECTED ", ioe);
            AppendTestUtil.LOG.info("sleep " + 5000 + "ms");
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
            }
        }
    }
    fail("recoverLeaseUsingCreate failed");
}

Example 5 with AlreadyBeingCreatedException

use of org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException in project storm by apache.

the class TestHdfsSemantics method testAppendSemantics.

@Test
public void testAppendSemantics() throws Exception {
    //1 try to append to an open file
    Path file1 = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1");
    FSDataOutputStream os1 = fs.create(file1, false);
    try {
        // should fail
        fs.append(file1);
        Assert.assertTrue("Append did not throw an exception", false);
    } catch (RemoteException e) {
        // expecting AlreadyBeingCreatedException inside RemoteException
        Assert.assertEquals(AlreadyBeingCreatedException.class, e.unwrapRemoteException().getClass());
    }
    //2 try to append to a closed file
    os1.close();
    // should pass
    FSDataOutputStream os2 = fs.append(file1);
    os2.close();
}