Examples with DiskErrorException org.apache.hadoop.util.DiskChecker.DiskErrorException used on opensource projects

Example 6 with DiskErrorException

use of org.apache.hadoop.util.DiskChecker.DiskErrorException in project hadoop by apache.

the class TestFetcher method testReduceOutOfDiskSpace.

@Test
public void testReduceOutOfDiskSpace() throws Throwable {
    LOG.info("testReduceOutOfDiskSpace");
    Fetcher<Text, Text> underTest = new FakeFetcher<Text, Text>(job, id, ss, mm, r, metrics, except, key, connection);
    String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key);
    ShuffleHeader header = new ShuffleHeader(map1ID.toString(), 10, 10, 1);
    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    header.write(new DataOutputStream(bout));
    ByteArrayInputStream in = new ByteArrayInputStream(bout.toByteArray());
    when(connection.getResponseCode()).thenReturn(200);
    when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_NAME)).thenReturn(ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
    when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_VERSION)).thenReturn(ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
    when(connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH)).thenReturn(replyHash);
    when(connection.getInputStream()).thenReturn(in);
    when(mm.reserve(any(TaskAttemptID.class), anyLong(), anyInt())).thenThrow(new DiskErrorException("No disk space available"));
    underTest.copyFromHost(host);
    verify(ss).reportLocalError(any(IOException.class));
}

Example 7 with DiskErrorException

use of org.apache.hadoop.util.DiskChecker.DiskErrorException in project hadoop by apache.

the class ReadWriteDiskValidator method checkStatus.

@Override
public void checkStatus(File dir) throws DiskErrorException {
    ReadWriteDiskValidatorMetrics metric = ReadWriteDiskValidatorMetrics.getMetric(dir.toString());
    Path tmpFile = null;
    try {
        if (!dir.isDirectory()) {
            metric.diskCheckFailed();
            throw new DiskErrorException(dir + " is not a directory!");
        }
        // check the directory presence and permission.
        DiskChecker.checkDir(dir);
        // create a tmp file under the dir
        tmpFile = Files.createTempFile(dir.toPath(), "test", "tmp");
        // write 16 bytes into the tmp file
        byte[] inputBytes = new byte[16];
        RANDOM.nextBytes(inputBytes);
        long startTime = System.nanoTime();
        Files.write(tmpFile, inputBytes);
        long writeLatency = TimeUnit.MICROSECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
        metric.addWriteFileLatency(writeLatency);
        // read back
        startTime = System.nanoTime();
        byte[] outputBytes = Files.readAllBytes(tmpFile);
        long readLatency = TimeUnit.MICROSECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
        metric.addReadFileLatency(readLatency);
        // validation
        if (!Arrays.equals(inputBytes, outputBytes)) {
            metric.diskCheckFailed();
            throw new DiskErrorException("Data in file has been corrupted.");
        }
    } catch (IOException e) {
        metric.diskCheckFailed();
        throw new DiskErrorException("Disk Check failed!", e);
    } finally {
        // delete the file
        if (tmpFile != null) {
            try {
                Files.delete(tmpFile);
            } catch (IOException e) {
                metric.diskCheckFailed();
                throw new DiskErrorException("File deletion failed!", e);
            }
        }
    }
}

Example 8 with DiskErrorException

use of org.apache.hadoop.util.DiskChecker.DiskErrorException in project hadoop by apache.

the class TestDiskChecker method _checkDirs.

private void _checkDirs(boolean isDir, FsPermission perm, boolean success) throws Throwable {
    File localDir = isDir ? createTempDir() : createTempFile();
    Shell.execCommand(Shell.getSetPermissionCommand(String.format("%04o", perm.toShort()), false, localDir.getAbsolutePath()));
    try {
        DiskChecker.checkDir(FileSystem.getLocal(new Configuration()), new Path(localDir.getAbsolutePath()), perm);
        assertTrue("checkDir success, expected failure", success);
    } catch (DiskErrorException e) {
        if (success) {
            // Unexpected exception!
            throw e;
        }
    }
    localDir.delete();
}

Example 9 with DiskErrorException

use of org.apache.hadoop.util.DiskChecker.DiskErrorException in project hadoop by apache.

the class DataNode method startDataNode.

/**
   * This method starts the data node with the specified conf.
   * 
   * If conf's CONFIG_PROPERTY_SIMULATED property is set
   * then a simulated storage based data node is created.
   * 
   * @param dataDirectories - only for a non-simulated storage data node
   * @throws IOException
   */
void startDataNode(List<StorageLocation> dataDirectories, SecureResources resources) throws IOException {
    // settings global for all BPs in the Data Node
    this.secureResources = resources;
    synchronized (this) {
        this.dataDirs = dataDirectories;
    }
    this.dnConf = new DNConf(this);
    checkSecureConfig(dnConf, getConf(), resources);
    if (dnConf.maxLockedMemory > 0) {
        if (!NativeIO.POSIX.getCacheManipulator().verifyCanMlock()) {
            throw new RuntimeException(String.format("Cannot start datanode because the configured max locked memory" + " size (%s) is greater than zero and native code is not available.", DFS_DATANODE_MAX_LOCKED_MEMORY_KEY));
        }
        if (Path.WINDOWS) {
            NativeIO.Windows.extendWorkingSetSize(dnConf.maxLockedMemory);
        } else {
            long ulimit = NativeIO.POSIX.getCacheManipulator().getMemlockLimit();
            if (dnConf.maxLockedMemory > ulimit) {
                throw new RuntimeException(String.format("Cannot start datanode because the configured max locked memory" + " size (%s) of %d bytes is more than the datanode's available" + " RLIMIT_MEMLOCK ulimit of %d bytes.", DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, dnConf.maxLockedMemory, ulimit));
            }
        }
    }
    LOG.info("Starting DataNode with maxLockedMemory = " + dnConf.maxLockedMemory);
    int volFailuresTolerated = dnConf.getVolFailuresTolerated();
    int volsConfigured = dnConf.getVolsConfigured();
    if (volFailuresTolerated < 0 || volFailuresTolerated >= volsConfigured) {
        throw new DiskErrorException("Invalid value configured for " + "dfs.datanode.failed.volumes.tolerated - " + volFailuresTolerated + ". Value configured is either less than 0 or >= " + "to the number of configured volumes (" + volsConfigured + ").");
    }
    storage = new DataStorage();
    // global DN settings
    registerMXBean();
    initDataXceiver();
    startInfoServer();
    pauseMonitor = new JvmPauseMonitor();
    pauseMonitor.init(getConf());
    pauseMonitor.start();
    // BlockPoolTokenSecretManager is required to create ipc server.
    this.blockPoolTokenSecretManager = new BlockPoolTokenSecretManager();
    // Login is done by now. Set the DN user name.
    dnUserName = UserGroupInformation.getCurrentUser().getUserName();
    LOG.info("dnUserName = " + dnUserName);
    LOG.info("supergroup = " + supergroup);
    initIpcServer();
    metrics = DataNodeMetrics.create(getConf(), getDisplayName());
    peerMetrics = dnConf.peerStatsEnabled ? DataNodePeerMetrics.create(getConf(), getDisplayName()) : null;
    metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
    ecWorker = new ErasureCodingWorker(getConf(), this);
    blockRecoveryWorker = new BlockRecoveryWorker(this);
    blockPoolManager = new BlockPoolManager(this);
    blockPoolManager.refreshNamenodes(getConf());
    // Create the ReadaheadPool from the DataNode context so we can
    // exit without having to explicitly shutdown its thread pool.
    readaheadPool = ReadaheadPool.getInstance();
    saslClient = new SaslDataTransferClient(dnConf.getConf(), dnConf.saslPropsResolver, dnConf.trustedChannelResolver);
    saslServer = new SaslDataTransferServer(dnConf, blockPoolTokenSecretManager);
    startMetricsLogger();
    if (dnConf.diskStatsEnabled) {
        diskMetrics = new DataNodeDiskMetrics(this, dnConf.outliersReportIntervalMs);
    }
}

Example 10 with DiskErrorException

use of org.apache.hadoop.util.DiskChecker.DiskErrorException in project hadoop by apache.

the class TestDatasetVolumeChecker method makeVolumes.

static List<FsVolumeSpi> makeVolumes(int numVolumes, VolumeCheckResult health) throws Exception {
    final List<FsVolumeSpi> volumes = new ArrayList<>(numVolumes);
    for (int i = 0; i < numVolumes; ++i) {
        final FsVolumeSpi volume = mock(FsVolumeSpi.class);
        final FsVolumeReference reference = mock(FsVolumeReference.class);
        final StorageLocation location = mock(StorageLocation.class);
        when(reference.getVolume()).thenReturn(volume);
        when(volume.obtainReference()).thenReturn(reference);
        when(volume.getStorageLocation()).thenReturn(location);
        if (health != null) {
            when(volume.check(anyObject())).thenReturn(health);
        } else {
            final DiskErrorException de = new DiskErrorException("Fake Exception");
            when(volume.check(anyObject())).thenThrow(de);
        }
        volumes.add(volume);
    }
    return volumes;
}