Examples with CorruptHFileException org.apache.hadoop.hbase.io.hfile.CorruptHFileException used on opensource projects

Example 1 with CorruptHFileException

use of org.apache.hadoop.hbase.io.hfile.CorruptHFileException in project hbase by apache.

the class HMobStore method readCell.

/**
 * Reads the cell from a mob file.
 * The mob file might be located in different directories.
 * 1. The working directory.
 * 2. The archive directory.
 * Reads the cell from the files located in both of the above directories.
 * @param locations The possible locations where the mob files are saved.
 * @param fileName The file to be read.
 * @param search The cell to be searched.
 * @param cacheMobBlocks Whether the scanner should cache blocks.
 * @param readPt the read point.
 * @param readEmptyValueOnMobCellMiss Whether return null value when the mob file is
 *        missing or corrupt.
 * @return The found cell. Null if there's no such a cell.
 * @throws IOException
 */
private MobCell readCell(List<Path> locations, String fileName, Cell search, boolean cacheMobBlocks, long readPt, boolean readEmptyValueOnMobCellMiss) throws IOException {
    FileSystem fs = getFileSystem();
    Throwable throwable = null;
    for (Path location : locations) {
        MobFile file = null;
        Path path = new Path(location, fileName);
        try {
            file = mobFileCache.openFile(fs, path, getCacheConfig());
            return readPt != -1 ? file.readCell(search, cacheMobBlocks, readPt) : file.readCell(search, cacheMobBlocks);
        } catch (IOException e) {
            mobFileCache.evictFile(fileName);
            throwable = e;
            if ((e instanceof FileNotFoundException) || (e.getCause() instanceof FileNotFoundException)) {
                LOG.debug("Fail to read the cell, the mob file " + path + " doesn't exist", e);
            } else if (e instanceof CorruptHFileException) {
                LOG.error("The mob file " + path + " is corrupt", e);
                break;
            } else {
                throw e;
            }
        } catch (NullPointerException e) {
            // HDFS 1.x - DFSInputStream.getBlockAt()
            mobFileCache.evictFile(fileName);
            LOG.debug("Fail to read the cell", e);
            throwable = e;
        } catch (AssertionError e) {
            // assert in HDFS 1.x - DFSInputStream.getBlockAt()
            mobFileCache.evictFile(fileName);
            LOG.debug("Fail to read the cell", e);
            throwable = e;
        } finally {
            if (file != null) {
                mobFileCache.closeFile(file);
            }
        }
    }
    LOG.error("The mob file " + fileName + " could not be found in the locations " + locations + " or it is corrupt");
    if (readEmptyValueOnMobCellMiss) {
        return null;
    } else if ((throwable instanceof FileNotFoundException) || (throwable.getCause() instanceof FileNotFoundException)) {
        // doesn't help fix the lost MOB files.
        throw new DoNotRetryIOException(throwable);
    } else if (throwable instanceof IOException) {
        throw (IOException) throwable;
    } else {
        throw new IOException(throwable);
    }
}

Example 2 with CorruptHFileException

use of org.apache.hadoop.hbase.io.hfile.CorruptHFileException in project hbase by apache.

the class FaultyMobStoreCompactor method performCompaction.

@Override
protected boolean performCompaction(FileDetails fd, InternalScanner scanner, long smallestReadPoint, boolean cleanSeqId, ThroughputController throughputController, boolean major, int numofFilesToCompact) throws IOException {
    totalCompactions.incrementAndGet();
    if (major) {
        totalMajorCompactions.incrementAndGet();
    }
    long bytesWrittenProgressForLog = 0;
    long bytesWrittenProgressForShippedCall = 0;
    // Clear old mob references
    mobRefSet.get().clear();
    boolean isUserRequest = userRequest.get();
    boolean compactMOBs = major && isUserRequest;
    boolean discardMobMiss = conf.getBoolean(MobConstants.MOB_UNSAFE_DISCARD_MISS_KEY, MobConstants.DEFAULT_MOB_DISCARD_MISS);
    boolean mustFail = false;
    if (compactMOBs) {
        mobCounter.incrementAndGet();
        double dv = rnd.nextDouble();
        if (dv < failureProb) {
            mustFail = true;
            totalFailures.incrementAndGet();
        }
    }
    FileSystem fs = store.getFileSystem();
    // Since scanner.next() can return 'false' but still be delivering data,
    // we have to use a do/while loop.
    List<Cell> cells = new ArrayList<>();
    // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
    long currentTime = EnvironmentEdgeManager.currentTime();
    long lastMillis = 0;
    if (LOG.isDebugEnabled()) {
        lastMillis = currentTime;
    }
    CloseChecker closeChecker = new CloseChecker(conf, currentTime);
    String compactionName = ThroughputControlUtil.getNameForThrottling(store, "compaction");
    long now = 0;
    boolean hasMore;
    Path path = MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName());
    byte[] fileName = null;
    StoreFileWriter mobFileWriter = null;
    long mobCells = 0;
    long cellsCountCompactedToMob = 0, cellsCountCompactedFromMob = 0;
    long cellsSizeCompactedToMob = 0, cellsSizeCompactedFromMob = 0;
    boolean finished = false;
    ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
    throughputController.start(compactionName);
    KeyValueScanner kvs = (scanner instanceof KeyValueScanner) ? (KeyValueScanner) scanner : null;
    long shippedCallSizeLimit = (long) numofFilesToCompact * this.store.getColumnFamilyDescriptor().getBlocksize();
    Cell mobCell = null;
    long counter = 0;
    long countFailAt = -1;
    if (mustFail) {
        // randomly fail fast
        countFailAt = rnd.nextInt(100);
    }
    try {
        try {
            mobFileWriter = mobStore.createWriterInTmp(new Date(fd.latestPutTs), fd.maxKeyCount, major ? majorCompactionCompression : minorCompactionCompression, store.getRegionInfo().getStartKey(), true);
            fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
        } catch (IOException e) {
            // Bailing out
            LOG.error("Failed to create mob writer, ", e);
            throw e;
        }
        if (compactMOBs) {
            // Add the only reference we get for compact MOB case
            // because new store file will have only one MOB reference
            // in this case - of newly compacted MOB file
            mobRefSet.get().put(store.getTableName(), mobFileWriter.getPath().getName());
        }
        do {
            hasMore = scanner.next(cells, scannerContext);
            currentTime = EnvironmentEdgeManager.currentTime();
            if (LOG.isDebugEnabled()) {
                now = currentTime;
            }
            if (closeChecker.isTimeLimit(store, currentTime)) {
                progress.cancel();
                return false;
            }
            for (Cell c : cells) {
                counter++;
                if (compactMOBs) {
                    if (MobUtils.isMobReferenceCell(c)) {
                        if (counter == countFailAt) {
                            LOG.warn("INJECTED FAULT mobCounter={}", mobCounter.get());
                            throw new CorruptHFileException("injected fault");
                        }
                        String fName = MobUtils.getMobFileName(c);
                        // Added to support migration
                        try {
                            mobCell = mobStore.resolve(c, true, false).getCell();
                        } catch (DoNotRetryIOException e) {
                            if (discardMobMiss && e.getCause() != null && e.getCause() instanceof FileNotFoundException) {
                                LOG.error("Missing MOB cell: file={} not found cell={}", fName, c);
                                continue;
                            } else {
                                throw e;
                            }
                        }
                        if (discardMobMiss && mobCell.getValueLength() == 0) {
                            LOG.error("Missing MOB cell value: file={} cell={}", fName, mobCell);
                            continue;
                        }
                        if (mobCell.getValueLength() > mobSizeThreshold) {
                            // put the mob data back to the store file
                            PrivateCellUtil.setSequenceId(mobCell, c.getSequenceId());
                            mobFileWriter.append(mobCell);
                            writer.append(MobUtils.createMobRefCell(mobCell, fileName, this.mobStore.getRefCellTags()));
                            mobCells++;
                        } else {
                            // If MOB value is less than threshold, append it directly to a store file
                            PrivateCellUtil.setSequenceId(mobCell, c.getSequenceId());
                            writer.append(mobCell);
                            cellsCountCompactedFromMob++;
                            cellsSizeCompactedFromMob += mobCell.getValueLength();
                        }
                    } else {
                        // Not a MOB reference cell
                        int size = c.getValueLength();
                        if (size > mobSizeThreshold) {
                            mobFileWriter.append(c);
                            writer.append(MobUtils.createMobRefCell(c, fileName, this.mobStore.getRefCellTags()));
                            mobCells++;
                            cellsCountCompactedToMob++;
                            cellsSizeCompactedToMob += c.getValueLength();
                        } else {
                            writer.append(c);
                        }
                    }
                } else if (c.getTypeByte() != KeyValue.Type.Put.getCode()) {
                    // Not a major compaction or major with MOB disabled
                    // If the kv type is not put, directly write the cell
                    // to the store file.
                    writer.append(c);
                } else if (MobUtils.isMobReferenceCell(c)) {
                    // Not a major MOB compaction, Put MOB reference
                    if (MobUtils.hasValidMobRefCellValue(c)) {
                        int size = MobUtils.getMobValueLength(c);
                        if (size > mobSizeThreshold) {
                            // If the value size is larger than the threshold, it's regarded as a mob. Since
                            // its value is already in the mob file, directly write this cell to the store file
                            Optional<TableName> refTable = MobUtils.getTableName(c);
                            if (refTable.isPresent()) {
                                mobRefSet.get().put(refTable.get(), MobUtils.getMobFileName(c));
                                writer.append(c);
                            } else {
                                throw new IOException(String.format("MOB cell did not contain a tablename " + "tag. should not be possible. see ref guide on mob troubleshooting. " + "store=%s cell=%s", getStoreInfo(), c));
                            }
                        } else {
                            // If the value is not larger than the threshold, it's not regarded a mob. Retrieve
                            // the mob cell from the mob file, and write it back to the store file.
                            mobCell = mobStore.resolve(c, true, false).getCell();
                            if (mobCell.getValueLength() != 0) {
                                // put the mob data back to the store file
                                PrivateCellUtil.setSequenceId(mobCell, c.getSequenceId());
                                writer.append(mobCell);
                                cellsCountCompactedFromMob++;
                                cellsSizeCompactedFromMob += mobCell.getValueLength();
                            } else {
                                // If the value of a file is empty, there might be issues when retrieving,
                                // directly write the cell to the store file, and leave it to be handled by the
                                // next compaction.
                                LOG.error("Empty value for: " + c);
                                Optional<TableName> refTable = MobUtils.getTableName(c);
                                if (refTable.isPresent()) {
                                    mobRefSet.get().put(refTable.get(), MobUtils.getMobFileName(c));
                                    writer.append(c);
                                } else {
                                    throw new IOException(String.format("MOB cell did not contain a tablename " + "tag. should not be possible. see ref guide on mob troubleshooting. " + "store=%s cell=%s", getStoreInfo(), c));
                                }
                            }
                        }
                    } else {
                        LOG.error("Corrupted MOB reference: {}", c);
                        writer.append(c);
                    }
                } else if (c.getValueLength() <= mobSizeThreshold) {
                    // If the value size of a cell is not larger than the threshold, directly write it to
                    // the store file.
                    writer.append(c);
                } else {
                    // If the value size of a cell is larger than the threshold, it's regarded as a mob,
                    // write this cell to a mob file, and write the path to the store file.
                    mobCells++;
                    // append the original keyValue in the mob file.
                    mobFileWriter.append(c);
                    Cell reference = MobUtils.createMobRefCell(c, fileName, this.mobStore.getRefCellTags());
                    // write the cell whose value is the path of a mob file to the store file.
                    writer.append(reference);
                    cellsCountCompactedToMob++;
                    cellsSizeCompactedToMob += c.getValueLength();
                    // Add ref we get for compact MOB case
                    mobRefSet.get().put(store.getTableName(), mobFileWriter.getPath().getName());
                }
                int len = c.getSerializedSize();
                ++progress.currentCompactedKVs;
                progress.totalCompactedSize += len;
                bytesWrittenProgressForShippedCall += len;
                if (LOG.isDebugEnabled()) {
                    bytesWrittenProgressForLog += len;
                }
                throughputController.control(compactionName, len);
                if (closeChecker.isSizeLimit(store, len)) {
                    progress.cancel();
                    return false;
                }
                if (kvs != null && bytesWrittenProgressForShippedCall > shippedCallSizeLimit) {
                    ((ShipperListener) writer).beforeShipped();
                    kvs.shipped();
                    bytesWrittenProgressForShippedCall = 0;
                }
            }
            // logging at DEBUG level
            if (LOG.isDebugEnabled()) {
                if ((now - lastMillis) >= COMPACTION_PROGRESS_LOG_INTERVAL) {
                    String rate = String.format("%.2f", (bytesWrittenProgressForLog / 1024.0) / ((now - lastMillis) / 1000.0));
                    LOG.debug("Compaction progress: {} {}, rate={} KB/sec, throughputController is {}", compactionName, progress, rate, throughputController);
                    lastMillis = now;
                    bytesWrittenProgressForLog = 0;
                }
            }
            cells.clear();
        } while (hasMore);
        finished = true;
    } catch (InterruptedException e) {
        progress.cancel();
        throw new InterruptedIOException("Interrupted while control throughput of compacting " + compactionName);
    } catch (FileNotFoundException e) {
        LOG.error("MOB Stress Test FAILED, region: " + store.getRegionInfo().getEncodedName(), e);
        System.exit(-1);
    } catch (IOException t) {
        LOG.error("Mob compaction failed for region: " + store.getRegionInfo().getEncodedName());
        throw t;
    } finally {
        // Clone last cell in the final because writer will append last cell when committing. If
        // don't clone here and once the scanner get closed, then the memory of last cell will be
        // released. (HBASE-22582)
        ((ShipperListener) writer).beforeShipped();
        throughputController.finish(compactionName);
        if (!finished && mobFileWriter != null) {
            // Remove all MOB references because compaction failed
            mobRefSet.get().clear();
            // Abort writer
            abortWriter(mobFileWriter);
        }
    }
    if (mobFileWriter != null) {
        if (mobCells > 0) {
            // If the mob file is not empty, commit it.
            mobFileWriter.appendMetadata(fd.maxSeqId, major, mobCells);
            mobFileWriter.close();
            mobStore.commitFile(mobFileWriter.getPath(), path);
        } else {
            // If the mob file is empty, delete it instead of committing.
            abortWriter(mobFileWriter);
        }
    }
    mobStore.updateCellsCountCompactedFromMob(cellsCountCompactedFromMob);
    mobStore.updateCellsCountCompactedToMob(cellsCountCompactedToMob);
    mobStore.updateCellsSizeCompactedFromMob(cellsSizeCompactedFromMob);
    mobStore.updateCellsSizeCompactedToMob(cellsSizeCompactedToMob);
    progress.complete();
    return true;
}

Example 3 with CorruptHFileException

use of org.apache.hadoop.hbase.io.hfile.CorruptHFileException in project hbase by apache.

the class TestWALSplitToHFile method testCorruptRecoveredHFile.

@Test
public void testCorruptRecoveredHFile() throws Exception {
    Pair<TableDescriptor, RegionInfo> pair = setupTableAndRegion();
    TableDescriptor td = pair.getFirst();
    RegionInfo ri = pair.getSecond();
    WAL wal = createWAL(this.conf, rootDir, logName);
    HRegion region = HRegion.openHRegion(this.conf, this.fs, rootDir, ri, td, wal);
    writeData(td, region);
    // Now close the region without flush
    region.close(true);
    wal.shutdown();
    // split the log
    WALSplitter.split(rootDir, logDir, oldLogDir, FileSystem.get(this.conf), this.conf, wals);
    // Write a corrupt recovered hfile
    Path regionDir = new Path(CommonFSUtils.getTableDir(rootDir, td.getTableName()), ri.getEncodedName());
    for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) {
        FileStatus[] files = WALSplitUtil.getRecoveredHFiles(this.fs, regionDir, cfd.getNameAsString());
        assertNotNull(files);
        assertTrue(files.length > 0);
        writeCorruptRecoveredHFile(files[0].getPath());
    }
    // Failed to reopen the region
    WAL wal2 = createWAL(this.conf, rootDir, logName);
    try {
        HRegion.openHRegion(this.conf, this.fs, rootDir, ri, td, wal2);
        fail("Should fail to open region");
    } catch (CorruptHFileException che) {
    // Expected
    }
    // Set skip errors to true and reopen the region
    this.conf.setBoolean(HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS, true);
    HRegion region2 = HRegion.openHRegion(this.conf, this.fs, rootDir, ri, td, wal2);
    Result result2 = region2.get(new Get(ROW));
    assertEquals(td.getColumnFamilies().length, result2.size());
    for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) {
        assertTrue(Bytes.equals(VALUE1, result2.getValue(cfd.getName(), QUALIFIER)));
        // Assert the corrupt file was skipped and still exist
        FileStatus[] files = WALSplitUtil.getRecoveredHFiles(this.fs, regionDir, cfd.getNameAsString());
        assertNotNull(files);
        assertEquals(1, files.length);
        assertTrue(files[0].getPath().getName().contains("corrupt"));
    }
}