Examples with SnapshotDescription org.apache.hadoop.hbase.client.SnapshotDescription used on opensource projects

Example 6 with SnapshotDescription

use of org.apache.hadoop.hbase.client.SnapshotDescription in project hbase by apache.

the class TestFlushSnapshotFromClient method testAsyncFlushSnapshot.

@Test
public void testAsyncFlushSnapshot() throws Exception {
    HBaseProtos.SnapshotDescription snapshot = HBaseProtos.SnapshotDescription.newBuilder().setName("asyncSnapshot").setTable(TABLE_NAME.getNameAsString()).setType(HBaseProtos.SnapshotDescription.Type.FLUSH).build();
    // take the snapshot async
    admin.takeSnapshotAsync(new SnapshotDescription("asyncSnapshot", TABLE_NAME, SnapshotType.FLUSH));
    // constantly loop, looking for the snapshot to complete
    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
    SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
    LOG.info(" === Async Snapshot Completed ===");
    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
    // make sure we get the snapshot
    SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
}

Example 7 with SnapshotDescription

use of org.apache.hadoop.hbase.client.SnapshotDescription in project hbase by apache.

the class TestFlushSnapshotFromClient method testConcurrentSnapshottingAttempts.

/**
   * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
   * same table currently running and that concurrent snapshots on different tables can both
   * succeed concurretly.
   */
@Test
public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
    final TableName TABLE2_NAME = TableName.valueOf(TABLE_NAME + "2");
    int ssNum = 20;
    // make sure we don't fail on listing snapshots
    SnapshotTestingUtils.assertNoSnapshots(admin);
    // create second testing table
    SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
    // load the table so we have some data
    SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
    SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
    final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
    // We'll have one of these per thread
    class SSRunnable implements Runnable {

        SnapshotDescription ss;

        SSRunnable(SnapshotDescription ss) {
            this.ss = ss;
        }

        @Override
        public void run() {
            try {
                LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ProtobufUtil.createHBaseProtosSnapshotDesc(ss)));
                admin.takeSnapshotAsync(ss);
            } catch (Exception e) {
                LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(ProtobufUtil.createHBaseProtosSnapshotDesc(ss)) + ".  This is ok, we expect some", e);
            }
            LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ProtobufUtil.createHBaseProtosSnapshotDesc(ss)));
            toBeSubmitted.countDown();
        }
    }
    ;
    // build descriptions
    SnapshotDescription[] descs = new SnapshotDescription[ssNum];
    for (int i = 0; i < ssNum; i++) {
        if (i % 2 == 0) {
            descs[i] = new SnapshotDescription("ss" + i, TABLE_NAME, SnapshotType.FLUSH);
        } else {
            descs[i] = new SnapshotDescription("ss" + i, TABLE2_NAME, SnapshotType.FLUSH);
        }
    }
    // kick each off its own thread
    for (int i = 0; i < ssNum; i++) {
        new Thread(new SSRunnable(descs[i])).start();
    }
    // wait until all have been submitted
    toBeSubmitted.await();
    // loop until all are done.
    while (true) {
        int doneCount = 0;
        for (SnapshotDescription ss : descs) {
            try {
                if (admin.isSnapshotFinished(ss)) {
                    doneCount++;
                }
            } catch (Exception e) {
                LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
                doneCount++;
            }
        }
        if (doneCount == descs.length) {
            break;
        }
        Thread.sleep(100);
    }
    // dump for debugging
    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
    List<SnapshotDescription> taken = admin.listSnapshots();
    int takenSize = taken.size();
    LOG.info("Taken " + takenSize + " snapshots:  " + taken);
    assertTrue("We expect at least 1 request to be rejected because of we concurrently" + " issued many requests", takenSize < ssNum && takenSize > 0);
    // Verify that there's at least one snapshot per table
    int t1SnapshotsCount = 0;
    int t2SnapshotsCount = 0;
    for (SnapshotDescription ss : taken) {
        if (ss.getTableName().equals(TABLE_NAME)) {
            t1SnapshotsCount++;
        } else if (ss.getTableName().equals(TABLE2_NAME)) {
            t2SnapshotsCount++;
        }
    }
    assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
    assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
    UTIL.deleteTable(TABLE2_NAME);
}

Example 8 with SnapshotDescription

use of org.apache.hadoop.hbase.client.SnapshotDescription in project hbase by apache.

the class SnapshotInfo method printFiles.

/**
   * Collect the hfiles and logs statistics of the snapshot and
   * dump the file list if requested and the collected information.
   */
private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
    if (showFiles) {
        System.out.println("Snapshot Files");
        System.out.println("----------------------------------------");
    }
    // Collect information about hfiles and logs in the snapshot
    final HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
    final String table = snapshotDesc.getTable();
    final SnapshotDescription desc = ProtobufUtil.createSnapshotDesc(snapshotDesc);
    final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc);
    SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest, "SnapshotInfo", new SnapshotReferenceUtil.SnapshotVisitor() {

        @Override
        public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
            if (storeFile.hasReference())
                return;
            SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile, null);
            if (showFiles) {
                String state = info.getStateToString();
                System.out.printf("%8s %s/%s/%s/%s %s%n", (info.isMissing() ? "-" : fileSizeToString(info.getSize())), table, regionInfo.getEncodedName(), family, storeFile.getName(), state == null ? "" : "(" + state + ")");
            }
        }
    });
    // Dump the stats
    System.out.println();
    if (stats.isSnapshotCorrupted()) {
        System.out.println("**************************************************************");
        System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n", stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
        System.out.printf("              %d hfile(s) corrupted.%n", stats.getCorruptedStoreFilesCount());
        System.out.println("**************************************************************");
    }
    if (showStats) {
        System.out.printf("%d HFiles (%d in archive, %d in mob storage), total size %s " + "(%.2f%% %s shared with the source table, %.2f%% %s in mob dir)%n", stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(), stats.getMobStoreFilesCount(), fileSizeToString(stats.getStoreFilesSize()), stats.getSharedStoreFilePercentage(), fileSizeToString(stats.getSharedStoreFilesSize()), stats.getMobStoreFilePercentage(), fileSizeToString(stats.getMobStoreFilesSize()));
        System.out.printf("%d Logs, total size %s%n", stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
        System.out.println();
    }
}

Example 9 with SnapshotDescription

use of org.apache.hadoop.hbase.client.SnapshotDescription in project hbase by apache.

the class TestRestoreSnapshotProcedure method setupSnapshotAndUpdateTable.

private void setupSnapshotAndUpdateTable() throws Exception {
    long tid = System.currentTimeMillis();
    final byte[] snapshotName = Bytes.toBytes("snapshot-" + tid);
    Admin admin = UTIL.getAdmin();
    // create Table
    SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF1, CF2);
    // Load data
    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF1, CF1);
    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF2, CF2);
    SnapshotTestingUtils.verifyRowCount(UTIL, snapshotTableName, rowCountCF1 + rowCountCF2);
    snapshotHTD = admin.getTableDescriptor(snapshotTableName);
    admin.disableTable(snapshotTableName);
    // take a snapshot
    admin.snapshot(snapshotName, snapshotTableName);
    List<SnapshotDescription> snapshotList = admin.listSnapshots();
    snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
    // modify the table
    HColumnDescriptor columnFamilyDescriptor3 = new HColumnDescriptor(CF3);
    HColumnDescriptor columnFamilyDescriptor4 = new HColumnDescriptor(CF4);
    admin.addColumnFamily(snapshotTableName, columnFamilyDescriptor3);
    admin.addColumnFamily(snapshotTableName, columnFamilyDescriptor4);
    admin.deleteColumnFamily(snapshotTableName, CF2);
    // enable table and insert data
    admin.enableTable(snapshotTableName);
    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF3, CF3);
    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF4, CF4);
    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF1addition, CF1);
    HTableDescriptor currentHTD = admin.getTableDescriptor(snapshotTableName);
    assertTrue(currentHTD.hasFamily(CF1));
    assertFalse(currentHTD.hasFamily(CF2));
    assertTrue(currentHTD.hasFamily(CF3));
    assertTrue(currentHTD.hasFamily(CF4));
    assertNotEquals(currentHTD.getFamiliesKeys().size(), snapshotHTD.getFamiliesKeys().size());
    SnapshotTestingUtils.verifyRowCount(UTIL, snapshotTableName, rowCountCF1 + rowCountCF3 + rowCountCF4 + rowCountCF1addition);
    admin.disableTable(snapshotTableName);
}

Example 10 with SnapshotDescription

use of org.apache.hadoop.hbase.client.SnapshotDescription in project hbase by apache.

the class TestCloneSnapshotProcedure method getSnapshot.

private HBaseProtos.SnapshotDescription getSnapshot() throws Exception {
    if (snapshot == null) {
        final TableName snapshotTableName = TableName.valueOf("testCloneSnapshot");
        long tid = System.currentTimeMillis();
        final byte[] snapshotName = Bytes.toBytes("snapshot-" + tid);
        Admin admin = UTIL.getAdmin();
        // create Table
        SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF);
        // Load data
        SnapshotTestingUtils.loadData(UTIL, snapshotTableName, 500, CF);
        admin.disableTable(snapshotTableName);
        // take a snapshot
        admin.snapshot(snapshotName, snapshotTableName);
        admin.enableTable(snapshotTableName);
        List<SnapshotDescription> snapshotList = admin.listSnapshots();
        snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
    }
    return snapshot;
}