Examples with SnapshotException org.apache.hadoop.hdfs.protocol.SnapshotException used on opensource projects

Example 6 with SnapshotException

use of org.apache.hadoop.hdfs.protocol.SnapshotException in project hive by apache.

the class ReplDumpTask method execute.

@Override
public int execute() {
    try {
        SecurityUtils.reloginExpiringKeytabUser();
        if (work.dataCopyIteratorsInitialized()) {
            initiateDataCopyTasks();
        } else {
            Path dumpRoot = ReplUtils.getEncodedDumpRootPath(conf, work.dbNameOrPattern.toLowerCase());
            if (ReplUtils.failedWithNonRecoverableError(ReplUtils.getLatestDumpPath(dumpRoot, conf), conf)) {
                LOG.error("Previous dump failed with non recoverable error. Needs manual intervention. ");
                setException(new SemanticException(ErrorMsg.REPL_FAILED_WITH_NON_RECOVERABLE_ERROR.format()));
                return ErrorMsg.REPL_FAILED_WITH_NON_RECOVERABLE_ERROR.getErrorCode();
            }
            Path previousValidHiveDumpPath = getPreviousValidDumpMetadataPath(dumpRoot);
            boolean isFailoverMarkerPresent = false;
            boolean isFailover = isFailover(work.dbNameOrPattern, getHive());
            LOG.debug("Database is {} going through failover", isFailover ? "" : "not");
            if (previousValidHiveDumpPath == null && !isFailover) {
                work.setBootstrap(true);
            } else {
                work.setOldReplScope(isFailover ? null : new DumpMetaData(previousValidHiveDumpPath, conf).getReplScope());
                isFailoverMarkerPresent = !isFailover && isDumpFailoverReady(previousValidHiveDumpPath);
            }
            // 2. Previous dump is already loaded and it is not in failover ready status.
            if (shouldDump(previousValidHiveDumpPath, isFailoverMarkerPresent, isFailover)) {
                Path currentDumpPath = getCurrentDumpPath(dumpRoot, work.isBootstrap());
                Path hiveDumpRoot = new Path(currentDumpPath, ReplUtils.REPL_HIVE_BASE_DIR);
                if (!work.isBootstrap() && !isFailover) {
                    preProcessFailoverIfRequired(previousValidHiveDumpPath, isFailoverMarkerPresent);
                }
                // Set distCp custom name corresponding to the replication policy.
                String mapRedCustomName = ReplUtils.getDistCpCustomName(conf, work.dbNameOrPattern);
                conf.set(JobContext.JOB_NAME, mapRedCustomName);
                work.setCurrentDumpPath(currentDumpPath);
                work.setMetricCollector(initMetricCollection(work.isBootstrap(), hiveDumpRoot));
                if (shouldDumpAtlasMetadata()) {
                    addAtlasDumpTask(work.isBootstrap(), previousValidHiveDumpPath);
                    LOG.info("Added task to dump atlas metadata.");
                }
                if (shouldDumpAuthorizationMetadata()) {
                    initiateAuthorizationDumpTask();
                }
                DumpMetaData dmd = new DumpMetaData(hiveDumpRoot, conf);
                // Initialize ReplChangeManager instance since we will require it to encode file URI.
                ReplChangeManager.getInstance(conf);
                Path cmRoot = new Path(conf.getVar(HiveConf.ConfVars.REPLCMDIR));
                Long lastReplId;
                LOG.info("Data copy at load enabled : {}", conf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET));
                if (isFailover) {
                    if (createEventMarker) {
                        LOG.info("Optimised Bootstrap Dump triggered for {}.", work.dbNameOrPattern);
                        // Before starting optimised bootstrap, check if the first incremental is done to ensure database is in
                        // consistent state.
                        isFirstIncrementalPending(work.dbNameOrPattern, getHive());
                        // Get the last replicated event id from the database.
                        String dbEventId = getReplEventIdFromDatabase(work.dbNameOrPattern, getHive());
                        // Get the last replicated event id from the database with respect to target.
                        String targetDbEventId = getTargetEventId(work.dbNameOrPattern, getHive());
                        // Check if the tableDiff directory is present or not.
                        boolean isTableDiffDirectoryPresent = checkFileExists(currentDumpPath, conf, TABLE_DIFF_COMPLETE_DIRECTORY);
                        LOG.info("Creating event_ack file for database {} with event id {}.", work.dbNameOrPattern, dbEventId);
                        lastReplId = createAndGetEventAckFile(currentDumpPath, dmd, cmRoot, dbEventId, targetDbEventId, conf, work);
                        finishRemainingTasks();
                    } else {
                        // We should be here only if TableDiff is Present.
                        boolean isTableDiffDirectoryPresent = checkFileExists(previousValidHiveDumpPath.getParent(), conf, TABLE_DIFF_COMPLETE_DIRECTORY);
                        assert isTableDiffDirectoryPresent;
                        // Set boolean to determine the db properties need to sorted once dump is complete
                        unsetDbPropertiesForOptimisedBootstrap = true;
                        long fromEventId = Long.parseLong(getEventIdFromFile(previousValidHiveDumpPath.getParent(), conf)[1]);
                        LOG.info("Starting optimised bootstrap from event id {} for database {}", fromEventId, work.dbNameOrPattern);
                        work.setEventFrom(fromEventId);
                        // Get the tables to be bootstrapped from the table diff
                        tablesForBootstrap = getTablesFromTableDiffFile(previousValidHiveDumpPath.getParent(), conf);
                        // Generate the bootstrapped table list and put it in the new dump directory for the load to consume.
                        createBootstrapTableList(currentDumpPath, tablesForBootstrap, conf);
                        // Call the normal dump with the tablesForBootstrap set.
                        lastReplId = incrementalDump(hiveDumpRoot, dmd, cmRoot, getHive());
                    }
                } else if (work.isBootstrap()) {
                    lastReplId = bootStrapDump(hiveDumpRoot, dmd, cmRoot, getHive());
                } else {
                    work.setEventFrom(getEventFromPreviousDumpMetadata(previousValidHiveDumpPath));
                    lastReplId = incrementalDump(hiveDumpRoot, dmd, cmRoot, getHive());
                }
                // The datacopy doesn't need to be initialised in case of optimised bootstrap first dump.
                if (lastReplId >= 0) {
                    work.setResultValues(Arrays.asList(currentDumpPath.toUri().toString(), String.valueOf(lastReplId)));
                    initiateDataCopyTasks();
                }
            } else {
                if (isFailoverMarkerPresent) {
                    LOG.info("Previous Dump is failover ready. Skipping this iteration.");
                } else {
                    LOG.info("Previous Dump is not yet loaded. Skipping this iteration.");
                }
            }
        }
    } catch (RuntimeException e) {
        LOG.error("replication failed with run time exception", e);
        setException(e);
        try {
            ReplUtils.handleException(true, e, work.getCurrentDumpPath().toString(), work.getMetricCollector(), getName(), conf);
        } catch (Exception ex) {
            LOG.error("Failed to collect replication metrics: ", ex);
        }
        throw e;
    } catch (Exception e) {
        setException(e);
        int errorCode;
        if (e instanceof SnapshotException) {
            errorCode = ErrorMsg.getErrorMsg("SNAPSHOT_ERROR").getErrorCode();
        } else {
            errorCode = ErrorMsg.getErrorMsg(e.getMessage()).getErrorCode();
        }
        try {
            return ReplUtils.handleException(true, e, work.getCurrentDumpPath().toString(), work.getMetricCollector(), getName(), conf);
        } catch (Exception ex) {
            LOG.error("Failed to collect replication metrics: ", ex);
            return errorCode;
        }
    }
    return 0;
}

Example 7 with SnapshotException

use of org.apache.hadoop.hdfs.protocol.SnapshotException in project hive by apache.

the class TestReplicationScenariosUsingSnapshots method testFailureScenarios.

@Test
public void testFailureScenarios() throws Throwable {
    DistributedFileSystem fs = primary.miniDFSCluster.getFileSystem();
    List<String> withClause = ReplicationTestUtils.includeExternalTableClause(true);
    // Create a table which couldn't create snapshot during dump.
    Path externalTableLocationSource = new Path("/" + testName.getMethodName() + "source1/tablesource/");
    fs.mkdirs(externalTableLocationSource, new FsPermission("777"));
    // Allow snapshot on the parent of table directory, the creation of snapshot shall fail for the table directory
    // during dump.
    fs.allowSnapshot(externalTableLocationSource.getParent());
    // Create a table which can not create snapshot on the destination.
    Path externalTableLocationDest = new Path("/" + testName.getMethodName() + "dest1/tabledest/");
    fs.mkdirs(externalTableLocationDest, new FsPermission("777"));
    Path externalTableLocationDestInDest = new Path(REPLICA_EXTERNAL_BASE, testName.getMethodName() + "dest1/tabledest/");
    withClause.add("'hive.repl.external.warehouse.single.copy.task.paths'='" + externalTableLocationSource.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + "'");
    try {
        WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName).run("create external table tablesource (place string) row format delimited fields terminated by ',' location '" + externalTableLocationSource.toString() + "'").run("insert into tablesource values ('bangalore')").dump(primaryDbName, withClause);
        fail("Should have thrown snapshot exception");
    } catch (SnapshotException se) {
    // Ignore
    }
    // Check if there is a non-recoverable error or not.
    Path baseDumpDir = new Path(primary.hiveConf.getVar(HiveConf.ConfVars.REPLDIR));
    Path nonRecoverablePath = TestReplicationScenarios.getNonRecoverablePath(baseDumpDir, primaryDbName, primary.hiveConf);
    assertTrue(fs.exists(nonRecoverablePath));
    // Fix the table and dump & load which should be success.
    fs.disallowSnapshot(externalTableLocationSource.getParent());
    fs.delete(nonRecoverablePath, true);
    primary.dump(primaryDbName, withClause);
    // Load and check if the data is there, we should have fallback to normal mode
    replica.load(replicatedDbName, primaryDbName, withClause).run("use " + replicatedDbName).run("select place from tablesource").verifyResults(new String[] { "bangalore" }).verifyReplTargetProperty(replicatedDbName);
    // Create a table for which target is non-snapshottable.
    // Allow snapshot on the parent of destination.
    fs.mkdirs(externalTableLocationDestInDest.getParent());
    fs.allowSnapshot(externalTableLocationDestInDest.getParent());
    withClause.add("'hive.repl.external.warehouse.single.copy.task.paths'='" + externalTableLocationSource.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + "," + externalTableLocationDest.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + "'");
    WarehouseInstance.Tuple tuple;
    try {
        tuple = primary.run("use " + primaryDbName).run("create external table tabledest (place string) partitioned by (country string) row format " + "delimited fields terminated by ',' location '" + externalTableLocationDest.toString() + "'").run("insert into tabledest partition(country='india') values ('kanpur')").run("insert into tabledest partition(country='india') values ('lucknow')").run("insert into tabledest partition(country='usa') values ('New York')").run("insert into tablesource values('chennai')").dump(primaryDbName, withClause);
        fail("Expected a snapshot exception.");
    } catch (SecurityException se) {
    // This is expected!!!
    }
    nonRecoverablePath = TestReplicationScenarios.getNonRecoverablePath(baseDumpDir, primaryDbName, primary.hiveConf);
    assertTrue(fs.exists(nonRecoverablePath));
    fs.delete(nonRecoverablePath, true);
    fs.disallowSnapshot(externalTableLocationDestInDest.getParent());
    primary.dump(primaryDbName, withClause);
    replica.load(replicatedDbName, primaryDbName, withClause).run("use " + replicatedDbName).run("select place from tabledest where country='usa'").verifyResults(new String[] { "New York" }).run("select place from tabledest where country='india'").verifyResults(new String[] { "kanpur", "lucknow" }).run("select place from tablesource").verifyResults(new String[] { "bangalore", "chennai" });
    // Add a new table which will fail to create snapshots, post snapshots for other tables have been created.
    Path externalTableLocationSource2 = new Path("/" + testName.getMethodName() + "source2/tablesource/");
    fs.mkdirs(externalTableLocationSource2, new FsPermission("777"));
    fs.allowSnapshot(externalTableLocationSource2.getParent());
    withClause.add("'hive.repl.external.warehouse.single.copy.task.paths'='" + externalTableLocationSource.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + "," + externalTableLocationDestInDest.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + "," + externalTableLocationSource2.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + "'");
    try {
        tuple = primary.run("use " + primaryDbName).run("create external table tablesource2 (place string) row format delimited fields terminated by ',' " + "location '" + externalTableLocationSource2.toString() + "'").run("insert into tablesource2 values ('delhi')").run("insert into tablesource2 values ('noida')").run("insert into tabledest partition(country='usa') values ('San Jose')").run("insert into tablesource values('kolkata')").dump(primaryDbName, withClause);
        fail("Expected a snapshot exception.");
    } catch (SnapshotException se) {
    // Expected, Ignore
    }
    nonRecoverablePath = TestReplicationScenarios.getNonRecoverablePath(baseDumpDir, primaryDbName, primary.hiveConf);
    assertTrue(fs.exists(nonRecoverablePath));
    fs.delete(nonRecoverablePath, true);
    fs.disallowSnapshot(externalTableLocationSource2.getParent());
    primary.run("insert into tablesource values('patna')").dump(primaryDbName, withClause);
    replica.load(replicatedDbName, primaryDbName, withClause).run("use " + replicatedDbName).run("select place from tabledest where country='usa'").verifyResults(new String[] { "New York", "San Jose" }).run("select place from tabledest where country='india'").verifyResults(new String[] { "kanpur", "lucknow" }).run("select place from tablesource").verifyResults(new String[] { "bangalore", "chennai", "kolkata", "patna" }).run("select place from tablesource2").verifyResults(new String[] { "delhi", "noida" });
}

Example 8 with SnapshotException

use of org.apache.hadoop.hdfs.protocol.SnapshotException in project hadoop by apache.

the class TestFileContextSnapshot method testCreateAndDeleteSnapshot.

@Test(timeout = 60000)
public void testCreateAndDeleteSnapshot() throws Exception {
    DFSTestUtil.createFile(dfs, filePath, BLOCKSIZE, REPLICATION, SEED);
    // disallow snapshot on dir
    dfs.disallowSnapshot(snapRootPath);
    try {
        fileContext.createSnapshot(snapRootPath, "s1");
    } catch (SnapshotException e) {
        GenericTestUtils.assertExceptionContains("Directory is not a snapshottable directory: " + snapRootPath, e);
    }
    // allow snapshot on dir
    dfs.allowSnapshot(snapRootPath);
    Path ssPath = fileContext.createSnapshot(snapRootPath, "s1");
    assertTrue("Failed to create snapshot", dfs.exists(ssPath));
    fileContext.deleteSnapshot(snapRootPath, "s1");
    assertFalse("Failed to delete snapshot", dfs.exists(ssPath));
}

Example 9 with SnapshotException

use of org.apache.hadoop.hdfs.protocol.SnapshotException in project hadoop by apache.

the class TestSnapshotManager method testSnapshotLimits.

/**
   * Test that the global limit on snapshots is honored.
   */
@Test(timeout = 10000)
public void testSnapshotLimits() throws Exception {
    // Setup mock objects for SnapshotManager.createSnapshot.
    //
    INodeDirectory ids = mock(INodeDirectory.class);
    FSDirectory fsdir = mock(FSDirectory.class);
    INodesInPath iip = mock(INodesInPath.class);
    SnapshotManager sm = spy(new SnapshotManager(fsdir));
    doReturn(ids).when(sm).getSnapshottableRoot((INodesInPath) anyObject());
    doReturn(testMaxSnapshotLimit).when(sm).getMaxSnapshotID();
    //
    for (Integer i = 0; i < testMaxSnapshotLimit; ++i) {
        sm.createSnapshot(iip, "dummy", i.toString());
    }
    //
    try {
        sm.createSnapshot(iip, "dummy", "shouldFailSnapshot");
        Assert.fail("Expected SnapshotException not thrown");
    } catch (SnapshotException se) {
        Assert.assertTrue(StringUtils.toLowerCase(se.getMessage()).contains("rollover"));
    }
    // Delete a snapshot to free up a slot.
    //
    sm.deleteSnapshot(iip, "", mock(INode.ReclaimContext.class));
    //
    try {
        sm.createSnapshot(iip, "dummy", "shouldFailSnapshot2");
        Assert.fail("Expected SnapshotException not thrown");
    } catch (SnapshotException se) {
        Assert.assertTrue(StringUtils.toLowerCase(se.getMessage()).contains("rollover"));
    }
}

Example 10 with SnapshotException

use of org.apache.hadoop.hdfs.protocol.SnapshotException in project hadoop by apache.

the class TestNestedSnapshots method testNestedSnapshots.

/**
   * Create a snapshot for /test/foo and create another snapshot for
   * /test/foo/bar.  Files created before the snapshots should appear in both
   * snapshots and the files created after the snapshots should not appear in
   * any of the snapshots.  
   */
@Test(timeout = 300000)
public void testNestedSnapshots() throws Exception {
    cluster.getNamesystem().getSnapshotManager().setAllowNestedSnapshots(true);
    final Path foo = new Path("/testNestedSnapshots/foo");
    final Path bar = new Path(foo, "bar");
    final Path file1 = new Path(bar, "file1");
    DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, SEED);
    print("create file " + file1);
    final String s1name = "foo-s1";
    final Path s1path = SnapshotTestHelper.getSnapshotRoot(foo, s1name);
    hdfs.allowSnapshot(foo);
    print("allow snapshot " + foo);
    hdfs.createSnapshot(foo, s1name);
    print("create snapshot " + s1name);
    final String s2name = "bar-s2";
    final Path s2path = SnapshotTestHelper.getSnapshotRoot(bar, s2name);
    hdfs.allowSnapshot(bar);
    print("allow snapshot " + bar);
    hdfs.createSnapshot(bar, s2name);
    print("create snapshot " + s2name);
    final Path file2 = new Path(bar, "file2");
    DFSTestUtil.createFile(hdfs, file2, BLOCKSIZE, REPLICATION, SEED);
    print("create file " + file2);
    assertFile(s1path, s2path, file1, true, true, true);
    assertFile(s1path, s2path, file2, true, false, false);
    //test root
    final String rootStr = "/";
    final Path rootPath = new Path(rootStr);
    hdfs.allowSnapshot(rootPath);
    print("allow snapshot " + rootStr);
    final Path rootSnapshot = hdfs.createSnapshot(rootPath);
    print("create snapshot " + rootSnapshot);
    hdfs.deleteSnapshot(rootPath, rootSnapshot.getName());
    print("delete snapshot " + rootSnapshot);
    hdfs.disallowSnapshot(rootPath);
    print("disallow snapshot " + rootStr);
    //change foo to non-snapshottable
    hdfs.deleteSnapshot(foo, s1name);
    hdfs.disallowSnapshot(foo);
    //test disallow nested snapshots
    cluster.getNamesystem().getSnapshotManager().setAllowNestedSnapshots(false);
    try {
        hdfs.allowSnapshot(rootPath);
        Assert.fail();
    } catch (SnapshotException se) {
        assertNestedSnapshotException(se, "subdirectory");
    }
    try {
        hdfs.allowSnapshot(foo);
        Assert.fail();
    } catch (SnapshotException se) {
        assertNestedSnapshotException(se, "subdirectory");
    }
    final Path sub1Bar = new Path(bar, "sub1");
    final Path sub2Bar = new Path(sub1Bar, "sub2");
    hdfs.mkdirs(sub2Bar);
    try {
        hdfs.allowSnapshot(sub1Bar);
        Assert.fail();
    } catch (SnapshotException se) {
        assertNestedSnapshotException(se, "ancestor");
    }
    try {
        hdfs.allowSnapshot(sub2Bar);
        Assert.fail();
    } catch (SnapshotException se) {
        assertNestedSnapshotException(se, "ancestor");
    }
}