Search in sources :

Example 1 with Version

use of org.apache.hadoop.yarn.server.records.Version in project hadoop by apache.

the class TestZKRMStateStore method testCheckMajorVersionChange.

@Test(timeout = 60000)
public void testCheckMajorVersionChange() throws Exception {
    TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester() {

        Version VERSION_INFO = Version.newInstance(Integer.MAX_VALUE, 0);

        @Override
        public Version getCurrentVersion() throws Exception {
            return VERSION_INFO;
        }

        @Override
        public RMStateStore getRMStateStore() throws Exception {
            YarnConfiguration conf = new YarnConfiguration();
            workingZnode = "/jira/issue/3077/rmstore";
            conf.set(YarnConfiguration.RM_ZK_ADDRESS, curatorTestingServer.getConnectString());
            conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
            this.store = new TestZKRMStateStoreInternal(conf, workingZnode) {

                Version storedVersion = null;

                @Override
                public Version getCurrentVersion() {
                    return VERSION_INFO;
                }

                @Override
                protected synchronized Version loadVersion() throws Exception {
                    return storedVersion;
                }

                @Override
                protected synchronized void storeVersion() throws Exception {
                    storedVersion = VERSION_INFO;
                }
            };
            return this.store;
        }
    };
    // default version
    RMStateStore store = zkTester.getRMStateStore();
    Version defaultVersion = zkTester.getCurrentVersion();
    store.checkVersion();
    Assert.assertEquals(defaultVersion, store.loadVersion());
}
Also used : Version(org.apache.hadoop.yarn.server.records.Version) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) KeeperException(org.apache.zookeeper.KeeperException) IOException(java.io.IOException) Test(org.junit.Test)

Example 2 with Version

use of org.apache.hadoop.yarn.server.records.Version in project hadoop by apache.

the class FileSystemRMStateStore method loadVersion.

@Override
protected synchronized Version loadVersion() throws Exception {
    Path versionNodePath = getNodePath(rootDirPath, VERSION_NODE);
    FileStatus status = getFileStatusWithRetries(versionNodePath);
    if (status != null) {
        byte[] data = readFileWithRetries(versionNodePath, status.getLen());
        Version version = new VersionPBImpl(VersionProto.parseFrom(data));
        return version;
    }
    return null;
}
Also used : Path(org.apache.hadoop.fs.Path) FileStatus(org.apache.hadoop.fs.FileStatus) VersionPBImpl(org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl) Version(org.apache.hadoop.yarn.server.records.Version)

Example 3 with Version

use of org.apache.hadoop.yarn.server.records.Version in project hadoop by apache.

the class TestShuffleHandler method testRecoveryFromOtherVersions.

@Test
public void testRecoveryFromOtherVersions() throws IOException {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(12345, 1);
    final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")), TestShuffleHandler.class.getName());
    Configuration conf = new Configuration();
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
    conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
    ShuffleHandler shuffle = new ShuffleHandler();
    // emulate aux services startup with recovery enabled
    shuffle.setRecoveryPath(new Path(tmpDir.toString()));
    tmpDir.mkdirs();
    try {
        shuffle.init(conf);
        shuffle.start();
        // setup a shuffle token for an application
        DataOutputBuffer outputBuffer = new DataOutputBuffer();
        outputBuffer.reset();
        Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService"));
        jt.write(outputBuffer);
        shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength())));
        // verify we are authorized to shuffle
        int rc = getShuffleResponseCode(shuffle, jt);
        Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
        // emulate shuffle handler restart
        shuffle.close();
        shuffle = new ShuffleHandler();
        shuffle.setRecoveryPath(new Path(tmpDir.toString()));
        shuffle.init(conf);
        shuffle.start();
        // verify we are still authorized to shuffle to the old application
        rc = getShuffleResponseCode(shuffle, jt);
        Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
        Version version = Version.newInstance(1, 0);
        Assert.assertEquals(version, shuffle.getCurrentVersion());
        // emulate shuffle handler restart with compatible version
        Version version11 = Version.newInstance(1, 1);
        // update version info before close shuffle
        shuffle.storeVersion(version11);
        Assert.assertEquals(version11, shuffle.loadVersion());
        shuffle.close();
        shuffle = new ShuffleHandler();
        shuffle.setRecoveryPath(new Path(tmpDir.toString()));
        shuffle.init(conf);
        shuffle.start();
        // shuffle version will be override by CURRENT_VERSION_INFO after restart
        // successfully.
        Assert.assertEquals(version, shuffle.loadVersion());
        // verify we are still authorized to shuffle to the old application
        rc = getShuffleResponseCode(shuffle, jt);
        Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
        // emulate shuffle handler restart with incompatible version
        Version version21 = Version.newInstance(2, 1);
        shuffle.storeVersion(version21);
        Assert.assertEquals(version21, shuffle.loadVersion());
        shuffle.close();
        shuffle = new ShuffleHandler();
        shuffle.setRecoveryPath(new Path(tmpDir.toString()));
        shuffle.init(conf);
        try {
            shuffle.start();
            Assert.fail("Incompatible version, should expect fail here.");
        } catch (ServiceStateException e) {
            Assert.assertTrue("Exception message mismatch", e.getMessage().contains("Incompatible version for state DB schema:"));
        }
    } finally {
        if (shuffle != null) {
            shuffle.close();
        }
        FileUtil.fullyDelete(tmpDir);
    }
}
Also used : Path(org.apache.hadoop.fs.Path) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) JobTokenIdentifier(org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier) Token(org.apache.hadoop.security.token.Token) Text(org.apache.hadoop.io.Text) Version(org.apache.hadoop.yarn.server.records.Version) DataOutputBuffer(org.apache.hadoop.io.DataOutputBuffer) ServiceStateException(org.apache.hadoop.service.ServiceStateException) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) File(java.io.File) ApplicationInitializationContext(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext) Test(org.junit.Test)

Example 4 with Version

use of org.apache.hadoop.yarn.server.records.Version in project hadoop by apache.

the class ShuffleHandler method checkVersion.

/**
   * 1) Versioning scheme: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
   * 2) Any incompatible change of DB schema is a major upgrade, and any
   *    compatible change of DB schema is a minor upgrade.
   * 3) Within a minor upgrade, say 1.1 to 1.2:
   *    overwrite the version info and proceed as normal.
   * 4) Within a major upgrade, say 1.2 to 2.0:
   *    throw exception and indicate user to use a separate upgrade tool to
   *    upgrade shuffle info or remove incompatible old state.
   */
private void checkVersion() throws IOException {
    Version loadedVersion = loadVersion();
    LOG.info("Loaded state DB schema version info " + loadedVersion);
    if (loadedVersion.equals(getCurrentVersion())) {
        return;
    }
    if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
        LOG.info("Storing state DB schema version info " + getCurrentVersion());
        storeVersion();
    } else {
        throw new IOException("Incompatible version for state DB schema: expecting DB schema version " + getCurrentVersion() + ", but loading version " + loadedVersion);
    }
}
Also used : Version(org.apache.hadoop.yarn.server.records.Version) IOException(java.io.IOException)

Example 5 with Version

use of org.apache.hadoop.yarn.server.records.Version in project hadoop by apache.

the class RollingLevelDBTimelineStore method loadVersion.

Version loadVersion() throws IOException {
    byte[] data = starttimedb.get(bytes(TIMELINE_STORE_VERSION_KEY));
    // if version is not stored previously, treat it as 1.0.
    if (data == null || data.length == 0) {
        return Version.newInstance(1, 0);
    }
    Version version = new VersionPBImpl(VersionProto.parseFrom(data));
    return version;
}
Also used : VersionPBImpl(org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl) Version(org.apache.hadoop.yarn.server.records.Version)

Aggregations

Version (org.apache.hadoop.yarn.server.records.Version)26 IOException (java.io.IOException)11 Test (org.junit.Test)9 VersionPBImpl (org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl)8 ServiceStateException (org.apache.hadoop.service.ServiceStateException)7 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)4 VisibleForTesting (com.google.common.annotations.VisibleForTesting)3 Path (org.apache.hadoop.fs.Path)3 File (java.io.File)2 Configuration (org.apache.hadoop.conf.Configuration)2 DataOutputBuffer (org.apache.hadoop.io.DataOutputBuffer)2 Text (org.apache.hadoop.io.Text)2 Token (org.apache.hadoop.security.token.Token)2 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)2 ApplicationInitializationContext (org.apache.hadoop.yarn.server.api.ApplicationInitializationContext)2 FileStatus (org.apache.hadoop.fs.FileStatus)1 HdfsConfiguration (org.apache.hadoop.hdfs.HdfsConfiguration)1 MiniDFSCluster (org.apache.hadoop.hdfs.MiniDFSCluster)1 JobTokenIdentifier (org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier)1 AccessControlException (org.apache.hadoop.security.AccessControlException)1