Examples with MetaMasterClient alluxio.client.meta.MetaMasterClient used on opensource projects

Example 1 with MetaMasterClient

use of alluxio.client.meta.MetaMasterClient in project alluxio by Alluxio.

the class JournalMigrationIntegrationTest method migration.

@Test
public void migration() throws Exception {
    MultiProcessCluster cluster = MultiProcessCluster.newBuilder(PortCoordination.JOURNAL_MIGRATION).setClusterName("journalMigration").setNumMasters(3).addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS.toString()).addProperty(PropertyKey.ZOOKEEPER_SESSION_TIMEOUT, "1sec").build();
    try {
        cluster.start();
        FileSystem fs = cluster.getFileSystemClient();
        MetaMasterClient metaClient = new RetryHandlingMetaMasterClient(MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).setMasterInquireClient(cluster.getMasterInquireClient()).build());
        for (int i = 0; i < NUM_DIRS; i++) {
            fs.createDirectory(new AlluxioURI("/dir" + i));
        }
        File backupsDir = AlluxioTestDirectory.createTemporaryDirectory("backups");
        AlluxioURI zkBackup = metaClient.backup(BackupPRequest.newBuilder().setTargetDirectory(backupsDir.getAbsolutePath()).setOptions(BackupPOptions.newBuilder().setLocalFileSystem(false)).build()).getBackupUri();
        cluster.updateMasterConf(PropertyKey.MASTER_JOURNAL_INIT_FROM_BACKUP, zkBackup.toString());
        // Migrate to embedded journal HA.
        cluster.stopMasters();
        cluster.formatJournal();
        cluster.updateDeployMode(DeployMode.EMBEDDED);
        cluster.startMasters();
        assertEquals(NUM_DIRS, fs.listStatus(new AlluxioURI("/")).size());
        // Migrate back to Zookeeper HA.
        cluster.stopMasters();
        cluster.formatJournal();
        cluster.updateDeployMode(DeployMode.ZOOKEEPER_HA);
        cluster.startMasters();
        assertEquals(NUM_DIRS, fs.listStatus(new AlluxioURI("/")).size());
        cluster.notifySuccess();
    } finally {
        cluster.destroy();
    }
}

Example 2 with MetaMasterClient

use of alluxio.client.meta.MetaMasterClient in project alluxio by Alluxio.

the class MultiProcessCluster method waitForAllNodesRegistered.

/**
 * Waits for all nodes to be registered.
 *
 * @param timeoutMs maximum amount of time to wait, in milliseconds
 */
public synchronized void waitForAllNodesRegistered(int timeoutMs) throws TimeoutException, InterruptedException {
    MetaMasterClient metaMasterClient = getMetaMasterClient();
    int nodeCount = mNumMasters + mNumWorkers;
    CommonUtils.waitFor("all nodes registered", () -> {
        try {
            MasterInfo masterInfo = metaMasterClient.getMasterInfo(Collections.emptySet());
            int liveNodeNum = masterInfo.getMasterAddressesList().size() + masterInfo.getWorkerAddressesList().size();
            if (liveNodeNum == nodeCount) {
                return true;
            } else {
                LOG.info("Master addresses: {}. Worker addresses: {}", masterInfo.getMasterAddressesList(), masterInfo.getWorkerAddressesList());
                return false;
            }
        } catch (UnavailableException e) {
            return false;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }, WaitForOptions.defaults().setInterval(200).setTimeoutMs(timeoutMs));
}

Example 3 with MetaMasterClient

use of alluxio.client.meta.MetaMasterClient in project alluxio by Alluxio.

the class ConfigCheckerIntegrationTest method getReport.

private ConfigCheckReport getReport() throws Exception {
    mCluster.waitForAllNodesRegistered(WAIT_TIMEOUT_MS);
    MetaMasterClient client = mCluster.getMetaMasterClient();
    return client.getConfigReport();
}

Example 4 with MetaMasterClient

use of alluxio.client.meta.MetaMasterClient in project alluxio by Alluxio.

the class JournalBackupIntegrationTest method backupRestoreMetaStoreTest.

private void backupRestoreMetaStoreTest() throws Exception {
    // Directory for backups.
    File backups = AlluxioTestDirectory.createTemporaryDirectory("backups");
    mCluster.start();
    // Needs workers to be up before the test.
    mCluster.waitForAllNodesRegistered(WAIT_NODES_REGISTERED_MS);
    // Acquire clients.
    FileSystem fs = mCluster.getFileSystemClient();
    MetaMasterClient metaClient = getMetaClient(mCluster);
    BlockMasterClient blockClient = getBlockClient(mCluster);
    // Create single test file.
    String testFilePath = "/file";
    AlluxioURI testFileUri = new AlluxioURI(testFilePath);
    // Create file THROUGH.
    FileSystemTestUtils.createByteFile(fs, testFilePath, 100, CreateFilePOptions.newBuilder().setWriteType(WritePType.THROUGH).build());
    // Delete it from Alluxio namespace.
    fs.delete(testFileUri, DeletePOptions.newBuilder().setAlluxioOnly(true).build());
    // List status on root to bring the file's meta back.
    fs.listStatus(new AlluxioURI("/"), ListStatusPOptions.newBuilder().setRecursive(true).setLoadMetadataType(LoadMetadataPType.ONCE).build());
    // Verify that file's meta is in Alluxio.
    assertNotNull(fs.getStatus(testFileUri));
    // Take a backup.
    AlluxioURI backup1 = metaClient.backup(BackupPRequest.newBuilder().setTargetDirectory(backups.getAbsolutePath()).setOptions(BackupPOptions.newBuilder().setLocalFileSystem(false)).build()).getBackupUri();
    // Restart with backup.
    restartMastersFromBackup(backup1);
    // Verify that file and its blocks are in Alluxio after restore.
    URIStatus fileStatus = fs.getStatus(testFileUri);
    assertNotNull(fileStatus);
    for (long blockId : fileStatus.getBlockIds()) {
        assertNotNull(blockClient.getBlockInfo(blockId));
    }
}

Example 5 with MetaMasterClient

use of alluxio.client.meta.MetaMasterClient in project alluxio by Alluxio.

the class JournalBackupIntegrationTest method backupRestoreTest.

private void backupRestoreTest(boolean testFailover) throws Exception {
    File backups = AlluxioTestDirectory.createTemporaryDirectory("backups");
    mCluster.start();
    List<Thread> opThreads = new ArrayList<>();
    // are happening.
    for (int i = 0; i < 10; i++) {
        AlluxioOperationThread thread = new AlluxioOperationThread(mCluster.getFileSystemClient());
        thread.start();
        opThreads.add(thread);
    }
    try {
        FileSystem fs = mCluster.getFileSystemClient();
        MetaMasterClient metaClient = getMetaClient(mCluster);
        AlluxioURI dir1 = new AlluxioURI("/dir1");
        fs.createDirectory(dir1, CreateDirectoryPOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build());
        AlluxioURI backup1 = metaClient.backup(BackupPRequest.newBuilder().setTargetDirectory(backups.getAbsolutePath()).setOptions(BackupPOptions.newBuilder().setLocalFileSystem(false)).build()).getBackupUri();
        AlluxioURI dir2 = new AlluxioURI("/dir2");
        fs.createDirectory(dir2, CreateDirectoryPOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build());
        AlluxioURI backup2 = metaClient.backup(BackupPRequest.newBuilder().setTargetDirectory(backups.getAbsolutePath()).setOptions(BackupPOptions.newBuilder().setLocalFileSystem(false)).build()).getBackupUri();
        restartMastersFromBackup(backup2);
        assertTrue(fs.exists(dir1));
        assertTrue(fs.exists(dir2));
        restartMastersFromBackup(backup1);
        assertTrue(fs.exists(dir1));
        assertFalse(fs.exists(dir2));
        // Restart normally and make sure we remember the state from backup 1.
        mCluster.stopMasters();
        mCluster.startMasters();
        assertTrue(fs.exists(dir1));
        assertFalse(fs.exists(dir2));
        if (testFailover) {
            // Verify that failover works correctly.
            mCluster.waitForAndKillPrimaryMaster(30 * Constants.SECOND_MS);
            assertTrue(fs.exists(dir1));
            assertFalse(fs.exists(dir2));
        }
        mCluster.notifySuccess();
    } finally {
        opThreads.forEach(Thread::interrupt);
    }
}