Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 31 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ClusterInitializationIntegrationTest method recoverClusterSuccess.

/**
 * When a user starts a cluster with journal logs, which are generated by previous running
 * cluster owned by the same user, it should succeed.
 */
@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.SECURITY_LOGIN_USERNAME, SUPER_USER })
public void recoverClusterSuccess() throws Exception {
    FileSystem fs = mLocalAlluxioClusterResource.get().getClient();
    fs.createFile(new AlluxioURI("/testFile")).close();
    mLocalAlluxioClusterResource.get().stopFS();
    // user alluxio can recover master from journal
    try (FsMasterResource masterResource = MasterTestUtils.createLeaderFileSystemMasterFromJournal()) {
        FileSystemMaster fileSystemMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        AuthenticatedClientUser.set(SUPER_USER);
        assertEquals(SUPER_USER, fileSystemMaster.getFileInfo(new AlluxioURI("/testFile"), GetStatusContext.defaults()).getOwner());
    }
}

Example 32 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ClusterInitializationIntegrationTest method startCluster.

/**
 * When a user starts a new cluster, an empty root dir is created and owned by the user.
 */
@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.SECURITY_LOGIN_USERNAME, SUPER_USER })
public void startCluster() throws Exception {
    FileSystem fs = mLocalAlluxioClusterResource.get().getClient();
    URIStatus status = fs.getStatus(ROOT);
    assertEquals(SUPER_USER, status.getOwner());
    assertEquals(0755, status.getMode());
    assertEquals(0, fs.listStatus(new AlluxioURI("/")).size());
}

Example 33 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class EmbeddedJournalIntegrationTestFaultTolerance method failover.

@Test
public void failover() throws Exception {
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.EMBEDDED_JOURNAL_FAILOVER).setClusterName("EmbeddedJournalFaultTolerance_failover").setNumMasters(NUM_MASTERS).setNumWorkers(NUM_WORKERS).addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED.toString()).addProperty(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "5min").addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "750ms").addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "1500ms").build();
    mCluster.start();
    AlluxioURI testDir = new AlluxioURI("/dir");
    FileSystem fs = mCluster.getFileSystemClient();
    fs.createDirectory(testDir);
    mCluster.waitForAndKillPrimaryMaster(MASTER_INDEX_WAIT_TIME);
    assertTrue(fs.exists(testDir));
    mCluster.notifySuccess();
}

Example 34 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class EmbeddedJournalIntegrationTestFaultTolerance method restartStress.

@Test
public void restartStress() throws Throwable {
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.EMBEDDED_JOURNAL_RESTART_STRESS).setClusterName("EmbeddedJournalFaultTolerance_restartStress").setNumMasters(NUM_MASTERS).setNumWorkers(NUM_WORKERS).addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED.toString()).addProperty(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "5min").addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "750ms").addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "1500ms").build();
    mCluster.start();
    // Run and verify operations while restarting the cluster multiple times.
    AtomicReference<Throwable> failure = new AtomicReference<>();
    AtomicInteger successes = new AtomicInteger(0);
    FileSystem fs = mCluster.getFileSystemClient();
    List<EmbeddedJournalIntegrationTestFaultTolerance.OperationThread> threads = new ArrayList<>();
    try {
        for (int i = 0; i < 10; i++) {
            EmbeddedJournalIntegrationTestFaultTolerance.OperationThread t = new EmbeddedJournalIntegrationTestFaultTolerance.OperationThread(fs, i, failure, successes);
            t.start();
            threads.add(t);
        }
        for (int i = 0; i < 2; i++) {
            restartMasters();
            successes.set(0);
            CommonUtils.waitFor("11 successes", () -> successes.get() >= 11, WaitForOptions.defaults().setTimeoutMs(RESTART_TIMEOUT_MS));
            if (failure.get() != null) {
                throw failure.get();
            }
        }
    } finally {
        threads.forEach(Thread::interrupt);
        for (Thread t : threads) {
            t.join();
        }
    }
    mCluster.notifySuccess();
}

Example 35 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class EmbeddedJournalIntegrationTestResizing method resizeCluster.

@Test
public void resizeCluster() throws Exception {
    final int NUM_MASTERS = 5;
    final int NUM_WORKERS = 0;
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.EMBEDDED_JOURNAL_RESIZE).setClusterName("EmbeddedJournalResizing_resizeCluster").setNumMasters(NUM_MASTERS).setNumWorkers(NUM_WORKERS).addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED.toString()).addProperty(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "5min").addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "750ms").addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "1500ms").build();
    mCluster.start();
    assertEquals(5, mCluster.getJournalMasterClientForMaster().getQuorumInfo().getServerInfoList().size());
    AlluxioURI testDir = new AlluxioURI("/" + CommonUtils.randomAlphaNumString(10));
    FileSystem fs = mCluster.getFileSystemClient();
    fs.createDirectory(testDir);
    assertTrue(fs.exists(testDir));
    // Stop 2 masters. Now cluster can't tolerate any loss.
    mCluster.stopMaster(0);
    mCluster.stopMaster(1);
    // Verify cluster is still serving requests.
    assertTrue(fs.exists(testDir));
    waitForQuorumPropertySize(info -> info.getServerState() == QuorumServerState.UNAVAILABLE, 2);
    // Get and verify list of unavailable masters.
    List<NetAddress> unavailableMasters = new LinkedList<>();
    for (QuorumServerInfo serverState : mCluster.getJournalMasterClientForMaster().getQuorumInfo().getServerInfoList()) {
        if (serverState.getServerState().equals(QuorumServerState.UNAVAILABLE)) {
            unavailableMasters.add(serverState.getServerAddress());
        }
    }
    assertEquals(2, unavailableMasters.size());
    // Remove unavailable masters from quorum.
    for (NetAddress unavailableMasterAddress : unavailableMasters) {
        mCluster.getJournalMasterClientForMaster().removeQuorumServer(unavailableMasterAddress);
    }
    // Verify quorum is down to 3 masters.
    assertEquals(3, mCluster.getJournalMasterClientForMaster().getQuorumInfo().getServerInfoList().size());
    // Validate that cluster can tolerate one more master failure after resizing.
    mCluster.stopMaster(2);
    assertTrue(fs.exists(testDir));
    mCluster.notifySuccess();
}