Examples with ConfigurationRule alluxio.ConfigurationRule used on opensource projects

Example 21 with ConfigurationRule

use of alluxio.ConfigurationRule in project alluxio by Alluxio.

the class UfsInputStreamCacheTest method releaseExpiredSameFile.

@Test
public void releaseExpiredSameFile() throws Exception {
    try (Closeable r = new ConfigurationRule(new HashMap<PropertyKey, Object>() {

        {
            put(PropertyKey.WORKER_UFS_INSTREAM_CACHE_EXPIRARTION_TIME, "2");
        }
    }, ServerConfiguration.global()).toResource()) {
        mManager = new UfsInputStreamCache();
        // check out a stream
        InputStream instream = mManager.acquire(mUfs, FILE_NAME, FILE_ID, OpenOptions.defaults().setOffset(2));
        // wait a bit for a stream to be expired
        Thread.sleep(100);
        // check out another stream should trigger the timeout
        mManager.acquire(mUfs, FILE_NAME, FILE_ID, OpenOptions.defaults().setOffset(4));
        // wait a bit so release occurs after removal listener
        Thread.sleep(100);
        mManager.release(instream);
        // verify the stream was closed once
        verify(mSeekableInStreams[0], timeout(2000).times(1)).close();
    }
}

Example 22 with ConfigurationRule

use of alluxio.ConfigurationRule in project alluxio by Alluxio.

the class UfsInputStreamCacheTest method concurrency.

@Test
public void concurrency() throws Exception {
    try (Closeable r = new ConfigurationRule(new HashMap<PropertyKey, Object>() {

        {
            // use very large number
            put(PropertyKey.WORKER_UFS_INSTREAM_CACHE_EXPIRARTION_TIME, "200000");
        }
    }, ServerConfiguration.global()).toResource()) {
        mManager = new UfsInputStreamCache();
        List<Thread> threads = new ArrayList<>();
        int numCheckOutPerThread = 10;
        for (int i = 0; i < mNumOfInputStreams / 2; i++) {
            Runnable runnable = () -> {
                for (int j = 0; j < numCheckOutPerThread; j++) {
                    InputStream instream;
                    try {
                        instream = mManager.acquire(mUfs, FILE_NAME, FILE_ID, OpenOptions.defaults().setOffset(j));
                        Thread.sleep(10);
                        mManager.release(instream);
                    } catch (Exception e) {
                        // the input streams created should not be more than mNumOfInputStreams
                        Assert.fail("input stream check in and out failed." + e);
                    }
                }
            };
            threads.add(new Thread(runnable));
        }
        ConcurrencyUtils.assertConcurrent(threads, 30);
        // Each subsequent check out per thread should be a seek operation
        int numSeek = 0;
        for (int i = 0; i < mNumOfInputStreams; i++) {
            for (Invocation invocation : mockingDetails(mSeekableInStreams[i]).getInvocations()) {
                if (invocation.getMethod().getName().equals("seek")) {
                    numSeek++;
                }
            }
        }
        Assert.assertEquals(mNumOfInputStreams / 2 * (numCheckOutPerThread - 1), numSeek);
    }
}

Example 23 with ConfigurationRule

use of alluxio.ConfigurationRule in project alluxio by Alluxio.

the class MultiProcessCluster method formatJournal.

/**
 * Formats the cluster journal.
 */
public synchronized void formatJournal() throws IOException {
    if (mDeployMode == DeployMode.EMBEDDED) {
        for (Master master : mMasters) {
            File journalDir = new File((String) master.getConf().get(PropertyKey.MASTER_JOURNAL_FOLDER));
            FileUtils.deleteDirectory(journalDir);
            journalDir.mkdirs();
        }
        return;
    }
    try (Closeable c = new ConfigurationRule(mProperties, ServerConfiguration.global()).toResource()) {
        Format.format(Format.Mode.MASTER, ServerConfiguration.global());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 24 with ConfigurationRule

use of alluxio.ConfigurationRule in project alluxio by Alluxio.

the class CopyFromLocalCommandIntegrationTest method copyFromLocalMustCacheThenCacheThrough.

@Test
public void copyFromLocalMustCacheThenCacheThrough() throws Exception {
    File file = mTestFolder.newFile();
    try (Closeable c = new ConfigurationRule(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.MUST_CACHE.toString(), ServerConfiguration.global()).toResource()) {
        Assert.assertEquals(0, sFsShell.run("copyFromLocal", file.getAbsolutePath(), "/"));
    }
    try (Closeable c = new ConfigurationRule(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.CACHE_THROUGH.toString(), ServerConfiguration.global()).toResource()) {
        mOutput.reset();
        sFsShell.run("copyFromLocal", file.getAbsolutePath(), "/");
    }
    Assert.assertThat(mOutput.toString(), containsString("Not allowed to create file because path already exists"));
}

Example 25 with ConfigurationRule

use of alluxio.ConfigurationRule in project alluxio by Alluxio.

the class AbstractFileSystemTest method getBlockLocationsNoMatchingWorkersWithFallback.

@Test
public void getBlockLocationsNoMatchingWorkersWithFallback() throws Exception {
    WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234);
    WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234);
    List<WorkerNetAddress> blockWorkers = Arrays.asList();
    List<String> ufsLocations = Arrays.asList("worker0", "worker3");
    List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2);
    List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1, worker2);
    try (Closeable conf = new ConfigurationRule(PropertyKey.USER_UFS_BLOCK_LOCATION_ALL_FALLBACK_ENABLED, true, mConfiguration).toResource()) {
        verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers);
    }
}