Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 96 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class PlanCoordinator method updateStatus.

/**
 * Updates the status of the job. When all the tasks are completed, run the join method in the
 * definition.
 */
private synchronized void updateStatus() {
    int completed = 0;
    List<TaskInfo> taskInfoList = mPlanInfo.getTaskInfoList();
    JobConfig config = mPlanInfo.getJobConfig();
    Preconditions.checkNotNull(config);
    FileSystem fileSystem = mJobServerContext.getFileSystem();
    for (TaskInfo info : taskInfoList) {
        Status status = info.getStatus();
        switch(status) {
            case FAILED:
                setJobAsFailed(info.getErrorType(), "Task execution failed: " + info.getErrorMessage());
                return;
            case CANCELED:
                if (mPlanInfo.getStatus() != Status.FAILED) {
                    mPlanInfo.setStatus(Status.CANCELED);
                    DistributedCmdMetrics.incrementForAllConfigsCancelStatus(config);
                }
                return;
            case RUNNING:
                if (mPlanInfo.getStatus() != Status.FAILED && mPlanInfo.getStatus() != Status.CANCELED) {
                    mPlanInfo.setStatus(Status.RUNNING);
                }
                break;
            case COMPLETED:
                completed++;
                break;
            case CREATED:
                // do nothing
                break;
            default:
                throw new IllegalArgumentException("Unsupported status " + info.getStatus());
        }
    }
    if (completed == taskInfoList.size()) {
        if (mPlanInfo.getStatus() == Status.COMPLETED) {
            return;
        }
        // all the tasks completed, run join
        try {
            // Try to join first, so that in case of failure we don't move to a completed state yet
            mPlanInfo.setResult(join(taskInfoList));
            mPlanInfo.setStatus(Status.COMPLETED);
            // Increment the counter for Complete status when all the tasks in a job are completed.
            DistributedCmdMetrics.incrementForAllConfigsCompleteStatus(config, fileSystem, new CountingRetry(5));
        } catch (Exception e) {
            LOG.warn("Job error when joining tasks Job Id={} Config={}", mPlanInfo.getId(), mPlanInfo.getJobConfig(), e);
            setJobAsFailed(ErrorUtils.getErrorType(e), e.getMessage());
        }
    }
}

Example 97 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class DistributedLoadCommandTest method loadFileMultiCopy.

@Test
public void loadFileMultiCopy() throws IOException, AlluxioException {
    FileSystem fs = sResource.get().getClient();
    FileSystemTestUtils.createByteFile(fs, "/testFile", WritePType.THROUGH, 10);
    AlluxioURI uri = new AlluxioURI("/testFile");
    URIStatus status = fs.getStatus(uri);
    Assert.assertNotEquals(100, status.getInMemoryPercentage());
    // Testing loading of a single file
    sFsShell.run("distributedLoad", "/testFile", "--replication", "1");
    status = fs.getStatus(uri);
    Assert.assertEquals(100, status.getInMemoryPercentage());
    Assert.assertEquals(1, status.getFileBlockInfos().get(0).getBlockInfo().getLocations().size());
    sFsShell.run("distributedLoad", "/testFile", "--replication", "3");
    status = fs.getStatus(uri);
    Assert.assertEquals(3, status.getFileBlockInfos().get(0).getBlockInfo().getLocations().size());
}

Example 98 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class DistributedLoadCommandTest method loadDirWithLotsFilesInBatch.

@Test
public void loadDirWithLotsFilesInBatch() throws IOException, AlluxioException {
    FileSystem fs = sResource.get().getClient();
    int fileSize = 100;
    List<AlluxioURI> uris = new ArrayList<>(fileSize);
    for (int i = 0; i < fileSize; i++) {
        FileSystemTestUtils.createByteFile(fs, "/testBatchRoot/testBatchFile" + i, WritePType.THROUGH, 10);
        AlluxioURI uri = new AlluxioURI("/testBatchRoot/testBatchFile" + i);
        uris.add(uri);
        URIStatus status = fs.getStatus(uri);
        Assert.assertNotEquals(100, status.getInMemoryPercentage());
    }
    // Testing loading of a directory
    sFsShell.run("distributedLoad", "/testBatchRoot", "--batch-size", "3");
    for (AlluxioURI uri : uris) {
        URIStatus status = fs.getStatus(uri);
        Assert.assertEquals(100, status.getInMemoryPercentage());
    }
}

Example 99 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class DistributedLoadCommandTest method loadFile.

@Test
public void loadFile() throws IOException, AlluxioException {
    FileSystem fs = sResource.get().getClient();
    FileSystemTestUtils.createByteFile(fs, "/testFileNew", WritePType.THROUGH, 10);
    AlluxioURI uri = new AlluxioURI("/testFileNew");
    URIStatus status = fs.getStatus(uri);
    Assert.assertNotEquals(100, status.getInMemoryPercentage());
    // Testing loading of a single file
    sFsShell.run("distributedLoad", "/testFileNew");
    status = fs.getStatus(uri);
    Assert.assertEquals(100, status.getInMemoryPercentage());
}

Example 100 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class DistributedLoadCommandTest method loadDirWithPassiveCacheInBatch.

@Test
public void loadDirWithPassiveCacheInBatch() throws IOException, AlluxioException {
    FileSystem fs = sResource.get().getClient();
    int fileSize = 20;
    List<AlluxioURI> uris = new ArrayList<>(fileSize);
    for (int i = 0; i < fileSize; i++) {
        FileSystemTestUtils.createByteFile(fs, "/testBatchRoot/testBatchFilePassive" + i, WritePType.THROUGH, 10);
        AlluxioURI uri = new AlluxioURI("/testBatchRoot/testBatchFilePassive" + i);
        uris.add(uri);
        URIStatus status = fs.getStatus(uri);
        Assert.assertNotEquals(100, status.getInMemoryPercentage());
    }
    // Testing loading of a directory
    sFsShell.run("distributedLoad", "/testBatchRoot", "--batch-size", "3", "--passive-cache");
    for (AlluxioURI uri : uris) {
        URIStatus status = fs.getStatus(uri);
        Assert.assertEquals(100, status.getInMemoryPercentage());
    }
}