Examples with Status alluxio.job.wire.Status used on opensource projects

Example 1 with Status

use of alluxio.job.wire.Status in project alluxio by Alluxio.

the class PlanCoordinatorTest method setTasksWithStatuses.

private void setTasksWithStatuses(PlanCoordinator planCoordinator, Status... statuses) throws Exception {
    List<TaskInfo> taskInfos = new ArrayList<>();
    int taskId = 0;
    for (Status status : statuses) {
        taskInfos.add(new TaskInfo().setTaskId(taskId++).setJobId(mJobId).setStatus(status));
    }
    planCoordinator.updateTasks(taskInfos);
}

Example 2 with Status

use of alluxio.job.wire.Status in project alluxio by Alluxio.

the class TableMasterJournalIntegrationTest method journalTransformDb.

@Test
public void journalTransformDb() throws Exception {
    LocalAlluxioCluster mCluster = sClusterResource.get();
    TableMaster tableMaster = mCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(TableMaster.class);
    LocalAlluxioJobCluster jobCluster = new LocalAlluxioJobCluster();
    jobCluster.start();
    JobMaster jobMaster = jobCluster.getMaster().getJobMaster();
    genTable(1, 2, true);
    tableMaster.attachDatabase(TestUdbFactory.TYPE, "connect", DB_NAME, DB_NAME, Collections.emptyMap(), false);
    List<String> tables = tableMaster.getAllTables(DB_NAME);
    assertFalse(tables.isEmpty());
    // all partitions are not transformed, so baselayout is the same as layout
    String tableName = tables.get(0);
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> partition.getBaseLayout() == partition.getLayout()));
    long jobid = tableMaster.transformTable(DB_NAME, tableName, null);
    assertNotEquals(0, jobid);
    JobTestUtils.waitForJobStatus(jobMaster, jobid, ImmutableSet.of(Status.COMPLETED, Status.CANCELED, Status.FAILED));
    final JobInfo status = jobMaster.getStatus(jobid);
    assertEquals("", status.getErrorMessage());
    assertEquals(Status.COMPLETED, status.getStatus());
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_TABLE_TRANSFORMATION_MONITOR);
    // all partitions are transformed, so baselayout should be different as layout
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> partition.getBaseLayout() != partition.getLayout()));
    restartMaster();
    genTable(1, 4, true);
    TableMaster tableMasterRestart = mCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(TableMaster.class);
    Table table = tableMaster.getTable(DB_NAME, tableName);
    // all partitions remain transformed
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> partition.getBaseLayout() != partition.getLayout()));
    tableMasterRestart.syncDatabase(DB_NAME);
    // The first two partitions should remain transformed, the new partitions are not transformed
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> (partition.getSpec().endsWith("0") || partition.getSpec().endsWith("1")) == (partition.getBaseLayout() != partition.getLayout())));
}

Example 3 with Status

use of alluxio.job.wire.Status in project alluxio by Alluxio.

the class PlanTracker method statusChangeCallback.

private void statusChangeCallback(PlanInfo planInfo) {
    if (planInfo == null) {
        return;
    }
    mWorkflowTracker.onPlanStatusChange(planInfo);
    Status status = planInfo.getStatus();
    if (!status.isFinished()) {
        return;
    }
    if (status.equals(Status.FAILED)) {
        while (mFailed.size() >= mCapacity) {
            mFailed.remove(mFailed.last());
        }
        mFailed.add(planInfo);
    }
    // Retry if offering to mFinished doesn't work
    for (int i = 0; i < 2; i++) {
        if (mFinished.offer(planInfo)) {
            return;
        }
        if (!removeFinished()) {
            // Still couldn't add to mFinished - remove from coordinators so that it doesn't
            // get stuck
            LOG.warn("Failed to remove any jobs from the finished queue in status change callback");
        }
    }
    if (mFinished.offer(planInfo)) {
        return;
    }
    // remove from the coordinator map preemptively so that it doesn't get lost forever even if
    // it's still within the retention time
    LOG.warn("Failed to offer job id {} to finished queue, removing from tracking preemptively", planInfo.getId());
}

Example 4 with Status

use of alluxio.job.wire.Status in project alluxio by Alluxio.

the class PinCommandMultipleMediaIntegrationTest method pinToMediumForceEviction.

@Test
public void pinToMediumForceEviction() throws Exception {
    FileSystem fileSystem = sLocalAlluxioClusterResource.get().getClient();
    FileSystemShell fsShell = new FileSystemShell(ServerConfiguration.global());
    AlluxioURI filePathA = new AlluxioURI("/testFileA");
    AlluxioURI dirPath = new AlluxioURI("/testDirA");
    AlluxioURI filePathB = new AlluxioURI(dirPath.getPath() + "/testFileB");
    AlluxioURI filePathC = new AlluxioURI("/testFileC");
    int fileSize = SIZE_BYTES / 2;
    FileSystemTestUtils.createByteFile(fileSystem, filePathA, WritePType.CACHE_THROUGH, fileSize);
    assertTrue(fileExists(fileSystem, filePathA));
    assertEquals(0, fsShell.run("pin", filePathA.toString(), "MEM"));
    URIStatus status = fileSystem.getStatus(filePathA);
    assertTrue(status.isPinned());
    assertTrue(status.getPinnedMediumTypes().contains("MEM"));
    fileSystem.createDirectory(dirPath);
    assertEquals(0, fsShell.run("pin", dirPath.toString(), "MEM"));
    FileSystemTestUtils.createByteFile(fileSystem, filePathB, WritePType.CACHE_THROUGH, fileSize);
    assertTrue(fileExists(fileSystem, filePathB));
    URIStatus statusB = fileSystem.getStatus(filePathB);
    assertTrue(statusB.isPinned());
    assertTrue(statusB.getPinnedMediumTypes().contains("MEM"));
    FileSystemTestUtils.createByteFile(fileSystem, filePathC, WritePType.THROUGH, fileSize);
    assertEquals(100, fileSystem.getStatus(filePathA).getInAlluxioPercentage());
    assertEquals(100, fileSystem.getStatus(filePathB).getInAlluxioPercentage());
    assertEquals(0, fileSystem.getStatus(filePathC).getInAlluxioPercentage());
    assertEquals(0, fsShell.run("pin", filePathC.toString(), "SSD"));
    // Verify files are replicated into the correct tier through job service
    CommonUtils.waitFor("File being loaded", () -> sJobCluster.getMaster().getJobMaster().listDetailed().stream().anyMatch(x -> x.getStatus().equals(Status.COMPLETED) && x.getName().equals("Replicate") && x.getAffectedPaths().contains(filePathC.getPath())), sWaitOptions);
    assertEquals(100, fileSystem.getStatus(filePathC).getInAlluxioPercentage());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathA).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathB).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(Constants.MEDIUM_SSD, fileSystem.getStatus(filePathC).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(0, fsShell.run("unpin", filePathA.toString()));
    assertEquals(0, fsShell.run("pin", filePathC.toString(), "MEM"));
    status = fileSystem.getStatus(filePathA);
    assertFalse(status.isPinned());
    assertTrue(status.getPinnedMediumTypes().isEmpty());
    // Verify files are migrated from another tier into the correct tier through job service
    // Also verify that eviction works
    CommonUtils.waitFor("File being moved", () -> sJobCluster.getMaster().getJobMaster().listDetailed().stream().anyMatch(x -> x.getStatus().equals(Status.COMPLETED) && x.getName().equals("Move") && x.getAffectedPaths().contains(filePathC.getPath())), sWaitOptions);
    assertEquals(0, fileSystem.getStatus(filePathA).getInAlluxioPercentage());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathC).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
}

Example 5 with Status

use of alluxio.job.wire.Status in project alluxio by Alluxio.

the class AbstractDistributedJobCommand method waitJob.

/**
 * Waits for at least one job to complete.
 */
protected void waitJob() {
    AtomicBoolean removed = new AtomicBoolean(false);
    while (true) {
        mSubmittedJobAttempts = mSubmittedJobAttempts.stream().filter((jobAttempt) -> {
            Status check = jobAttempt.check();
            switch(check) {
                case CREATED:
                case RUNNING:
                    return true;
                case CANCELED:
                case COMPLETED:
                    mCompletedCount++;
                    removed.set(true);
                    return false;
                case FAILED:
                    mFailedCount++;
                    removed.set(true);
                    return false;
                default:
                    throw new IllegalStateException(String.format("Unexpected Status: %s", check));
            }
        }).collect(Collectors.toList());
        if (removed.get()) {
            return;
        }
        CommonUtils.sleepMs(5);
    }
}