Examples with Closer org.apache.flink.shaded.guava30.com.google.common.io.Closer used on opensource projects

Example 96 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project incubator-gobblin by apache.

the class JobLauncherTestHelper method runTestWithMultipleDatasets.

public void runTestWithMultipleDatasets(Properties jobProps) throws Exception {
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    String jobId = JobLauncherUtils.newJobId(jobName).toString();
    jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId);
    jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, MultiDatasetTestSource.class.getName());
    Closer closer = Closer.create();
    try {
        JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps));
        jobLauncher.launchJob(null);
    } finally {
        closer.close();
    }
    for (int i = 0; i < 4; i++) {
        List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, "Dataset" + i + "-current.jst");
        DatasetState datasetState = datasetStateList.get(0);
        Assert.assertEquals(datasetState.getDatasetUrn(), "Dataset" + i);
        Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED);
        Assert.assertEquals(datasetState.getCompletedTasks(), 1);
        Assert.assertEquals(datasetState.getJobFailures(), 0);
        for (TaskState taskState : datasetState.getTaskStates()) {
            Assert.assertEquals(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY), "Dataset" + i);
            Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED);
            Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS);
        }
    }
}

Example 97 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project incubator-gobblin by apache.

the class JobLauncherTestHelper method runTestWithCancellation.

public void runTestWithCancellation(final Properties jobProps) throws Exception {
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    String jobId = JobLauncherUtils.newJobId(jobName).toString();
    jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId);
    Closer closer = Closer.create();
    try {
        final JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps));
        final AtomicBoolean isCancelled = new AtomicBoolean(false);
        // This thread will cancel the job after some time
        Thread thread = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    Thread.sleep(500);
                    jobLauncher.cancelJob(null);
                    isCancelled.set(true);
                } catch (Exception je) {
                // Ignored
                }
            }
        });
        thread.start();
        jobLauncher.launchJob(null);
        Assert.assertTrue(isCancelled.get());
    } finally {
        closer.close();
    }
    List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst");
    Assert.assertTrue(datasetStateList.isEmpty());
}

Example 98 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project incubator-gobblin by apache.

the class JobLauncherTestHelper method runTestWithSkippedTask.

/**
 * Test when a test with the matching suffix is skipped.
 * @param jobProps job properties
 * @param skippedTaskSuffix the suffix for the task that is skipped
 */
public void runTestWithSkippedTask(Properties jobProps, String skippedTaskSuffix) throws Exception {
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    String jobId = JobLauncherUtils.newJobId(jobName).toString();
    jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId);
    jobProps.setProperty(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.FALSE.toString());
    jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "successful");
    jobProps.setProperty(ConfigurationKeys.MAX_TASK_RETRIES_KEY, "0");
    Closer closer = Closer.create();
    try {
        JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps));
        jobLauncher.launchJob(null);
    } finally {
        closer.close();
    }
    List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst");
    JobState jobState = datasetStateList.get(0);
    Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED);
    // one task is skipped out of 4
    Assert.assertEquals(jobState.getCompletedTasks(), 3);
    for (TaskState taskState : jobState.getTaskStates()) {
        if (taskState.getTaskId().endsWith(skippedTaskSuffix)) {
            Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.PENDING);
        } else {
            Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED);
            Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS);
        }
    }
}

Example 99 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project incubator-gobblin by apache.

the class JobStateTest method testSerDe.

@Test(dependsOnMethods = { "testSetAndGet" })
public void testSerDe() throws IOException {
    Closer closer = Closer.create();
    try {
        ByteArrayOutputStream baos = closer.register(new ByteArrayOutputStream());
        DataOutputStream dos = closer.register(new DataOutputStream(baos));
        this.jobState.write(dos);
        ByteArrayInputStream bais = closer.register((new ByteArrayInputStream(baos.toByteArray())));
        DataInputStream dis = closer.register((new DataInputStream(bais)));
        JobState newJobState = new JobState();
        newJobState.readFields(dis);
        doAsserts(newJobState, true, false);
    } catch (Throwable t) {
        throw closer.rethrow(t);
    } finally {
        closer.close();
    }
}

Example 100 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project incubator-gobblin by apache.

the class LegacyJobLockFactoryManagerTest method testInvalidJobLockType_ThrowsException.

@Test(expectedExceptions = { JobLockException.class })
public void testInvalidJobLockType_ThrowsException() throws JobLockException, IOException {
    Closer closer = Closer.create();
    try {
        Properties properties = new Properties();
        properties.setProperty(ConfigurationKeys.JOB_LOCK_TYPE, "ThisIsATest");
        JobLock jobLock = closer.register(LegacyJobLockFactoryManager.getJobLock(properties, new JobLockEventListener()));
        MatcherAssert.assertThat(jobLock, Matchers.instanceOf(FileBasedJobLock.class));
    } finally {
        closer.close();
    }
}