Examples with Closer org.apache.flink.shaded.guava30.com.google.common.io.Closer used on opensource projects

Example 41 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project incubator-gobblin by apache.

the class JobLauncherTestHelper method runTestWithSkippedTask.

/**
 * Test when a test with the matching suffix is skipped.
 * @param jobProps job properties
 * @param skippedTaskSuffix the suffix for the task that is skipped
 */
public void runTestWithSkippedTask(Properties jobProps, String skippedTaskSuffix) throws Exception {
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    String jobId = JobLauncherUtils.newJobId(jobName).toString();
    jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId);
    jobProps.setProperty(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.FALSE.toString());
    jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "successful");
    jobProps.setProperty(ConfigurationKeys.MAX_TASK_RETRIES_KEY, "0");
    Closer closer = Closer.create();
    try {
        JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps));
        jobLauncher.launchJob(null);
    } finally {
        closer.close();
    }
    List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst");
    JobState jobState = datasetStateList.get(0);
    Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED);
    // one task is skipped out of 4
    Assert.assertEquals(jobState.getCompletedTasks(), 3);
    for (TaskState taskState : jobState.getTaskStates()) {
        if (taskState.getTaskId().endsWith(skippedTaskSuffix)) {
            Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.PENDING);
        } else {
            Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED);
            Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS);
        }
    }
}

Example 42 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project incubator-gobblin by apache.

the class JobLauncherTestHelper method runTestWithMultipleDatasets.

public void runTestWithMultipleDatasets(Properties jobProps) throws Exception {
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    String jobId = JobLauncherUtils.newJobId(jobName).toString();
    jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId);
    jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, MultiDatasetTestSource.class.getName());
    Closer closer = Closer.create();
    try {
        JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps));
        jobLauncher.launchJob(null);
    } finally {
        closer.close();
    }
    for (int i = 0; i < 4; i++) {
        List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, "Dataset" + i + "-current.jst");
        DatasetState datasetState = datasetStateList.get(0);
        Assert.assertEquals(datasetState.getDatasetUrn(), "Dataset" + i);
        Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED);
        Assert.assertEquals(datasetState.getCompletedTasks(), 1);
        Assert.assertEquals(datasetState.getJobFailures(), 0);
        for (TaskState taskState : datasetState.getTaskStates()) {
            Assert.assertEquals(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY), "Dataset" + i);
            Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED);
            Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS);
        }
    }
}

Example 43 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project incubator-gobblin by apache.

the class JobLauncherTestHelper method runTestWithCancellation.

public void runTestWithCancellation(final Properties jobProps) throws Exception {
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    String jobId = JobLauncherUtils.newJobId(jobName).toString();
    jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId);
    Closer closer = Closer.create();
    try {
        final JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps));
        final AtomicBoolean isCancelled = new AtomicBoolean(false);
        // This thread will cancel the job after some time
        Thread thread = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    Thread.sleep(500);
                    jobLauncher.cancelJob(null);
                    isCancelled.set(true);
                } catch (Exception je) {
                // Ignored
                }
            }
        });
        thread.start();
        jobLauncher.launchJob(null);
        Assert.assertTrue(isCancelled.get());
    } finally {
        closer.close();
    }
    List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst");
    Assert.assertTrue(datasetStateList.isEmpty());
}

Example 44 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project incubator-gobblin by apache.

the class JobStateTest method testSerDe.

@Test(dependsOnMethods = { "testSetAndGet" })
public void testSerDe() throws IOException {
    Closer closer = Closer.create();
    try {
        ByteArrayOutputStream baos = closer.register(new ByteArrayOutputStream());
        DataOutputStream dos = closer.register(new DataOutputStream(baos));
        this.jobState.write(dos);
        ByteArrayInputStream bais = closer.register((new ByteArrayInputStream(baos.toByteArray())));
        DataInputStream dis = closer.register((new DataInputStream(bais)));
        JobState newJobState = new JobState();
        newJobState.readFields(dis);
        doAsserts(newJobState, true, false);
    } catch (Throwable t) {
        throw closer.rethrow(t);
    } finally {
        closer.close();
    }
}

Example 45 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project incubator-gobblin by apache.

the class JobLockTest method testLocalJobLock.

public void testLocalJobLock() throws Exception {
    Closer closer = Closer.create();
    try {
        // Set to true or false to enable debug logging in the threads
        final AtomicBoolean debugEnabled = new AtomicBoolean(true);
        final JobLock lock = closer.register(getJobLock());
        final CountDownLatch numTestsToPass = new CountDownLatch(2);
        final Lock stepsLock = new ReentrantLock();
        final AtomicBoolean thread1Locked = new AtomicBoolean(false);
        final AtomicBoolean thread2Locked = new AtomicBoolean(false);
        final Condition thread1Done = stepsLock.newCondition();
        final Condition thread2Done = stepsLock.newCondition();
        Thread thread1 = new Thread(new Runnable() {

            @Override
            public void run() {
                final Logger log = LoggerFactory.getLogger("testLocalJobLock.thread1");
                if (debugEnabled.get()) {
                    org.apache.log4j.Logger.getLogger(log.getName()).setLevel(Level.DEBUG);
                }
                try {
                    stepsLock.lock();
                    try {
                        log.debug("Acquire the lock");
                        Assert.assertTrue(lock.tryLock());
                        thread1Locked.set(true);
                        log.debug("Notify thread2 to check the lock");
                        thread1Done.signal();
                        log.debug("Wait for thread2 to check the lock");
                        thread2Done.await();
                        log.debug("Release the file lock");
                        lock.unlock();
                        thread1Locked.set(false);
                        log.debug("Notify and wait for thread2 to acquired the lock");
                        thread1Done.signal();
                        while (!thread2Locked.get()) thread2Done.await();
                        Assert.assertFalse(lock.tryLock());
                        log.debug("Notify thread2 that we are done with the check");
                        thread1Done.signal();
                    } finally {
                        stepsLock.unlock();
                    }
                    numTestsToPass.countDown();
                } catch (Exception e) {
                    log.error("error: " + e, e);
                }
            }
        }, "testLocalJobLock.thread1");
        thread1.setDaemon(true);
        thread1.start();
        Thread thread2 = new Thread(new Runnable() {

            @Override
            public void run() {
                final Logger log = LoggerFactory.getLogger("testLocalJobLock.thread2");
                if (debugEnabled.get()) {
                    org.apache.log4j.Logger.getLogger(log.getName()).setLevel(Level.DEBUG);
                }
                try {
                    stepsLock.lock();
                    try {
                        log.debug("Wait for thread1 to acquire the lock and verify we can't acquire it.");
                        while (!thread1Locked.get()) thread1Done.await();
                        Assert.assertFalse(lock.tryLock());
                        log.debug("Notify thread1 that we are done with the check.");
                        thread2Done.signal();
                        log.debug("Wait for thread1 to release the lock and try to acquire it.");
                        while (thread1Locked.get()) thread1Done.await();
                        Assert.assertTrue(lock.tryLock());
                        thread2Locked.set(true);
                        thread2Done.signal();
                        log.debug("Wait for thread1 to check the lock");
                        thread1Done.await();
                        // clean up the file lock
                        lock.unlock();
                    } finally {
                        stepsLock.unlock();
                    }
                    lock.unlock();
                    numTestsToPass.countDown();
                } catch (Exception e) {
                    log.error("error: " + e, e);
                }
            }
        }, "testLocalJobLock.thread2");
        thread2.setDaemon(true);
        thread2.start();
        // Wait for some time for the threads to die.
        Assert.assertTrue(numTestsToPass.await(30, TimeUnit.SECONDS));
    } finally {
        closer.close();
    }
}