Examples with PermanentBlobKey org.apache.flink.runtime.blob.PermanentBlobKey used on opensource projects

Example 6 with PermanentBlobKey

use of org.apache.flink.runtime.blob.PermanentBlobKey in project flink by apache.

the class ClientUtilsTest method uploadAndSetUserJars.

@Test
public void uploadAndSetUserJars() throws Exception {
    java.nio.file.Path tmpDir = temporaryFolder.newFolder().toPath();
    JobGraph jobGraph = JobGraphTestUtils.emptyJobGraph();
    Collection<Path> jars = Arrays.asList(new Path(Files.createFile(tmpDir.resolve("jar1.jar")).toString()), new Path(Files.createFile(tmpDir.resolve("jar2.jar")).toString()));
    jars.forEach(jobGraph::addJar);
    assertEquals(jars.size(), jobGraph.getUserJars().size());
    assertEquals(0, jobGraph.getUserJarBlobKeys().size());
    ClientUtils.extractAndUploadJobGraphFiles(jobGraph, () -> new BlobClient(new InetSocketAddress("localhost", blobServer.getPort()), new Configuration()));
    assertEquals(jars.size(), jobGraph.getUserJars().size());
    assertEquals(jars.size(), jobGraph.getUserJarBlobKeys().size());
    assertEquals(jars.size(), jobGraph.getUserJarBlobKeys().stream().distinct().count());
    for (PermanentBlobKey blobKey : jobGraph.getUserJarBlobKeys()) {
        blobServer.getFile(jobGraph.getJobID(), blobKey);
    }
}

Example 7 with PermanentBlobKey

use of org.apache.flink.runtime.blob.PermanentBlobKey in project flink by apache.

the class JobSubmissionFailsITCase method getJobGraphWithMissingBlobKey.

@Nonnull
private static JobGraph getJobGraphWithMissingBlobKey() {
    final JobGraph jobGraph = getWorkingJobGraph();
    jobGraph.addUserJarBlobKey(new PermanentBlobKey());
    return jobGraph;
}

Example 8 with PermanentBlobKey

use of org.apache.flink.runtime.blob.PermanentBlobKey in project flink by apache.

the class BlobsCleanupITCase method testBlobServerCleanup.

private void testBlobServerCleanup(final TestCase testCase) throws Exception {
    final MiniCluster miniCluster = miniClusterResource.getMiniCluster();
    final int numTasks = 2;
    final Deadline timeout = Deadline.fromNow(Duration.ofSeconds(30L));
    final JobGraph jobGraph = createJobGraph(testCase, numTasks);
    final JobID jid = jobGraph.getJobID();
    // upload a blob
    final File tempBlob = File.createTempFile("Required", ".jar");
    final int blobPort = miniCluster.getClusterInformation().getBlobServerPort();
    List<PermanentBlobKey> keys = BlobClient.uploadFiles(new InetSocketAddress("localhost", blobPort), configuration, jid, Collections.singletonList(new Path(tempBlob.getAbsolutePath())));
    assertThat(keys, hasSize(1));
    jobGraph.addUserJarBlobKey(keys.get(0));
    if (testCase == TestCase.JOB_SUBMISSION_FAILS) {
        // add an invalid key so that the submission fails
        jobGraph.addUserJarBlobKey(new PermanentBlobKey());
    }
    final CompletableFuture<JobSubmissionResult> submissionFuture = miniCluster.submitJob(jobGraph);
    if (testCase == TestCase.JOB_SUBMISSION_FAILS) {
        try {
            submissionFuture.get();
            fail("Expected job submission failure.");
        } catch (ExecutionException e) {
            assertThat(ExceptionUtils.findThrowable(e, JobSubmissionException.class).isPresent(), is(true));
        }
    } else {
        final JobSubmissionResult jobSubmissionResult = submissionFuture.get();
        assertThat(jobSubmissionResult.getJobID(), is(jid));
        final CompletableFuture<JobResult> resultFuture = miniCluster.requestJobResult(jid);
        if (testCase == TestCase.JOB_FAILS) {
            // fail a task so that the job is going to be recovered (we actually do not
            // need the blocking part of the invokable and can start throwing right away)
            FailingBlockingInvokable.unblock();
            // job will get restarted, BlobCache may re-download the BLOB if already deleted
            // then the tasks will fail again and the restart strategy will finalise the job
            final JobResult jobResult = resultFuture.get();
            assertThat(jobResult.isSuccess(), is(false));
            assertThat(jobResult.getApplicationStatus(), is(ApplicationStatus.FAILED));
        } else if (testCase == TestCase.JOB_IS_CANCELLED) {
            miniCluster.cancelJob(jid);
            final JobResult jobResult = resultFuture.get();
            assertThat(jobResult.isSuccess(), is(false));
            assertThat(jobResult.getApplicationStatus(), is(ApplicationStatus.CANCELED));
        } else {
            final JobResult jobResult = resultFuture.get();
            Throwable cause = jobResult.getSerializedThrowable().map(throwable -> throwable.deserializeError(getClass().getClassLoader())).orElse(null);
            assertThat(ExceptionUtils.stringifyException(cause), jobResult.isSuccess(), is(true));
        }
    }
    // both BlobServer and BlobCache should eventually delete all files
    File[] blobDirs = blobBaseDir.listFiles((dir, name) -> name.startsWith("blobStore-"));
    assertNotNull(blobDirs);
    for (File blobDir : blobDirs) {
        waitForEmptyBlobDir(blobDir, timeout.timeLeft());
    }
}

Example 9 with PermanentBlobKey

use of org.apache.flink.runtime.blob.PermanentBlobKey in project flink by apache.

the class DefaultExecutionGraphDeploymentWithBlobServerTest method setupBlobServer.

@Before
public void setupBlobServer() throws IOException {
    Configuration config = new Configuration();
    // always offload the serialized job and task information
    config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
    blobServer = Mockito.spy(new BlobServer(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore()));
    blobWriter = blobServer;
    blobCache = blobServer;
    seenHashes.clear();
    // verify that we do not upload the same content more than once
    doAnswer(invocation -> {
        PermanentBlobKey key = (PermanentBlobKey) invocation.callRealMethod();
        assertTrue(seenHashes.add(key.getHash()));
        return key;
    }).when(blobServer).putPermanent(any(JobID.class), Matchers.<byte[]>any());
    blobServer.start();
}

Example 10 with PermanentBlobKey

use of org.apache.flink.runtime.blob.PermanentBlobKey in project flink by apache.

the class JobGraphTest method testSetUserArtifactBlobKey.

@Test
public void testSetUserArtifactBlobKey() throws IOException, ClassNotFoundException {
    JobGraph jb = JobGraphTestUtils.emptyJobGraph();
    final DistributedCache.DistributedCacheEntry[] entries = { new DistributedCache.DistributedCacheEntry("p1", true, true), new DistributedCache.DistributedCacheEntry("p2", true, false), new DistributedCache.DistributedCacheEntry("p3", false, true), new DistributedCache.DistributedCacheEntry("p4", true, false) };
    for (DistributedCache.DistributedCacheEntry entry : entries) {
        jb.addUserArtifact(entry.filePath, entry);
    }
    for (DistributedCache.DistributedCacheEntry entry : entries) {
        PermanentBlobKey blobKey = new PermanentBlobKey();
        jb.setUserArtifactBlobKey(entry.filePath, blobKey);
        DistributedCache.DistributedCacheEntry jobGraphEntry = jb.getUserArtifacts().get(entry.filePath);
        assertNotNull(jobGraphEntry);
        assertEquals(blobKey, InstantiationUtil.deserializeObject(jobGraphEntry.blobKey, ClassLoader.getSystemClassLoader(), false));
        assertEquals(entry.isExecutable, jobGraphEntry.isExecutable);
        assertEquals(entry.isZipped, jobGraphEntry.isZipped);
        assertEquals(entry.filePath, jobGraphEntry.filePath);
    }
}