Examples with BlobClient org.apache.flink.runtime.blob.BlobClient used on opensource projects

Example 1 with BlobClient

use of org.apache.flink.runtime.blob.BlobClient in project flink by apache.

the class JobSubmitTest method testFailureWhenJarBlobsMissing.

@Test
public void testFailureWhenJarBlobsMissing() {
    try {
        // create a simple job graph
        JobVertex jobVertex = new JobVertex("Test Vertex");
        jobVertex.setInvokableClass(NoOpInvokable.class);
        JobGraph jg = new JobGraph("test job", jobVertex);
        // request the blob port from the job manager
        Future<Object> future = jmGateway.ask(JobManagerMessages.getRequestBlobManagerPort(), timeout);
        int blobPort = (Integer) Await.result(future, timeout);
        // upload two dummy bytes and add their keys to the job graph as dependencies
        BlobKey key1, key2;
        BlobClient bc = new BlobClient(new InetSocketAddress("localhost", blobPort), jmConfig);
        try {
            key1 = bc.put(new byte[10]);
            key2 = bc.put(new byte[10]);
            // delete one of the blobs to make sure that the startup failed
            bc.delete(key2);
        } finally {
            bc.close();
        }
        jg.addBlob(key1);
        jg.addBlob(key2);
        // submit the job
        Future<Object> submitFuture = jmGateway.ask(new JobManagerMessages.SubmitJob(jg, ListeningBehaviour.EXECUTION_RESULT), timeout);
        try {
            Await.result(submitFuture, timeout);
        } catch (JobExecutionException e) {
            // that is what we expect
            assertTrue(e.getCause() instanceof IOException);
        } catch (Exception e) {
            fail("Wrong exception type");
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with BlobClient

use of org.apache.flink.runtime.blob.BlobClient in project flink by apache.

the class ClientUtilsTest method uploadAndSetUserArtifacts.

@Test
public void uploadAndSetUserArtifacts() throws Exception {
    java.nio.file.Path tmpDir = temporaryFolder.newFolder().toPath();
    JobGraph jobGraph = JobGraphTestUtils.emptyJobGraph();
    Collection<DistributedCache.DistributedCacheEntry> localArtifacts = Arrays.asList(new DistributedCache.DistributedCacheEntry(Files.createFile(tmpDir.resolve("art1")).toString(), true, true), new DistributedCache.DistributedCacheEntry(Files.createFile(tmpDir.resolve("art2")).toString(), true, false), new DistributedCache.DistributedCacheEntry(Files.createFile(tmpDir.resolve("art3")).toString(), false, true), new DistributedCache.DistributedCacheEntry(Files.createFile(tmpDir.resolve("art4")).toString(), true, false));
    Collection<DistributedCache.DistributedCacheEntry> distributedArtifacts = Arrays.asList(new DistributedCache.DistributedCacheEntry("hdfs://localhost:1234/test", true, false));
    for (DistributedCache.DistributedCacheEntry entry : localArtifacts) {
        jobGraph.addUserArtifact(entry.filePath, entry);
    }
    for (DistributedCache.DistributedCacheEntry entry : distributedArtifacts) {
        jobGraph.addUserArtifact(entry.filePath, entry);
    }
    final int totalNumArtifacts = localArtifacts.size() + distributedArtifacts.size();
    assertEquals(totalNumArtifacts, jobGraph.getUserArtifacts().size());
    assertEquals(0, jobGraph.getUserArtifacts().values().stream().filter(entry -> entry.blobKey != null).count());
    ClientUtils.extractAndUploadJobGraphFiles(jobGraph, () -> new BlobClient(new InetSocketAddress("localhost", blobServer.getPort()), new Configuration()));
    assertEquals(totalNumArtifacts, jobGraph.getUserArtifacts().size());
    assertEquals(localArtifacts.size(), jobGraph.getUserArtifacts().values().stream().filter(entry -> entry.blobKey != null).count());
    assertEquals(distributedArtifacts.size(), jobGraph.getUserArtifacts().values().stream().filter(entry -> entry.blobKey == null).count());
    // 1 unique key for each local artifact, and null for distributed artifacts
    assertEquals(localArtifacts.size() + 1, jobGraph.getUserArtifacts().values().stream().map(entry -> entry.blobKey).distinct().count());
    for (DistributedCache.DistributedCacheEntry original : localArtifacts) {
        assertState(original, jobGraph.getUserArtifacts().get(original.filePath), false, jobGraph.getJobID());
    }
    for (DistributedCache.DistributedCacheEntry original : distributedArtifacts) {
        assertState(original, jobGraph.getUserArtifacts().get(original.filePath), true, jobGraph.getJobID());
    }
}

Example 3 with BlobClient

use of org.apache.flink.runtime.blob.BlobClient in project flink by apache.

the class ClientUtilsTest method uploadAndSetUserJars.

@Test
public void uploadAndSetUserJars() throws Exception {
    java.nio.file.Path tmpDir = temporaryFolder.newFolder().toPath();
    JobGraph jobGraph = JobGraphTestUtils.emptyJobGraph();
    Collection<Path> jars = Arrays.asList(new Path(Files.createFile(tmpDir.resolve("jar1.jar")).toString()), new Path(Files.createFile(tmpDir.resolve("jar2.jar")).toString()));
    jars.forEach(jobGraph::addJar);
    assertEquals(jars.size(), jobGraph.getUserJars().size());
    assertEquals(0, jobGraph.getUserJarBlobKeys().size());
    ClientUtils.extractAndUploadJobGraphFiles(jobGraph, () -> new BlobClient(new InetSocketAddress("localhost", blobServer.getPort()), new Configuration()));
    assertEquals(jars.size(), jobGraph.getUserJars().size());
    assertEquals(jars.size(), jobGraph.getUserJarBlobKeys().size());
    assertEquals(jars.size(), jobGraph.getUserJarBlobKeys().stream().distinct().count());
    for (PermanentBlobKey blobKey : jobGraph.getUserJarBlobKeys()) {
        blobServer.getFile(jobGraph.getJobID(), blobKey);
    }
}

Example 4 with BlobClient

use of org.apache.flink.runtime.blob.BlobClient in project flink by apache.

the class EmbeddedExecutor method submitJob.

private static CompletableFuture<JobID> submitJob(final Configuration configuration, final DispatcherGateway dispatcherGateway, final JobGraph jobGraph, final Time rpcTimeout) {
    checkNotNull(jobGraph);
    LOG.info("Submitting Job with JobId={}.", jobGraph.getJobID());
    return dispatcherGateway.getBlobServerPort(rpcTimeout).thenApply(blobServerPort -> new InetSocketAddress(dispatcherGateway.getHostname(), blobServerPort)).thenCompose(blobServerAddress -> {
        try {
            ClientUtils.extractAndUploadJobGraphFiles(jobGraph, () -> new BlobClient(blobServerAddress, configuration));
        } catch (FlinkException e) {
            throw new CompletionException(e);
        }
        return dispatcherGateway.submitJob(jobGraph, rpcTimeout);
    }).thenApply(ack -> jobGraph.getJobID());
}

Example 5 with BlobClient

use of org.apache.flink.runtime.blob.BlobClient in project flink by apache.

the class JobGraph method uploadRequiredJarFiles.

/**
	 * Uploads the previously added user jar file to the job manager through the job manager's BLOB server.
	 *
	 * @param serverAddress
	 *        the network address of the BLOB server
	 * @param blobClientConfig
	 *        the blob client configuration
	 * @throws IOException
	 *         thrown if an I/O error occurs during the upload
	 */
public void uploadRequiredJarFiles(InetSocketAddress serverAddress, Configuration blobClientConfig) throws IOException {
    if (this.userJars.isEmpty()) {
        return;
    }
    BlobClient bc = null;
    try {
        bc = new BlobClient(serverAddress, blobClientConfig);
        for (final Path jar : this.userJars) {
            final FileSystem fs = jar.getFileSystem();
            FSDataInputStream is = null;
            try {
                is = fs.open(jar);
                final BlobKey key = bc.put(is);
                this.userJarBlobKeys.add(key);
            } finally {
                if (is != null) {
                    is.close();
                }
            }
        }
    } finally {
        if (bc != null) {
            bc.close();
        }
    }
}