use of org.apache.flink.runtime.blob.PermanentBlobCache in project flink by apache.
the class BlobLibraryCacheManagerTest method testLibraryCacheManagerCleanup.
/**
* Tests that the {@link BlobLibraryCacheManager} cleans up after all class loader leases for a
* single job a closed.
*/
@Test
public void testLibraryCacheManagerCleanup() throws Exception {
JobID jobId = new JobID();
List<PermanentBlobKey> keys = new ArrayList<>();
BlobServer server = null;
PermanentBlobCache cache = null;
BlobLibraryCacheManager libCache = null;
final byte[] buf = new byte[128];
try {
Configuration config = new Configuration();
config.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);
server = new BlobServer(config, temporaryFolder.newFolder(), new VoidBlobStore());
server.start();
InetSocketAddress serverAddress = new InetSocketAddress("localhost", server.getPort());
cache = new PermanentBlobCache(config, temporaryFolder.newFolder(), new VoidBlobStore(), serverAddress);
keys.add(server.putPermanent(jobId, buf));
buf[0] += 1;
keys.add(server.putPermanent(jobId, buf));
libCache = createBlobLibraryCacheManager(cache);
cache.registerJob(jobId);
assertEquals(0, libCache.getNumberOfManagedJobs());
assertEquals(0, libCache.getNumberOfReferenceHolders(jobId));
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(0, jobId, cache);
final LibraryCacheManager.ClassLoaderLease classLoaderLease1 = libCache.registerClassLoaderLease(jobId);
UserCodeClassLoader classLoader1 = classLoaderLease1.getOrResolveClassLoader(keys, Collections.emptyList());
assertEquals(1, libCache.getNumberOfManagedJobs());
assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
assertEquals(2, checkFilesExist(jobId, keys, cache, true));
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(2, jobId, cache);
final LibraryCacheManager.ClassLoaderLease classLoaderLease2 = libCache.registerClassLoaderLease(jobId);
final UserCodeClassLoader classLoader2 = classLoaderLease2.getOrResolveClassLoader(keys, Collections.emptyList());
assertThat(classLoader1, sameInstance(classLoader2));
try {
classLoaderLease1.getOrResolveClassLoader(Collections.emptyList(), Collections.emptyList());
fail("Should fail with an IllegalStateException");
} catch (IllegalStateException e) {
// that's what we want
}
try {
classLoaderLease1.getOrResolveClassLoader(keys, Collections.singletonList(new URL("file:///tmp/does-not-exist")));
fail("Should fail with an IllegalStateException");
} catch (IllegalStateException e) {
// that's what we want
}
assertEquals(1, libCache.getNumberOfManagedJobs());
assertEquals(2, libCache.getNumberOfReferenceHolders(jobId));
assertEquals(2, checkFilesExist(jobId, keys, cache, true));
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(2, jobId, cache);
classLoaderLease1.release();
assertEquals(1, libCache.getNumberOfManagedJobs());
assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
assertEquals(2, checkFilesExist(jobId, keys, cache, true));
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(2, jobId, cache);
classLoaderLease2.release();
assertEquals(0, libCache.getNumberOfManagedJobs());
assertEquals(0, libCache.getNumberOfReferenceHolders(jobId));
assertEquals(2, checkFilesExist(jobId, keys, cache, true));
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(2, jobId, cache);
// only PermanentBlobCache#releaseJob() calls clean up files (tested in
// BlobCacheCleanupTest etc.
} finally {
if (libCache != null) {
libCache.shutdown();
}
// should have been closed by the libraryCacheManager, but just in case
if (cache != null) {
cache.close();
}
if (server != null) {
server.close();
}
}
}
use of org.apache.flink.runtime.blob.PermanentBlobCache in project flink by apache.
the class BlobLibraryCacheManagerTest method testRegisterAndDownload.
@Test
public void testRegisterAndDownload() throws IOException {
// setWritable doesn't work on Windows.
assumeTrue(!OperatingSystem.isWindows());
JobID jobId = new JobID();
BlobServer server = null;
PermanentBlobCache cache = null;
BlobLibraryCacheManager libCache = null;
File cacheDir = null;
try {
// create the blob transfer services
Configuration config = new Configuration();
config.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1_000_000L);
server = new BlobServer(config, temporaryFolder.newFolder(), new VoidBlobStore());
server.start();
InetSocketAddress serverAddress = new InetSocketAddress("localhost", server.getPort());
cache = new PermanentBlobCache(config, temporaryFolder.newFolder(), new VoidBlobStore(), serverAddress);
// upload some meaningless data to the server
PermanentBlobKey dataKey1 = server.putPermanent(jobId, new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 });
PermanentBlobKey dataKey2 = server.putPermanent(jobId, new byte[] { 11, 12, 13, 14, 15, 16, 17, 18 });
libCache = createBlobLibraryCacheManager(cache);
assertEquals(0, libCache.getNumberOfManagedJobs());
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(0, jobId, cache);
// first try to access a non-existing entry
assertEquals(0, libCache.getNumberOfReferenceHolders(new JobID()));
// register some BLOBs as libraries
{
Collection<PermanentBlobKey> keys = Collections.singleton(dataKey1);
cache.registerJob(jobId);
final LibraryCacheManager.ClassLoaderLease classLoaderLease1 = libCache.registerClassLoaderLease(jobId);
final UserCodeClassLoader classLoader1 = classLoaderLease1.getOrResolveClassLoader(keys, Collections.emptyList());
assertEquals(1, libCache.getNumberOfManagedJobs());
assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
assertEquals(1, checkFilesExist(jobId, keys, cache, true));
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(1, jobId, cache);
final LibraryCacheManager.ClassLoaderLease classLoaderLease2 = libCache.registerClassLoaderLease(jobId);
final UserCodeClassLoader classLoader2 = classLoaderLease2.getOrResolveClassLoader(keys, Collections.emptyList());
assertThat(classLoader1, sameInstance(classLoader2));
assertEquals(1, libCache.getNumberOfManagedJobs());
assertEquals(2, libCache.getNumberOfReferenceHolders(jobId));
assertEquals(1, checkFilesExist(jobId, keys, cache, true));
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(1, jobId, cache);
// un-register the job
classLoaderLease1.release();
// still one task
assertEquals(1, libCache.getNumberOfManagedJobs());
assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
assertEquals(1, checkFilesExist(jobId, keys, cache, true));
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(1, jobId, cache);
// unregister the task registration
classLoaderLease2.release();
assertEquals(0, libCache.getNumberOfManagedJobs());
assertEquals(0, libCache.getNumberOfReferenceHolders(jobId));
// changing the libCache registration does not influence the BLOB stores...
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(1, jobId, cache);
cache.releaseJob(jobId);
// library is still cached (but not associated with job any more)
checkFileCountForJob(2, jobId, server);
checkFileCountForJob(1, jobId, cache);
}
// see BlobUtils for the directory layout
cacheDir = cache.getStorageLocation(jobId, new PermanentBlobKey()).getParentFile();
assertTrue(cacheDir.exists());
// make sure no further blobs can be downloaded by removing the write
// permissions from the directory
assertTrue("Could not remove write permissions from cache directory", cacheDir.setWritable(false, false));
// since we cannot download this library any more, this call should fail
try {
cache.registerJob(jobId);
final LibraryCacheManager.ClassLoaderLease classLoaderLease = libCache.registerClassLoaderLease(jobId);
classLoaderLease.getOrResolveClassLoader(Collections.singleton(dataKey2), Collections.emptyList());
fail("This should fail with an IOException");
} catch (IOException e) {
// splendid!
cache.releaseJob(jobId);
}
} finally {
if (cacheDir != null) {
if (!cacheDir.setWritable(true, false)) {
System.err.println("Could not re-add write permissions to cache directory.");
}
}
if (cache != null) {
cache.close();
}
if (libCache != null) {
libCache.shutdown();
}
if (server != null) {
server.close();
}
}
}
use of org.apache.flink.runtime.blob.PermanentBlobCache in project flink by apache.
the class DefaultExecutionGraphDeploymentWithBlobCacheTest method setupBlobServer.
@Before
@Override
public void setupBlobServer() throws IOException {
Configuration config = new Configuration();
// always offload the serialized job and task information
config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
blobServer = new BlobServer(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore());
blobServer.start();
blobWriter = blobServer;
InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort());
blobCache = new PermanentBlobCache(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore(), serverAddress);
}
use of org.apache.flink.runtime.blob.PermanentBlobCache in project flink by apache.
the class DefaultExecutionGraphDeploymentWithSmallBlobCacheSizeLimitTest method testDeployMultipleTasksWithSmallBlobCacheSizeLimit.
/**
* Test the deployment works well even the size limit of {@link BlobCacheSizeTracker} in {@link
* PermanentBlobCache} is set to the minimum value.
*
* <p>In this extreme case, since the size limit is 1, every time a task is deployed, all the
* existing **tracked** BLOBs on the cache must be untracked and deleted before the new BLOB is
* stored onto the cache.
*
* <p>This extreme case covers the situation of the normal case, where the size limit is much
* larger than 1 and the deletion won't happen so frequently.
*/
@Test
public void testDeployMultipleTasksWithSmallBlobCacheSizeLimit() throws Exception {
final int numberOfVertices = 4;
final int parallelism = 10;
final ExecutionGraph eg = createAndSetupExecutionGraph(numberOfVertices, parallelism);
final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
final BlockingQueue<TaskDeploymentDescriptor> tdds = new ArrayBlockingQueue<>(numberOfVertices * parallelism);
taskManagerGateway.setSubmitConsumer(FunctionUtils.uncheckedConsumer(taskDeploymentDescriptor -> {
taskDeploymentDescriptor.loadBigData(blobCache);
tdds.offer(taskDeploymentDescriptor);
}));
for (ExecutionJobVertex ejv : eg.getVerticesTopologically()) {
for (ExecutionVertex ev : ejv.getTaskVertices()) {
assertEquals(ExecutionState.CREATED, ev.getExecutionState());
LogicalSlot slot = new TestingLogicalSlotBuilder().setTaskManagerGateway(taskManagerGateway).createTestingLogicalSlot();
final Execution execution = ev.getCurrentExecutionAttempt();
execution.transitionState(ExecutionState.SCHEDULED);
execution.registerProducedPartitions(slot.getTaskManagerLocation(), true).get();
ev.deployToSlot(slot);
assertEquals(ExecutionState.DEPLOYING, ev.getExecutionState());
TaskDeploymentDescriptor tdd = tdds.take();
assertNotNull(tdd);
List<InputGateDeploymentDescriptor> igdds = tdd.getInputGates();
assertEquals(ev.getAllConsumedPartitionGroups().size(), igdds.size());
if (igdds.size() > 0) {
checkShuffleDescriptors(igdds.get(0), ev.getConsumedPartitionGroup(0));
}
}
}
}
use of org.apache.flink.runtime.blob.PermanentBlobCache in project flink by apache.
the class DefaultExecutionGraphDeploymentWithSmallBlobCacheSizeLimitTest method setupBlobServer.
@Before
@Override
public void setupBlobServer() throws IOException {
Configuration config = new Configuration();
// Always offload the serialized JobInformation, TaskInformation and cached
// ShuffleDescriptors
config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
blobServer = new BlobServer(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore());
blobServer.start();
blobWriter = blobServer;
InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort());
// Set the size limit of the blob cache to 1
BlobCacheSizeTracker blobCacheSizeTracker = new BlobCacheSizeTracker(1L);
blobCache = new PermanentBlobCache(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore(), serverAddress, blobCacheSizeTracker);
}
Aggregations