Search in sources :

Example 1 with PermanentBlobCache

use of org.apache.flink.runtime.blob.PermanentBlobCache in project flink by apache.

the class BlobLibraryCacheManagerTest method testLibraryCacheManagerCleanup.

/**
 * Tests that the {@link BlobLibraryCacheManager} cleans up after all class loader leases for a
 * single job a closed.
 */
@Test
public void testLibraryCacheManagerCleanup() throws Exception {
    JobID jobId = new JobID();
    List<PermanentBlobKey> keys = new ArrayList<>();
    BlobServer server = null;
    PermanentBlobCache cache = null;
    BlobLibraryCacheManager libCache = null;
    final byte[] buf = new byte[128];
    try {
        Configuration config = new Configuration();
        config.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);
        server = new BlobServer(config, temporaryFolder.newFolder(), new VoidBlobStore());
        server.start();
        InetSocketAddress serverAddress = new InetSocketAddress("localhost", server.getPort());
        cache = new PermanentBlobCache(config, temporaryFolder.newFolder(), new VoidBlobStore(), serverAddress);
        keys.add(server.putPermanent(jobId, buf));
        buf[0] += 1;
        keys.add(server.putPermanent(jobId, buf));
        libCache = createBlobLibraryCacheManager(cache);
        cache.registerJob(jobId);
        assertEquals(0, libCache.getNumberOfManagedJobs());
        assertEquals(0, libCache.getNumberOfReferenceHolders(jobId));
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(0, jobId, cache);
        final LibraryCacheManager.ClassLoaderLease classLoaderLease1 = libCache.registerClassLoaderLease(jobId);
        UserCodeClassLoader classLoader1 = classLoaderLease1.getOrResolveClassLoader(keys, Collections.emptyList());
        assertEquals(1, libCache.getNumberOfManagedJobs());
        assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
        assertEquals(2, checkFilesExist(jobId, keys, cache, true));
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(2, jobId, cache);
        final LibraryCacheManager.ClassLoaderLease classLoaderLease2 = libCache.registerClassLoaderLease(jobId);
        final UserCodeClassLoader classLoader2 = classLoaderLease2.getOrResolveClassLoader(keys, Collections.emptyList());
        assertThat(classLoader1, sameInstance(classLoader2));
        try {
            classLoaderLease1.getOrResolveClassLoader(Collections.emptyList(), Collections.emptyList());
            fail("Should fail with an IllegalStateException");
        } catch (IllegalStateException e) {
        // that's what we want
        }
        try {
            classLoaderLease1.getOrResolveClassLoader(keys, Collections.singletonList(new URL("file:///tmp/does-not-exist")));
            fail("Should fail with an IllegalStateException");
        } catch (IllegalStateException e) {
        // that's what we want
        }
        assertEquals(1, libCache.getNumberOfManagedJobs());
        assertEquals(2, libCache.getNumberOfReferenceHolders(jobId));
        assertEquals(2, checkFilesExist(jobId, keys, cache, true));
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(2, jobId, cache);
        classLoaderLease1.release();
        assertEquals(1, libCache.getNumberOfManagedJobs());
        assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
        assertEquals(2, checkFilesExist(jobId, keys, cache, true));
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(2, jobId, cache);
        classLoaderLease2.release();
        assertEquals(0, libCache.getNumberOfManagedJobs());
        assertEquals(0, libCache.getNumberOfReferenceHolders(jobId));
        assertEquals(2, checkFilesExist(jobId, keys, cache, true));
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(2, jobId, cache);
    // only PermanentBlobCache#releaseJob() calls clean up files (tested in
    // BlobCacheCleanupTest etc.
    } finally {
        if (libCache != null) {
            libCache.shutdown();
        }
        // should have been closed by the libraryCacheManager, but just in case
        if (cache != null) {
            cache.close();
        }
        if (server != null) {
            server.close();
        }
    }
}
Also used : Configuration(org.apache.flink.configuration.Configuration) InetSocketAddress(java.net.InetSocketAddress) ArrayList(java.util.ArrayList) URL(java.net.URL) UserCodeClassLoader(org.apache.flink.util.UserCodeClassLoader) VoidBlobStore(org.apache.flink.runtime.blob.VoidBlobStore) PermanentBlobCache(org.apache.flink.runtime.blob.PermanentBlobCache) PermanentBlobKey(org.apache.flink.runtime.blob.PermanentBlobKey) BlobServer(org.apache.flink.runtime.blob.BlobServer) JobID(org.apache.flink.api.common.JobID) Test(org.junit.Test)

Example 2 with PermanentBlobCache

use of org.apache.flink.runtime.blob.PermanentBlobCache in project flink by apache.

the class BlobLibraryCacheManagerTest method testRegisterAndDownload.

@Test
public void testRegisterAndDownload() throws IOException {
    // setWritable doesn't work on Windows.
    assumeTrue(!OperatingSystem.isWindows());
    JobID jobId = new JobID();
    BlobServer server = null;
    PermanentBlobCache cache = null;
    BlobLibraryCacheManager libCache = null;
    File cacheDir = null;
    try {
        // create the blob transfer services
        Configuration config = new Configuration();
        config.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1_000_000L);
        server = new BlobServer(config, temporaryFolder.newFolder(), new VoidBlobStore());
        server.start();
        InetSocketAddress serverAddress = new InetSocketAddress("localhost", server.getPort());
        cache = new PermanentBlobCache(config, temporaryFolder.newFolder(), new VoidBlobStore(), serverAddress);
        // upload some meaningless data to the server
        PermanentBlobKey dataKey1 = server.putPermanent(jobId, new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 });
        PermanentBlobKey dataKey2 = server.putPermanent(jobId, new byte[] { 11, 12, 13, 14, 15, 16, 17, 18 });
        libCache = createBlobLibraryCacheManager(cache);
        assertEquals(0, libCache.getNumberOfManagedJobs());
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(0, jobId, cache);
        // first try to access a non-existing entry
        assertEquals(0, libCache.getNumberOfReferenceHolders(new JobID()));
        // register some BLOBs as libraries
        {
            Collection<PermanentBlobKey> keys = Collections.singleton(dataKey1);
            cache.registerJob(jobId);
            final LibraryCacheManager.ClassLoaderLease classLoaderLease1 = libCache.registerClassLoaderLease(jobId);
            final UserCodeClassLoader classLoader1 = classLoaderLease1.getOrResolveClassLoader(keys, Collections.emptyList());
            assertEquals(1, libCache.getNumberOfManagedJobs());
            assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
            assertEquals(1, checkFilesExist(jobId, keys, cache, true));
            checkFileCountForJob(2, jobId, server);
            checkFileCountForJob(1, jobId, cache);
            final LibraryCacheManager.ClassLoaderLease classLoaderLease2 = libCache.registerClassLoaderLease(jobId);
            final UserCodeClassLoader classLoader2 = classLoaderLease2.getOrResolveClassLoader(keys, Collections.emptyList());
            assertThat(classLoader1, sameInstance(classLoader2));
            assertEquals(1, libCache.getNumberOfManagedJobs());
            assertEquals(2, libCache.getNumberOfReferenceHolders(jobId));
            assertEquals(1, checkFilesExist(jobId, keys, cache, true));
            checkFileCountForJob(2, jobId, server);
            checkFileCountForJob(1, jobId, cache);
            // un-register the job
            classLoaderLease1.release();
            // still one task
            assertEquals(1, libCache.getNumberOfManagedJobs());
            assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
            assertEquals(1, checkFilesExist(jobId, keys, cache, true));
            checkFileCountForJob(2, jobId, server);
            checkFileCountForJob(1, jobId, cache);
            // unregister the task registration
            classLoaderLease2.release();
            assertEquals(0, libCache.getNumberOfManagedJobs());
            assertEquals(0, libCache.getNumberOfReferenceHolders(jobId));
            // changing the libCache registration does not influence the BLOB stores...
            checkFileCountForJob(2, jobId, server);
            checkFileCountForJob(1, jobId, cache);
            cache.releaseJob(jobId);
            // library is still cached (but not associated with job any more)
            checkFileCountForJob(2, jobId, server);
            checkFileCountForJob(1, jobId, cache);
        }
        // see BlobUtils for the directory layout
        cacheDir = cache.getStorageLocation(jobId, new PermanentBlobKey()).getParentFile();
        assertTrue(cacheDir.exists());
        // make sure no further blobs can be downloaded by removing the write
        // permissions from the directory
        assertTrue("Could not remove write permissions from cache directory", cacheDir.setWritable(false, false));
        // since we cannot download this library any more, this call should fail
        try {
            cache.registerJob(jobId);
            final LibraryCacheManager.ClassLoaderLease classLoaderLease = libCache.registerClassLoaderLease(jobId);
            classLoaderLease.getOrResolveClassLoader(Collections.singleton(dataKey2), Collections.emptyList());
            fail("This should fail with an IOException");
        } catch (IOException e) {
            // splendid!
            cache.releaseJob(jobId);
        }
    } finally {
        if (cacheDir != null) {
            if (!cacheDir.setWritable(true, false)) {
                System.err.println("Could not re-add write permissions to cache directory.");
            }
        }
        if (cache != null) {
            cache.close();
        }
        if (libCache != null) {
            libCache.shutdown();
        }
        if (server != null) {
            server.close();
        }
    }
}
Also used : Configuration(org.apache.flink.configuration.Configuration) InetSocketAddress(java.net.InetSocketAddress) IOException(java.io.IOException) UserCodeClassLoader(org.apache.flink.util.UserCodeClassLoader) VoidBlobStore(org.apache.flink.runtime.blob.VoidBlobStore) PermanentBlobCache(org.apache.flink.runtime.blob.PermanentBlobCache) PermanentBlobKey(org.apache.flink.runtime.blob.PermanentBlobKey) Collection(java.util.Collection) BlobServer(org.apache.flink.runtime.blob.BlobServer) File(java.io.File) JobID(org.apache.flink.api.common.JobID) Test(org.junit.Test)

Example 3 with PermanentBlobCache

use of org.apache.flink.runtime.blob.PermanentBlobCache in project flink by apache.

the class DefaultExecutionGraphDeploymentWithBlobCacheTest method setupBlobServer.

@Before
@Override
public void setupBlobServer() throws IOException {
    Configuration config = new Configuration();
    // always offload the serialized job and task information
    config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
    blobServer = new BlobServer(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore());
    blobServer.start();
    blobWriter = blobServer;
    InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort());
    blobCache = new PermanentBlobCache(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore(), serverAddress);
}
Also used : VoidBlobStore(org.apache.flink.runtime.blob.VoidBlobStore) PermanentBlobCache(org.apache.flink.runtime.blob.PermanentBlobCache) Configuration(org.apache.flink.configuration.Configuration) InetSocketAddress(java.net.InetSocketAddress) BlobServer(org.apache.flink.runtime.blob.BlobServer) Before(org.junit.Before)

Example 4 with PermanentBlobCache

use of org.apache.flink.runtime.blob.PermanentBlobCache in project flink by apache.

the class DefaultExecutionGraphDeploymentWithSmallBlobCacheSizeLimitTest method testDeployMultipleTasksWithSmallBlobCacheSizeLimit.

/**
 * Test the deployment works well even the size limit of {@link BlobCacheSizeTracker} in {@link
 * PermanentBlobCache} is set to the minimum value.
 *
 * <p>In this extreme case, since the size limit is 1, every time a task is deployed, all the
 * existing **tracked** BLOBs on the cache must be untracked and deleted before the new BLOB is
 * stored onto the cache.
 *
 * <p>This extreme case covers the situation of the normal case, where the size limit is much
 * larger than 1 and the deletion won't happen so frequently.
 */
@Test
public void testDeployMultipleTasksWithSmallBlobCacheSizeLimit() throws Exception {
    final int numberOfVertices = 4;
    final int parallelism = 10;
    final ExecutionGraph eg = createAndSetupExecutionGraph(numberOfVertices, parallelism);
    final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
    final BlockingQueue<TaskDeploymentDescriptor> tdds = new ArrayBlockingQueue<>(numberOfVertices * parallelism);
    taskManagerGateway.setSubmitConsumer(FunctionUtils.uncheckedConsumer(taskDeploymentDescriptor -> {
        taskDeploymentDescriptor.loadBigData(blobCache);
        tdds.offer(taskDeploymentDescriptor);
    }));
    for (ExecutionJobVertex ejv : eg.getVerticesTopologically()) {
        for (ExecutionVertex ev : ejv.getTaskVertices()) {
            assertEquals(ExecutionState.CREATED, ev.getExecutionState());
            LogicalSlot slot = new TestingLogicalSlotBuilder().setTaskManagerGateway(taskManagerGateway).createTestingLogicalSlot();
            final Execution execution = ev.getCurrentExecutionAttempt();
            execution.transitionState(ExecutionState.SCHEDULED);
            execution.registerProducedPartitions(slot.getTaskManagerLocation(), true).get();
            ev.deployToSlot(slot);
            assertEquals(ExecutionState.DEPLOYING, ev.getExecutionState());
            TaskDeploymentDescriptor tdd = tdds.take();
            assertNotNull(tdd);
            List<InputGateDeploymentDescriptor> igdds = tdd.getInputGates();
            assertEquals(ev.getAllConsumedPartitionGroups().size(), igdds.size());
            if (igdds.size() > 0) {
                checkShuffleDescriptors(igdds.get(0), ev.getConsumedPartitionGroup(0));
            }
        }
    }
}
Also used : ComponentMainThreadExecutorServiceAdapter(org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) BlobServer(org.apache.flink.runtime.blob.BlobServer) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) BlobCacheSizeTracker(org.apache.flink.runtime.blob.BlobCacheSizeTracker) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) ResultPartitionType(org.apache.flink.runtime.io.network.partition.ResultPartitionType) TaskDeploymentDescriptor(org.apache.flink.runtime.deployment.TaskDeploymentDescriptor) ArrayList(java.util.ArrayList) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) TestingLogicalSlotBuilder(org.apache.flink.runtime.jobmaster.TestingLogicalSlotBuilder) DirectScheduledExecutorService(org.apache.flink.runtime.testutils.DirectScheduledExecutorService) JobException(org.apache.flink.runtime.JobException) FunctionUtils(org.apache.flink.util.function.FunctionUtils) JobGraphTestUtils(org.apache.flink.runtime.jobgraph.JobGraphTestUtils) JobExecutionException(org.apache.flink.runtime.client.JobExecutionException) DistributionPattern(org.apache.flink.runtime.jobgraph.DistributionPattern) Before(org.junit.Before) BlobServerOptions(org.apache.flink.configuration.BlobServerOptions) VoidBlobStore(org.apache.flink.runtime.blob.VoidBlobStore) Assert.assertNotNull(org.junit.Assert.assertNotNull) Configuration(org.apache.flink.configuration.Configuration) ExecutionState(org.apache.flink.runtime.execution.ExecutionState) LogicalSlot(org.apache.flink.runtime.jobmaster.LogicalSlot) Test(org.junit.Test) IOException(java.io.IOException) BlockingQueue(java.util.concurrent.BlockingQueue) InetSocketAddress(java.net.InetSocketAddress) SimpleAckingTaskManagerGateway(org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) List(java.util.List) ConsumedPartitionGroup(org.apache.flink.runtime.scheduler.strategy.ConsumedPartitionGroup) PermanentBlobCache(org.apache.flink.runtime.blob.PermanentBlobCache) BatchTask(org.apache.flink.runtime.operators.BatchTask) InputGateDeploymentDescriptor(org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor) Assert.assertEquals(org.junit.Assert.assertEquals) InputGateDeploymentDescriptor(org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor) LogicalSlot(org.apache.flink.runtime.jobmaster.LogicalSlot) SimpleAckingTaskManagerGateway(org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) TaskDeploymentDescriptor(org.apache.flink.runtime.deployment.TaskDeploymentDescriptor) TestingLogicalSlotBuilder(org.apache.flink.runtime.jobmaster.TestingLogicalSlotBuilder) Test(org.junit.Test)

Example 5 with PermanentBlobCache

use of org.apache.flink.runtime.blob.PermanentBlobCache in project flink by apache.

the class DefaultExecutionGraphDeploymentWithSmallBlobCacheSizeLimitTest method setupBlobServer.

@Before
@Override
public void setupBlobServer() throws IOException {
    Configuration config = new Configuration();
    // Always offload the serialized JobInformation, TaskInformation and cached
    // ShuffleDescriptors
    config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
    blobServer = new BlobServer(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore());
    blobServer.start();
    blobWriter = blobServer;
    InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort());
    // Set the size limit of the blob cache to 1
    BlobCacheSizeTracker blobCacheSizeTracker = new BlobCacheSizeTracker(1L);
    blobCache = new PermanentBlobCache(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore(), serverAddress, blobCacheSizeTracker);
}
Also used : VoidBlobStore(org.apache.flink.runtime.blob.VoidBlobStore) PermanentBlobCache(org.apache.flink.runtime.blob.PermanentBlobCache) Configuration(org.apache.flink.configuration.Configuration) InetSocketAddress(java.net.InetSocketAddress) BlobCacheSizeTracker(org.apache.flink.runtime.blob.BlobCacheSizeTracker) BlobServer(org.apache.flink.runtime.blob.BlobServer) Before(org.junit.Before)

Aggregations

InetSocketAddress (java.net.InetSocketAddress)7 Configuration (org.apache.flink.configuration.Configuration)7 BlobServer (org.apache.flink.runtime.blob.BlobServer)7 PermanentBlobCache (org.apache.flink.runtime.blob.PermanentBlobCache)7 VoidBlobStore (org.apache.flink.runtime.blob.VoidBlobStore)6 Test (org.junit.Test)5 ArrayList (java.util.ArrayList)4 JobID (org.apache.flink.api.common.JobID)4 PermanentBlobKey (org.apache.flink.runtime.blob.PermanentBlobKey)4 Before (org.junit.Before)3 File (java.io.File)2 IOException (java.io.IOException)2 URL (java.net.URL)2 BlobCacheSizeTracker (org.apache.flink.runtime.blob.BlobCacheSizeTracker)2 UserCodeClassLoader (org.apache.flink.util.UserCodeClassLoader)2 FileInputStream (java.io.FileInputStream)1 Collection (java.util.Collection)1 List (java.util.List)1 Random (java.util.Random)1 ArrayBlockingQueue (java.util.concurrent.ArrayBlockingQueue)1