Examples with UserCodeClassLoader org.apache.flink.util.UserCodeClassLoader used on opensource projects

Example 1 with UserCodeClassLoader

use of org.apache.flink.util.UserCodeClassLoader in project flink by apache.

the class BlobLibraryCacheManagerTest method testLibraryCacheManagerCleanup.

/**
 * Tests that the {@link BlobLibraryCacheManager} cleans up after all class loader leases for a
 * single job a closed.
 */
@Test
public void testLibraryCacheManagerCleanup() throws Exception {
    JobID jobId = new JobID();
    List<PermanentBlobKey> keys = new ArrayList<>();
    BlobServer server = null;
    PermanentBlobCache cache = null;
    BlobLibraryCacheManager libCache = null;
    final byte[] buf = new byte[128];
    try {
        Configuration config = new Configuration();
        config.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);
        server = new BlobServer(config, temporaryFolder.newFolder(), new VoidBlobStore());
        server.start();
        InetSocketAddress serverAddress = new InetSocketAddress("localhost", server.getPort());
        cache = new PermanentBlobCache(config, temporaryFolder.newFolder(), new VoidBlobStore(), serverAddress);
        keys.add(server.putPermanent(jobId, buf));
        buf[0] += 1;
        keys.add(server.putPermanent(jobId, buf));
        libCache = createBlobLibraryCacheManager(cache);
        cache.registerJob(jobId);
        assertEquals(0, libCache.getNumberOfManagedJobs());
        assertEquals(0, libCache.getNumberOfReferenceHolders(jobId));
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(0, jobId, cache);
        final LibraryCacheManager.ClassLoaderLease classLoaderLease1 = libCache.registerClassLoaderLease(jobId);
        UserCodeClassLoader classLoader1 = classLoaderLease1.getOrResolveClassLoader(keys, Collections.emptyList());
        assertEquals(1, libCache.getNumberOfManagedJobs());
        assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
        assertEquals(2, checkFilesExist(jobId, keys, cache, true));
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(2, jobId, cache);
        final LibraryCacheManager.ClassLoaderLease classLoaderLease2 = libCache.registerClassLoaderLease(jobId);
        final UserCodeClassLoader classLoader2 = classLoaderLease2.getOrResolveClassLoader(keys, Collections.emptyList());
        assertThat(classLoader1, sameInstance(classLoader2));
        try {
            classLoaderLease1.getOrResolveClassLoader(Collections.emptyList(), Collections.emptyList());
            fail("Should fail with an IllegalStateException");
        } catch (IllegalStateException e) {
        // that's what we want
        }
        try {
            classLoaderLease1.getOrResolveClassLoader(keys, Collections.singletonList(new URL("file:///tmp/does-not-exist")));
            fail("Should fail with an IllegalStateException");
        } catch (IllegalStateException e) {
        // that's what we want
        }
        assertEquals(1, libCache.getNumberOfManagedJobs());
        assertEquals(2, libCache.getNumberOfReferenceHolders(jobId));
        assertEquals(2, checkFilesExist(jobId, keys, cache, true));
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(2, jobId, cache);
        classLoaderLease1.release();
        assertEquals(1, libCache.getNumberOfManagedJobs());
        assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
        assertEquals(2, checkFilesExist(jobId, keys, cache, true));
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(2, jobId, cache);
        classLoaderLease2.release();
        assertEquals(0, libCache.getNumberOfManagedJobs());
        assertEquals(0, libCache.getNumberOfReferenceHolders(jobId));
        assertEquals(2, checkFilesExist(jobId, keys, cache, true));
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(2, jobId, cache);
    // only PermanentBlobCache#releaseJob() calls clean up files (tested in
    // BlobCacheCleanupTest etc.
    } finally {
        if (libCache != null) {
            libCache.shutdown();
        }
        // should have been closed by the libraryCacheManager, but just in case
        if (cache != null) {
            cache.close();
        }
        if (server != null) {
            server.close();
        }
    }
}

Example 2 with UserCodeClassLoader

use of org.apache.flink.util.UserCodeClassLoader in project flink by apache.

the class BlobLibraryCacheManagerTest method releaseUserCodeClassLoader_willRegisterOnce.

@Test
public void releaseUserCodeClassLoader_willRegisterOnce() throws IOException, InterruptedException {
    final BlobLibraryCacheManager libraryCacheManager = new TestingBlobLibraryCacheManagerBuilder().build();
    final LibraryCacheManager.ClassLoaderLease classLoaderLease = libraryCacheManager.registerClassLoaderLease(new JobID());
    final UserCodeClassLoader userCodeClassLoader = classLoaderLease.getOrResolveClassLoader(Collections.emptyList(), Collections.emptyList());
    final OneShotLatch releaseHookLatch = new OneShotLatch();
    userCodeClassLoader.registerReleaseHookIfAbsent("test", releaseHookLatch::trigger);
    userCodeClassLoader.registerReleaseHookIfAbsent("test", () -> {
        throw new RuntimeException("This hook is not expected to be executed");
    });
    classLoaderLease.release();
    // this will wait forever if the second hook gets registered
    releaseHookLatch.await();
}

Example 3 with UserCodeClassLoader

use of org.apache.flink.util.UserCodeClassLoader in project flink by apache.

the class BlobLibraryCacheManagerTest method testRegisterAndDownload.

@Test
public void testRegisterAndDownload() throws IOException {
    // setWritable doesn't work on Windows.
    assumeTrue(!OperatingSystem.isWindows());
    JobID jobId = new JobID();
    BlobServer server = null;
    PermanentBlobCache cache = null;
    BlobLibraryCacheManager libCache = null;
    File cacheDir = null;
    try {
        // create the blob transfer services
        Configuration config = new Configuration();
        config.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1_000_000L);
        server = new BlobServer(config, temporaryFolder.newFolder(), new VoidBlobStore());
        server.start();
        InetSocketAddress serverAddress = new InetSocketAddress("localhost", server.getPort());
        cache = new PermanentBlobCache(config, temporaryFolder.newFolder(), new VoidBlobStore(), serverAddress);
        // upload some meaningless data to the server
        PermanentBlobKey dataKey1 = server.putPermanent(jobId, new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 });
        PermanentBlobKey dataKey2 = server.putPermanent(jobId, new byte[] { 11, 12, 13, 14, 15, 16, 17, 18 });
        libCache = createBlobLibraryCacheManager(cache);
        assertEquals(0, libCache.getNumberOfManagedJobs());
        checkFileCountForJob(2, jobId, server);
        checkFileCountForJob(0, jobId, cache);
        // first try to access a non-existing entry
        assertEquals(0, libCache.getNumberOfReferenceHolders(new JobID()));
        // register some BLOBs as libraries
        {
            Collection<PermanentBlobKey> keys = Collections.singleton(dataKey1);
            cache.registerJob(jobId);
            final LibraryCacheManager.ClassLoaderLease classLoaderLease1 = libCache.registerClassLoaderLease(jobId);
            final UserCodeClassLoader classLoader1 = classLoaderLease1.getOrResolveClassLoader(keys, Collections.emptyList());
            assertEquals(1, libCache.getNumberOfManagedJobs());
            assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
            assertEquals(1, checkFilesExist(jobId, keys, cache, true));
            checkFileCountForJob(2, jobId, server);
            checkFileCountForJob(1, jobId, cache);
            final LibraryCacheManager.ClassLoaderLease classLoaderLease2 = libCache.registerClassLoaderLease(jobId);
            final UserCodeClassLoader classLoader2 = classLoaderLease2.getOrResolveClassLoader(keys, Collections.emptyList());
            assertThat(classLoader1, sameInstance(classLoader2));
            assertEquals(1, libCache.getNumberOfManagedJobs());
            assertEquals(2, libCache.getNumberOfReferenceHolders(jobId));
            assertEquals(1, checkFilesExist(jobId, keys, cache, true));
            checkFileCountForJob(2, jobId, server);
            checkFileCountForJob(1, jobId, cache);
            // un-register the job
            classLoaderLease1.release();
            // still one task
            assertEquals(1, libCache.getNumberOfManagedJobs());
            assertEquals(1, libCache.getNumberOfReferenceHolders(jobId));
            assertEquals(1, checkFilesExist(jobId, keys, cache, true));
            checkFileCountForJob(2, jobId, server);
            checkFileCountForJob(1, jobId, cache);
            // unregister the task registration
            classLoaderLease2.release();
            assertEquals(0, libCache.getNumberOfManagedJobs());
            assertEquals(0, libCache.getNumberOfReferenceHolders(jobId));
            // changing the libCache registration does not influence the BLOB stores...
            checkFileCountForJob(2, jobId, server);
            checkFileCountForJob(1, jobId, cache);
            cache.releaseJob(jobId);
            // library is still cached (but not associated with job any more)
            checkFileCountForJob(2, jobId, server);
            checkFileCountForJob(1, jobId, cache);
        }
        // see BlobUtils for the directory layout
        cacheDir = cache.getStorageLocation(jobId, new PermanentBlobKey()).getParentFile();
        assertTrue(cacheDir.exists());
        // make sure no further blobs can be downloaded by removing the write
        // permissions from the directory
        assertTrue("Could not remove write permissions from cache directory", cacheDir.setWritable(false, false));
        // since we cannot download this library any more, this call should fail
        try {
            cache.registerJob(jobId);
            final LibraryCacheManager.ClassLoaderLease classLoaderLease = libCache.registerClassLoaderLease(jobId);
            classLoaderLease.getOrResolveClassLoader(Collections.singleton(dataKey2), Collections.emptyList());
            fail("This should fail with an IOException");
        } catch (IOException e) {
            // splendid!
            cache.releaseJob(jobId);
        }
    } finally {
        if (cacheDir != null) {
            if (!cacheDir.setWritable(true, false)) {
                System.err.println("Could not re-add write permissions to cache directory.");
            }
        }
        if (cache != null) {
            cache.close();
        }
        if (libCache != null) {
            libCache.shutdown();
        }
        if (server != null) {
            server.close();
        }
    }
}

Example 4 with UserCodeClassLoader

use of org.apache.flink.util.UserCodeClassLoader in project flink by apache.

the class DeserializationSchemaAdapter method createDeserialization.

private DeserializationSchema<RowData> createDeserialization() throws IOException {
    try {
        DeserializationSchema<RowData> deserialization = InstantiationUtil.clone(deserializationSchema);
        deserialization.open(new DeserializationSchema.InitializationContext() {

            @Override
            public MetricGroup getMetricGroup() {
                throw new UnsupportedOperationException("MetricGroup is unsupported in BulkFormat.");
            }

            @Override
            public UserCodeClassLoader getUserCodeClassLoader() {
                return (UserCodeClassLoader) Thread.currentThread().getContextClassLoader();
            }
        });
        return deserialization;
    } catch (Exception e) {
        throw new IOException(e);
    }
}

Example 5 with UserCodeClassLoader

use of org.apache.flink.util.UserCodeClassLoader in project flink by apache.

the class BatchTask method initInputsSerializersAndComparators.

/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) {
    this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
    this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
    this.inputIterators = new MutableObjectIterator<?>[numInputs];
    ClassLoader userCodeClassLoader = getUserCodeClassLoader();
    for (int i = 0; i < numInputs; i++) {
        final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
        this.inputSerializers[i] = serializerFactory;
        this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
    }
    // ---------------- create the driver's comparators ---------------------
    for (int i = 0; i < numComparators; i++) {
        if (this.inputComparators != null) {
            final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
            this.inputComparators[i] = comparatorFactory.createComparator();
        }
    }
}