Examples with URL java.net.URL used on opensource projects

Example 91 with URL

use of java.net.URL in project flink by apache.

the class BlobCacheRetriesTest method testBlobFetchRetries.

/**
	 * A test where the BlobCache must use the BlobServer and the connection
	 * fails twice and then the get operation succeeds.
	 *
	 * @param config
	 * 		configuration to use (the BlobCache will get some additional settings
	 * 		set compared to this one)
	 */
private void testBlobFetchRetries(final Configuration config) {
    final byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 };
    BlobServer server = null;
    BlobCache cache = null;
    try {
        server = new TestingFailingBlobServer(config, 2);
        final InetSocketAddress serverAddress = new InetSocketAddress("localhost", server.getPort());
        // upload some blob
        BlobClient blobClient = null;
        BlobKey key;
        try {
            blobClient = new BlobClient(serverAddress, config);
            key = blobClient.put(data);
        } finally {
            if (blobClient != null) {
                blobClient.close();
            }
        }
        // create a separate config for the cache with no access to
        // the (shared) storage path if available so that the cache
        // will always bother the BlobServer!
        final Configuration cacheConfig = new Configuration(config);
        cacheConfig.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.getRoot().getPath() + "/does-not-exist");
        cache = new BlobCache(serverAddress, cacheConfig);
        // trigger a download - it should fail the first two times, but retry, and succeed eventually
        URL url = cache.getURL(key);
        InputStream is = url.openStream();
        try {
            byte[] received = new byte[data.length];
            assertEquals(data.length, is.read(received));
            assertArrayEquals(data, received);
        } finally {
            is.close();
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (cache != null) {
            cache.shutdown();
        }
        if (server != null) {
            server.shutdown();
        }
    }
}

Example 92 with URL

use of java.net.URL in project flink by apache.

the class BlobCacheSuccessTest method uploadFileGetTest.

private void uploadFileGetTest(final Configuration config, boolean cacheWorksWithoutServer, boolean cacheHasAccessToFs) {
    // First create two BLOBs and upload them to BLOB server
    final byte[] buf = new byte[128];
    final List<BlobKey> blobKeys = new ArrayList<BlobKey>(2);
    BlobServer blobServer = null;
    BlobCache blobCache = null;
    try {
        // Start the BLOB server
        blobServer = new BlobServer(config);
        final InetSocketAddress serverAddress = new InetSocketAddress(blobServer.getPort());
        // Upload BLOBs
        BlobClient blobClient = null;
        try {
            blobClient = new BlobClient(serverAddress, config);
            blobKeys.add(blobClient.put(buf));
            // Make sure the BLOB key changes
            buf[0] = 1;
            blobKeys.add(blobClient.put(buf));
        } finally {
            if (blobClient != null) {
                blobClient.close();
            }
        }
        if (cacheWorksWithoutServer) {
            // Now, shut down the BLOB server, the BLOBs must still be accessible through the cache.
            blobServer.shutdown();
            blobServer = null;
        }
        final Configuration cacheConfig;
        if (cacheHasAccessToFs) {
            cacheConfig = config;
        } else {
            // just in case parameters are still read from the server,
            // create a separate configuration object for the cache
            cacheConfig = new Configuration(config);
            cacheConfig.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.getRoot().getPath() + "/does-not-exist");
        }
        blobCache = new BlobCache(serverAddress, cacheConfig);
        for (BlobKey blobKey : blobKeys) {
            blobCache.getURL(blobKey);
        }
        if (blobServer != null) {
            // Now, shut down the BLOB server, the BLOBs must still be accessible through the cache.
            blobServer.shutdown();
            blobServer = null;
        }
        final URL[] urls = new URL[blobKeys.size()];
        for (int i = 0; i < blobKeys.size(); i++) {
            urls[i] = blobCache.getURL(blobKeys.get(i));
        }
        // Verify the result
        assertEquals(blobKeys.size(), urls.length);
        for (final URL url : urls) {
            assertNotNull(url);
            try {
                final File cachedFile = new File(url.toURI());
                assertTrue(cachedFile.exists());
                assertEquals(buf.length, cachedFile.length());
            } catch (URISyntaxException e) {
                fail(e.getMessage());
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (blobServer != null) {
            blobServer.shutdown();
        }
        if (blobCache != null) {
            blobCache.shutdown();
        }
    }
}

Example 93 with URL

use of java.net.URL in project flink by apache.

the class TestBaseUtils method getFromHTTP.

//---------------------------------------------------------------------------------------------
// Web utils
//---------------------------------------------------------------------------------------------
public static String getFromHTTP(String url) throws Exception {
    URL u = new URL(url);
    LOG.info("Accessing URL " + url + " as URL: " + u);
    HttpURLConnection connection = (HttpURLConnection) u.openConnection();
    connection.setConnectTimeout(100000);
    connection.connect();
    InputStream is;
    if (connection.getResponseCode() >= 400) {
        // error!
        LOG.warn("HTTP Response code when connecting to {} was {}", url, connection.getResponseCode());
        is = connection.getErrorStream();
    } else {
        is = connection.getInputStream();
    }
    return IOUtils.toString(is, connection.getContentEncoding() != null ? connection.getContentEncoding() : "UTF-8");
}

Example 94 with URL

use of java.net.URL in project flink by apache.

the class CustomInputSplitProgram method main.

public static void main(String[] args) throws Exception {
    final String[] jarFile = (args[0].equals("")) ? null : new String[] { args[0] };
    final URL[] classpath = (args[1].equals("")) ? null : new URL[] { new URL(args[1]) };
    final String host = args[2];
    final int port = Integer.parseInt(args[3]);
    final int parallelism = Integer.parseInt(args[4]);
    RemoteEnvironment env = new RemoteEnvironment(host, port, null, jarFile, classpath);
    env.setParallelism(parallelism);
    env.getConfig().disableSysoutLogging();
    DataSet<Integer> data = env.createInput(new CustomInputFormat());
    data.map(new MapFunction<Integer, Tuple2<Integer, Double>>() {

        @Override
        public Tuple2<Integer, Double> map(Integer value) {
            return new Tuple2<Integer, Double>(value, value * 0.5);
        }
    }).output(new DiscardingOutputFormat<Tuple2<Integer, Double>>());
    env.execute();
}

Example 95 with URL

use of java.net.URL in project groovy by apache.

the class GroovyShellTest method testWithGCSWithURL.

public void testWithGCSWithURL() throws Exception {
    String scriptFileName = "src/test/groovy/bugs/GROOVY3934Helper.groovy";
    File helperScript = new File(scriptFileName);
    if (!helperScript.exists()) {
        fail("File " + scriptFileName + " does not exist");
    } else {
        URL url = helperScript.toURI().toURL();
        GroovyCodeSource gcs = new GroovyCodeSource(url);
        GroovyShell shell = new GroovyShell();
        Object result = shell.evaluate(gcs);
        assertEquals("GROOVY3934Helper script called", result);
    }
}