Examples with Collection java.util.Collection used on opensource projects

Example 46 with Collection

use of java.util.Collection in project flink by apache.

the class ReusingHashJoinIteratorITCase method testBuildFirstWithMixedDataTypes.

@Test
public void testBuildFirstWithMixedDataTypes() {
    try {
        MutableObjectIterator<IntPair> input1 = new UniformIntPairGenerator(500, 40, false);
        final TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
        // collect expected data
        final Map<Integer, Collection<TupleIntPairMatch>> expectedMatchesMap = joinIntPairs(collectIntPairData(input1), collectTupleData(input2));
        final FlatJoinFunction<IntPair, Tuple2<Integer, String>, Tuple2<Integer, String>> matcher = new TupleIntPairMatchRemovingMatcher(expectedMatchesMap);
        final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
        // reset the generators
        input1 = new UniformIntPairGenerator(500, 40, false);
        generator2.reset();
        input2.reset();
        // compare with iterator values
        ReusingBuildSecondHashJoinIterator<IntPair, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new ReusingBuildSecondHashJoinIterator<>(input1, input2, this.pairSerializer, this.pairComparator, this.recordSerializer, this.record2Comparator, this.pairRecordPairComparator, this.memoryManager, this.ioManager, this.parentTask, 1.0, false, false, true);
        iterator.open();
        while (iterator.callWithNextKey(matcher, collector)) ;
        iterator.close();
        // assert that each expected match was seen
        for (Entry<Integer, Collection<TupleIntPairMatch>> entry : expectedMatchesMap.entrySet()) {
            if (!entry.getValue().isEmpty()) {
                Assert.fail("Collection for key " + entry.getKey() + " is not empty");
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
}

Example 47 with Collection

use of java.util.Collection in project groovy by apache.

the class DefaultTypeTransformation method continueCastOnSAM.

private static Object continueCastOnSAM(Object object, Class type) {
    if (object instanceof Closure) {
        Method m = CachedSAMClass.getSAMMethod(type);
        if (m != null) {
            return CachedSAMClass.coerceToSAM((Closure) object, m, type, type.isInterface());
        }
    }
    Object[] args = null;
    if (object instanceof Collection) {
        // let's try invoke the constructor with the list as arguments
        // such as for creating a Dimension, Point, Color etc.
        Collection collection = (Collection) object;
        args = collection.toArray();
    } else if (object instanceof Object[]) {
        args = (Object[]) object;
    } else if (object instanceof Map) {
        // emulate named params constructor
        args = new Object[1];
        args[0] = object;
    }
    Exception nested = null;
    if (args != null) {
        try {
            return InvokerHelper.invokeConstructorOf(type, args);
        } catch (InvokerInvocationException iie) {
            throw iie;
        } catch (GroovyRuntimeException e) {
            if (e.getMessage().contains("Could not find matching constructor for")) {
                try {
                    return InvokerHelper.invokeConstructorOf(type, object);
                } catch (InvokerInvocationException iie) {
                    throw iie;
                } catch (Exception ex) {
                    // let's ignore exception and return the original object
                    // as the caller has more context to be able to throw a more
                    // meaningful exception (but stash to get message later)
                    nested = e;
                }
            } else {
                nested = e;
            }
        } catch (Exception e) {
            // let's ignore exception and return the original object
            // as the caller has more context to be able to throw a more
            // meaningful exception (but stash to get message later)
            nested = e;
        }
    }
    GroovyCastException gce;
    if (nested != null) {
        gce = new GroovyCastException(object, type, nested);
    } else {
        gce = new GroovyCastException(object, type);
    }
    throw gce;
}

Example 48 with Collection

use of java.util.Collection in project groovy by apache.

the class DefaultTypeTransformation method asArray.

public static Object asArray(Object object, Class type) {
    if (type.isAssignableFrom(object.getClass())) {
        return object;
    }
    Collection list = asCollection(object);
    int size = list.size();
    Class elementType = type.getComponentType();
    Object array = Array.newInstance(elementType, size);
    int idx = 0;
    for (Iterator iter = list.iterator(); iter.hasNext(); idx++) {
        Object element = iter.next();
        Array.set(array, idx, castToType(element, elementType));
    }
    return array;
}

Example 49 with Collection

use of java.util.Collection in project flink by apache.

the class BlobLibraryCacheManagerTest method testRegisterAndDownload.

@Test
public void testRegisterAndDownload() {
    //setWritable doesn't work on Windows.
    assumeTrue(!OperatingSystem.isWindows());
    BlobServer server = null;
    BlobCache cache = null;
    File cacheDir = null;
    try {
        // create the blob transfer services
        Configuration config = new Configuration();
        server = new BlobServer(config);
        InetSocketAddress serverAddress = new InetSocketAddress("localhost", server.getPort());
        cache = new BlobCache(serverAddress, config);
        // upload some meaningless data to the server
        BlobClient uploader = new BlobClient(serverAddress, config);
        BlobKey dataKey1 = uploader.put(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 });
        BlobKey dataKey2 = uploader.put(new byte[] { 11, 12, 13, 14, 15, 16, 17, 18 });
        uploader.close();
        BlobLibraryCacheManager libCache = new BlobLibraryCacheManager(cache, 1000000000L);
        assertEquals(0, libCache.getNumberOfCachedLibraries());
        // first try to access a non-existing entry
        try {
            libCache.getClassLoader(new JobID());
            fail("Should fail with an IllegalStateException");
        } catch (IllegalStateException e) {
        // that#s what we want
        }
        // now register some BLOBs as libraries
        {
            JobID jid = new JobID();
            ExecutionAttemptID executionId = new ExecutionAttemptID();
            Collection<BlobKey> keys = Collections.singleton(dataKey1);
            libCache.registerTask(jid, executionId, keys, Collections.<URL>emptyList());
            assertEquals(1, libCache.getNumberOfReferenceHolders(jid));
            assertEquals(1, libCache.getNumberOfCachedLibraries());
            assertNotNull(libCache.getClassLoader(jid));
            // un-register them again
            libCache.unregisterTask(jid, executionId);
            // Don't fail if called again
            libCache.unregisterTask(jid, executionId);
            assertEquals(0, libCache.getNumberOfReferenceHolders(jid));
            // library is still cached (but not associated with job any more)
            assertEquals(1, libCache.getNumberOfCachedLibraries());
            // should not be able to access the classloader any more
            try {
                libCache.getClassLoader(jid);
                fail("Should fail with an IllegalStateException");
            } catch (IllegalStateException e) {
            // that#s what we want
            }
        }
        cacheDir = new File(cache.getStorageDir(), "cache");
        assertTrue(cacheDir.exists());
        // make sure no further blobs can be downloaded by removing the write
        // permissions from the directory
        assertTrue("Could not remove write permissions from cache directory", cacheDir.setWritable(false, false));
        // since we cannot download this library any more, this call should fail
        try {
            libCache.registerTask(new JobID(), new ExecutionAttemptID(), Collections.singleton(dataKey2), Collections.<URL>emptyList());
            fail("This should fail with an IOException");
        } catch (IOException e) {
        // splendid!
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (cacheDir != null) {
            if (!cacheDir.setWritable(true, false)) {
                System.err.println("Could not re-add write permissions to cache directory.");
            }
        }
        if (cache != null) {
            cache.shutdown();
        }
        if (server != null) {
            server.shutdown();
        }
    }
}

Example 50 with Collection

use of java.util.Collection in project flink by apache.

the class ReusingSortMergeCoGroupIteratorITCase method testMerge.

@Test
public void testMerge() {
    try {
        generator1 = new TupleGenerator(SEED1, 500, 4096, KeyMode.SORTED, ValueMode.RANDOM_LENGTH);
        generator2 = new TupleGenerator(SEED2, 500, 2048, KeyMode.SORTED, ValueMode.RANDOM_LENGTH);
        reader1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
        reader2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
        // collect expected data
        Map<Integer, Collection<String>> expectedStringsMap1 = collectData(generator1, INPUT_1_SIZE);
        Map<Integer, Collection<String>> expectedStringsMap2 = collectData(generator2, INPUT_2_SIZE);
        Map<Integer, List<Collection<String>>> expectedCoGroupsMap = coGroupValues(expectedStringsMap1, expectedStringsMap2);
        // reset the generators
        generator1.reset();
        generator2.reset();
        // compare with iterator values
        ReusingSortMergeCoGroupIterator<Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new ReusingSortMergeCoGroupIterator<>(this.reader1, this.reader2, this.serializer1, this.comparator1, this.serializer2, this.comparator2, this.pairComparator);
        iterator.open();
        int key = 0;
        while (iterator.next()) {
            Iterator<Tuple2<Integer, String>> iter1 = iterator.getValues1().iterator();
            Iterator<Tuple2<Integer, String>> iter2 = iterator.getValues2().iterator();
            String v1 = null;
            String v2 = null;
            if (iter1.hasNext()) {
                Tuple2<Integer, String> rec = iter1.next();
                key = rec.f0;
                v1 = rec.f1;
            } else if (iter2.hasNext()) {
                Tuple2<Integer, String> rec = iter2.next();
                key = rec.f0;
                v2 = rec.f1;
            } else {
                Assert.fail("No input on both sides.");
            }
            // assert that matches for this key exist
            Assert.assertTrue("No matches for key " + key, expectedCoGroupsMap.containsKey(key));
            Collection<String> expValues1 = expectedCoGroupsMap.get(key).get(0);
            Collection<String> expValues2 = expectedCoGroupsMap.get(key).get(1);
            if (v1 != null) {
                expValues1.remove(v1);
            } else {
                expValues2.remove(v2);
            }
            while (iter1.hasNext()) {
                Tuple2<Integer, String> rec = iter1.next();
                Assert.assertTrue("String not in expected set of first input", expValues1.remove(rec.f1));
            }
            Assert.assertTrue("Expected set of first input not empty", expValues1.isEmpty());
            while (iter2.hasNext()) {
                Tuple2<Integer, String> rec = iter2.next();
                Assert.assertTrue("String not in expected set of second input", expValues2.remove(rec.f1));
            }
            Assert.assertTrue("Expected set of second input not empty", expValues2.isEmpty());
            expectedCoGroupsMap.remove(key);
        }
        iterator.close();
        Assert.assertTrue("Expected key set not empty", expectedCoGroupsMap.isEmpty());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
}