Examples with Set java.util.Set used on opensource projects

Example 91 with Set

use of java.util.Set in project hive by apache.

the class TestVectorGroupByOperator method testAggregateLongKeyIterable.

public void testAggregateLongKeyIterable(String aggregateName, Iterable<VectorizedRowBatch> data, HashMap<Object, Object> expected) throws HiveException {
    List<String> mapColumnNames = new ArrayList<String>();
    mapColumnNames.add("Key");
    mapColumnNames.add("Value");
    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
    Set<Object> keys = new HashSet<Object>();
    GroupByDesc desc = buildKeyGroupByDesc(ctx, aggregateName, "Value", TypeInfoFactory.longTypeInfo, "Key", TypeInfoFactory.longTypeInfo);
    CompilationOpContext cCtx = new CompilationOpContext();
    Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc);
    VectorGroupByOperator vgo = (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx);
    FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo);
    vgo.initialize(hconf, null);
    out.setOutputInspector(new FakeCaptureOutputOperator.OutputInspector() {

        private String aggregateName;

        private HashMap<Object, Object> expected;

        private Set<Object> keys;

        @Override
        public void inspectRow(Object row, int tag) throws HiveException {
            assertTrue(row instanceof Object[]);
            Object[] fields = (Object[]) row;
            assertEquals(2, fields.length);
            Object key = fields[0];
            Long keyValue = null;
            if (null != key) {
                assertTrue(key instanceof LongWritable);
                LongWritable lwKey = (LongWritable) key;
                keyValue = lwKey.get();
            }
            assertTrue(expected.containsKey(keyValue));
            String keyAsString = String.format("%s", key);
            Object expectedValue = expected.get(keyValue);
            Object value = fields[1];
            Validator validator = getValidator(aggregateName);
            validator.validate(keyAsString, expectedValue, new Object[] { value });
            keys.add(keyValue);
        }

        private FakeCaptureOutputOperator.OutputInspector init(String aggregateName, HashMap<Object, Object> expected, Set<Object> keys) {
            this.aggregateName = aggregateName;
            this.expected = expected;
            this.keys = keys;
            return this;
        }
    }.init(aggregateName, expected, keys));
    for (VectorizedRowBatch unit : data) {
        vgo.process(unit, 0);
    }
    vgo.close(false);
    List<Object> outBatchList = out.getCapturedRows();
    assertNotNull(outBatchList);
    assertEquals(expected.size(), outBatchList.size());
    assertEquals(expected.size(), keys.size());
}

Example 92 with Set

use of java.util.Set in project hive by apache.

the class TestVectorGroupByOperator method testKeyTypeAggregate.

private void testKeyTypeAggregate(String aggregateName, FakeVectorRowBatchFromObjectIterables data, Map<Object, Object> expected) throws HiveException {
    List<String> mapColumnNames = new ArrayList<String>();
    mapColumnNames.add("Key");
    mapColumnNames.add("Value");
    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
    Set<Object> keys = new HashSet<Object>();
    AggregationDesc agg = buildAggregationDesc(ctx, aggregateName, GenericUDAFEvaluator.Mode.PARTIAL1, "Value", TypeInfoFactory.getPrimitiveTypeInfo(data.getTypes()[1]));
    ArrayList<AggregationDesc> aggs = new ArrayList<AggregationDesc>();
    aggs.add(agg);
    ArrayList<String> outputColumnNames = new ArrayList<String>();
    outputColumnNames.add("_col0");
    outputColumnNames.add("_col1");
    GroupByDesc desc = new GroupByDesc();
    desc.setVectorDesc(new VectorGroupByDesc());
    desc.setOutputColumnNames(outputColumnNames);
    desc.setAggregators(aggs);
    ((VectorGroupByDesc) desc.getVectorDesc()).setProcessingMode(ProcessingMode.HASH);
    ExprNodeDesc keyExp = buildColumnDesc(ctx, "Key", TypeInfoFactory.getPrimitiveTypeInfo(data.getTypes()[0]));
    ArrayList<ExprNodeDesc> keysDesc = new ArrayList<ExprNodeDesc>();
    keysDesc.add(keyExp);
    desc.setKeys(keysDesc);
    CompilationOpContext cCtx = new CompilationOpContext();
    Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc);
    VectorGroupByOperator vgo = (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx);
    FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo);
    vgo.initialize(hconf, null);
    out.setOutputInspector(new FakeCaptureOutputOperator.OutputInspector() {

        private int rowIndex;

        private String aggregateName;

        private Map<Object, Object> expected;

        private Set<Object> keys;

        @Override
        public void inspectRow(Object row, int tag) throws HiveException {
            assertTrue(row instanceof Object[]);
            Object[] fields = (Object[]) row;
            assertEquals(2, fields.length);
            Object key = fields[0];
            Object keyValue = null;
            if (null == key) {
                keyValue = null;
            } else if (key instanceof ByteWritable) {
                ByteWritable bwKey = (ByteWritable) key;
                keyValue = bwKey.get();
            } else if (key instanceof ShortWritable) {
                ShortWritable swKey = (ShortWritable) key;
                keyValue = swKey.get();
            } else if (key instanceof IntWritable) {
                IntWritable iwKey = (IntWritable) key;
                keyValue = iwKey.get();
            } else if (key instanceof LongWritable) {
                LongWritable lwKey = (LongWritable) key;
                keyValue = lwKey.get();
            } else if (key instanceof TimestampWritable) {
                TimestampWritable twKey = (TimestampWritable) key;
                keyValue = twKey.getTimestamp();
            } else if (key instanceof DoubleWritable) {
                DoubleWritable dwKey = (DoubleWritable) key;
                keyValue = dwKey.get();
            } else if (key instanceof FloatWritable) {
                FloatWritable fwKey = (FloatWritable) key;
                keyValue = fwKey.get();
            } else if (key instanceof BooleanWritable) {
                BooleanWritable bwKey = (BooleanWritable) key;
                keyValue = bwKey.get();
            } else if (key instanceof HiveDecimalWritable) {
                HiveDecimalWritable hdwKey = (HiveDecimalWritable) key;
                keyValue = hdwKey.getHiveDecimal();
            } else {
                Assert.fail(String.format("Not implemented key output type %s: %s", key.getClass().getName(), key));
            }
            String keyValueAsString = String.format("%s", keyValue);
            assertTrue(expected.containsKey(keyValue));
            Object expectedValue = expected.get(keyValue);
            Object value = fields[1];
            Validator validator = getValidator(aggregateName);
            validator.validate(keyValueAsString, expectedValue, new Object[] { value });
            keys.add(keyValue);
        }

        private FakeCaptureOutputOperator.OutputInspector init(String aggregateName, Map<Object, Object> expected, Set<Object> keys) {
            this.aggregateName = aggregateName;
            this.expected = expected;
            this.keys = keys;
            return this;
        }
    }.init(aggregateName, expected, keys));
    for (VectorizedRowBatch unit : data) {
        vgo.process(unit, 0);
    }
    vgo.close(false);
    List<Object> outBatchList = out.getCapturedRows();
    assertNotNull(outBatchList);
    assertEquals(expected.size(), outBatchList.size());
    assertEquals(expected.size(), keys.size());
}

Example 93 with Set

use of java.util.Set in project hive by apache.

the class TestCloudExecutionContextProvider method testRetry.

@org.junit.Test
public void testRetry() throws Exception {
    when(cloudComputeService.createNodes(anyInt())).then(new Answer<Set<NodeMetadata>>() {

        int count = 0;

        @Override
        public Set<NodeMetadata> answer(InvocationOnMock invocation) throws Throwable {
            if (count++ == 0) {
                throw runNodesException;
            }
            return Collections.singleton(node3);
        }
    });
    CloudExecutionContextProvider provider = new CloudExecutionContextProvider(dataDir, NUM_NODES, cloudComputeService, sshCommandExecutor, workingDir, PRIVATE_KEY, USER, SLAVE_DIRS, 1, 0, 1);
    ExecutionContext executionContext = provider.createExecutionContext();
    Set<String> hosts = Sets.newHashSet();
    for (Host host : executionContext.getHosts()) {
        hosts.add(host.getName());
    }
    Assert.assertEquals(Sets.newHashSet("1.1.1.1", "1.1.1.3"), hosts);
}

Example 94 with Set

use of java.util.Set in project kafka by apache.

the class AssignmentInfo method encode.

/**
     * @throws TaskAssignmentException if method fails to encode the data, e.g., if there is an
     * IO exception during encoding
     */
public ByteBuffer encode() {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    try {
        // Encode version
        out.writeInt(version);
        // Encode active tasks
        out.writeInt(activeTasks.size());
        for (TaskId id : activeTasks) {
            id.writeTo(out);
        }
        // Encode standby tasks
        out.writeInt(standbyTasks.size());
        for (Map.Entry<TaskId, Set<TopicPartition>> entry : standbyTasks.entrySet()) {
            TaskId id = entry.getKey();
            id.writeTo(out);
            Set<TopicPartition> partitions = entry.getValue();
            writeTopicPartitions(out, partitions);
        }
        out.writeInt(partitionsByHost.size());
        for (Map.Entry<HostInfo, Set<TopicPartition>> entry : partitionsByHost.entrySet()) {
            final HostInfo hostInfo = entry.getKey();
            out.writeUTF(hostInfo.host());
            out.writeInt(hostInfo.port());
            writeTopicPartitions(out, entry.getValue());
        }
        out.flush();
        out.close();
        return ByteBuffer.wrap(baos.toByteArray());
    } catch (IOException ex) {
        throw new TaskAssignmentException("Failed to encode AssignmentInfo", ex);
    }
}

Example 95 with Set

use of java.util.Set in project kafka by apache.

the class AssignmentInfo method decode.

/**
     * @throws TaskAssignmentException if method fails to decode the data or if the data version is unknown
     */
public static AssignmentInfo decode(ByteBuffer data) {
    // ensure we are at the beginning of the ByteBuffer
    data.rewind();
    DataInputStream in = new DataInputStream(new ByteBufferInputStream(data));
    try {
        // Decode version
        int version = in.readInt();
        if (version < 0 || version > CURRENT_VERSION) {
            TaskAssignmentException ex = new TaskAssignmentException("Unknown assignment data version: " + version);
            log.error(ex.getMessage(), ex);
            throw ex;
        }
        // Decode active tasks
        int count = in.readInt();
        List<TaskId> activeTasks = new ArrayList<>(count);
        for (int i = 0; i < count; i++) {
            activeTasks.add(TaskId.readFrom(in));
        }
        // Decode standby tasks
        count = in.readInt();
        Map<TaskId, Set<TopicPartition>> standbyTasks = new HashMap<>(count);
        for (int i = 0; i < count; i++) {
            TaskId id = TaskId.readFrom(in);
            standbyTasks.put(id, readTopicPartitions(in));
        }
        Map<HostInfo, Set<TopicPartition>> hostStateToTopicPartitions = new HashMap<>();
        if (version == CURRENT_VERSION) {
            int numEntries = in.readInt();
            for (int i = 0; i < numEntries; i++) {
                HostInfo hostInfo = new HostInfo(in.readUTF(), in.readInt());
                hostStateToTopicPartitions.put(hostInfo, readTopicPartitions(in));
            }
        }
        return new AssignmentInfo(activeTasks, standbyTasks, hostStateToTopicPartitions);
    } catch (IOException ex) {
        throw new TaskAssignmentException("Failed to decode AssignmentInfo", ex);
    }
}