Examples with ArrayBuffer scala.collection.mutable.ArrayBuffer used on opensource projects

Example 6 with ArrayBuffer

use of scala.collection.mutable.ArrayBuffer in project Gaffer by gchq.

the class ImportKeyValuePairRDDToAccumuloHandlerTest method checkImportRDDOfElements.

@Test
public void checkImportRDDOfElements(@TempDir Path tempDir) throws OperationException, IOException {
    final Graph graph1 = new Graph.Builder().config(new GraphConfig.Builder().graphId("graphId").build()).addSchema(getClass().getResourceAsStream("/schema/elements.json")).addSchema(getClass().getResourceAsStream("/schema/types.json")).addSchema(getClass().getResourceAsStream("/schema/serialisation.json")).storeProperties(PROPERTIES).build();
    final ArrayBuffer<Element> elements = new ArrayBuffer<>();
    for (int i = 0; i < 10; i++) {
        final Entity entity = new Entity.Builder().group(TestGroups.ENTITY).vertex("" + i).build();
        final Edge edge1 = new Edge.Builder().group(TestGroups.EDGE).source("" + i).dest("B").directed(false).property(TestPropertyNames.COUNT, 2).build();
        final Edge edge2 = new Edge.Builder().group(TestGroups.EDGE).source("" + i).dest("C").directed(false).property(TestPropertyNames.COUNT, 4).build();
        elements.$plus$eq(edge1);
        elements.$plus$eq(edge2);
        elements.$plus$eq(entity);
    }
    final User user = new User();
    final SparkSession sparkSession = SparkSessionProvider.getSparkSession();
    // Create Hadoop configuration and serialise to a string
    final Configuration configuration = new Configuration();
    final String configurationString = AbstractGetRDDHandler.convertConfigurationToString(configuration);
    final String outputPath = tempDir.resolve("output").toAbsolutePath().toString();
    final String failurePath = tempDir.resolve("failure").toAbsolutePath().toString();
    final ElementConverterFunction func = new ElementConverterFunction(sparkSession.sparkContext().broadcast(new ByteEntityAccumuloElementConverter(graph1.getSchema()), ACCUMULO_ELEMENT_CONVERTER_CLASS_TAG));
    final RDD<Tuple2<Key, Value>> elementRDD = sparkSession.sparkContext().parallelize(elements, 1, ELEMENT_CLASS_TAG).flatMap(func, TUPLE2_CLASS_TAG);
    final ImportKeyValuePairRDDToAccumulo addRdd = new ImportKeyValuePairRDDToAccumulo.Builder().input(elementRDD).outputPath(outputPath).failurePath(failurePath).build();
    graph1.execute(addRdd, user);
    // Check all elements were added
    final GetRDDOfAllElements rddQuery = new GetRDDOfAllElements.Builder().option(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString).build();
    final RDD<Element> rdd = graph1.execute(rddQuery, user);
    if (rdd == null) {
        fail("No RDD returned");
    }
    final Set<Element> results = new HashSet<>();
    final Element[] returnedElements = (Element[]) rdd.collect();
    Collections.addAll(results, returnedElements);
    assertEquals(elements.size(), results.size());
}

Example 7 with ArrayBuffer

use of scala.collection.mutable.ArrayBuffer in project Gaffer by gchq.

the class SplitStoreFromRDDOfElementsHandlerTest method createElements.

private ArrayBuffer<Element> createElements() {
    final ArrayBuffer<Element> elements = new ArrayBuffer<>();
    for (int i = 0; i < 10; i++) {
        final Entity entity = new Entity.Builder().group(TestGroups.ENTITY).vertex("" + i).build();
        final Edge edge1 = new Edge.Builder().group(TestGroups.EDGE).source("" + i).dest("B").directed(false).property(TestPropertyNames.COUNT, 2).build();
        final Edge edge2 = new Edge.Builder().group(TestGroups.EDGE).source("" + i).dest("C").directed(false).property(TestPropertyNames.COUNT, 4).build();
        elements.$plus$eq(edge1);
        elements.$plus$eq(edge2);
        elements.$plus$eq(entity);
    }
    return elements;
}

Example 8 with ArrayBuffer

use of scala.collection.mutable.ArrayBuffer in project flink by apache.

the class PythonCorrelateSplitRule method createNewFieldNames.

private List<String> createNewFieldNames(RelDataType rowType, RexBuilder rexBuilder, int primitiveFieldCount, ArrayBuffer<RexNode> extractedRexNodes, List<RexNode> calcProjects) {
    for (int i = 0; i < primitiveFieldCount; i++) {
        calcProjects.add(RexInputRef.of(i, rowType));
    }
    // change RexCorrelVariable to RexInputRef.
    RexDefaultVisitor<RexNode> visitor = new RexDefaultVisitor<RexNode>() {

        @Override
        public RexNode visitFieldAccess(RexFieldAccess fieldAccess) {
            RexNode expr = fieldAccess.getReferenceExpr();
            if (expr instanceof RexCorrelVariable) {
                RelDataTypeField field = fieldAccess.getField();
                return new RexInputRef(field.getIndex(), field.getType());
            } else {
                return rexBuilder.makeFieldAccess(expr.accept(this), fieldAccess.getField().getIndex());
            }
        }

        @Override
        public RexNode visitNode(RexNode rexNode) {
            return rexNode;
        }
    };
    // add the fields of the extracted rex calls.
    Iterator<RexNode> iterator = extractedRexNodes.iterator();
    while (iterator.hasNext()) {
        RexNode rexNode = iterator.next();
        if (rexNode instanceof RexCall) {
            RexCall rexCall = (RexCall) rexNode;
            List<RexNode> newProjects = rexCall.getOperands().stream().map(x -> x.accept(visitor)).collect(Collectors.toList());
            RexCall newRexCall = rexCall.clone(rexCall.getType(), newProjects);
            calcProjects.add(newRexCall);
        } else {
            calcProjects.add(rexNode);
        }
    }
    List<String> nameList = new LinkedList<>();
    for (int i = 0; i < primitiveFieldCount; i++) {
        nameList.add(rowType.getFieldNames().get(i));
    }
    Iterator<Object> indicesIterator = extractedRexNodes.indices().iterator();
    while (indicesIterator.hasNext()) {
        nameList.add("f" + indicesIterator.next());
    }
    return SqlValidatorUtil.uniquify(nameList, rexBuilder.getTypeFactory().getTypeSystem().isSchemaCaseSensitive());
}

Example 9 with ArrayBuffer

use of scala.collection.mutable.ArrayBuffer in project flink by apache.

the class PythonCorrelateSplitRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    FlinkLogicalCorrelate correlate = call.rel(0);
    RexBuilder rexBuilder = call.builder().getRexBuilder();
    RelNode left = ((HepRelVertex) correlate.getLeft()).getCurrentRel();
    RelNode right = ((HepRelVertex) correlate.getRight()).getCurrentRel();
    int primitiveLeftFieldCount = left.getRowType().getFieldCount();
    ArrayBuffer<RexNode> extractedRexNodes = new ArrayBuffer<>();
    RelNode rightNewInput;
    if (right instanceof FlinkLogicalTableFunctionScan) {
        FlinkLogicalTableFunctionScan scan = (FlinkLogicalTableFunctionScan) right;
        rightNewInput = createNewScan(scan, createScalarFunctionSplitter(null, rexBuilder, primitiveLeftFieldCount, extractedRexNodes, scan.getCall()));
    } else {
        FlinkLogicalCalc calc = (FlinkLogicalCalc) right;
        FlinkLogicalTableFunctionScan scan = StreamPhysicalCorrelateRule.getTableScan(calc);
        FlinkLogicalCalc mergedCalc = StreamPhysicalCorrelateRule.getMergedCalc(calc);
        FlinkLogicalTableFunctionScan newScan = createNewScan(scan, createScalarFunctionSplitter(null, rexBuilder, primitiveLeftFieldCount, extractedRexNodes, scan.getCall()));
        rightNewInput = mergedCalc.copy(mergedCalc.getTraitSet(), newScan, mergedCalc.getProgram());
    }
    FlinkLogicalCorrelate newCorrelate;
    if (extractedRexNodes.size() > 0) {
        FlinkLogicalCalc leftCalc = createNewLeftCalc(left, rexBuilder, extractedRexNodes, correlate);
        newCorrelate = new FlinkLogicalCorrelate(correlate.getCluster(), correlate.getTraitSet(), leftCalc, rightNewInput, correlate.getCorrelationId(), correlate.getRequiredColumns(), correlate.getJoinType());
    } else {
        newCorrelate = new FlinkLogicalCorrelate(correlate.getCluster(), correlate.getTraitSet(), left, rightNewInput, correlate.getCorrelationId(), correlate.getRequiredColumns(), correlate.getJoinType());
    }
    FlinkLogicalCalc newTopCalc = createTopCalc(primitiveLeftFieldCount, rexBuilder, extractedRexNodes, correlate.getRowType(), newCorrelate);
    call.transformTo(newTopCalc);
}