Examples with SplitOperatorDescriptor org.apache.hyracks.algebricks.runtime.operators.std.SplitOperatorDescriptor used on opensource projects

Example 1 with SplitOperatorDescriptor

use of org.apache.hyracks.algebricks.runtime.operators.std.SplitOperatorDescriptor in project asterixdb by apache.

the class PushRuntimeTest method scanSplitWrite.

@Test
public void scanSplitWrite() throws Exception {
    final int outputArity = 2;
    JobSpecification spec = new JobSpecification(FRAME_SIZE);
    String[] inputFileName = { "data" + File.separator + "simple" + File.separator + "int-string-part1.tbl", "data" + File.separator + "simple" + File.separator + "int-string-part1-split-0.tbl", "data" + File.separator + "simple" + File.separator + "int-string-part1-split-1.tbl" };
    File[] inputFiles = new File[inputFileName.length];
    for (int i = 0; i < inputFileName.length; i++) {
        inputFiles[i] = new File(inputFileName[i]);
    }
    File[] outputFile = new File[outputArity];
    FileSplit[] outputFileSplit = new FileSplit[outputArity];
    for (int i = 0; i < outputArity; i++) {
        outputFileSplit[i] = createFile(AlgebricksHyracksIntegrationUtil.nc1);
        outputFile[i] = outputFileSplit[i].getFile(AlgebricksHyracksIntegrationUtil.nc1.getIoManager());
    }
    FileSplit[] inputSplits = new FileSplit[] { new ManagedFileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, inputFileName[0]) };
    IFileSplitProvider intSplitProvider = new ConstantFileSplitProvider(inputSplits);
    RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() });
    IValueParserFactory[] valueParsers = new IValueParserFactory[] { IntegerParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE };
    FileScanOperatorDescriptor intScanner = new FileScanOperatorDescriptor(spec, intSplitProvider, new DelimitedDataTupleParserFactory(valueParsers, '|'), scannerDesc);
    PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, intScanner, DEFAULT_NODES);
    SplitOperatorDescriptor splitOp = new SplitOperatorDescriptor(spec, scannerDesc, outputArity, new TupleFieldEvaluatorFactory(0), BinaryIntegerInspectorImpl.FACTORY);
    PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, splitOp, new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
    IOperatorDescriptor[] outputOp = new IOperatorDescriptor[outputFile.length];
    for (int i = 0; i < outputArity; i++) {
        outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] { outputFileSplit[i] });
        PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, outputOp[i], new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
    }
    spec.connect(new OneToOneConnectorDescriptor(spec), intScanner, 0, splitOp, 0);
    for (int i = 0; i < outputArity; i++) {
        spec.connect(new OneToOneConnectorDescriptor(spec), splitOp, i, outputOp[i], 0);
    }
    for (int i = 0; i < outputArity; i++) {
        spec.addRoot(outputOp[i]);
    }
    AlgebricksHyracksIntegrationUtil.runJob(spec);
    for (int i = 0; i < outputArity; i++) {
        compareFiles("data" + File.separator + "device0" + File.separator + inputFileName[i + 1], outputFile[i].getAbsolutePath());
    }
}

Example 2 with SplitOperatorDescriptor

use of org.apache.hyracks.algebricks.runtime.operators.std.SplitOperatorDescriptor in project asterixdb by apache.

the class SplitPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    SplitOperator sop = (SplitOperator) op;
    int outputArity = sop.getOutputArity();
    int defaultBranch = sop.getDefaultBranch();
    boolean propageToAllBranchAsDefault = sop.getPropageToAllBranchAsDefault();
    IOperatorDescriptorRegistry spec = builder.getJobSpec();
    RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
    IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
    IScalarEvaluatorFactory brachingExprEvalFactory = expressionRuntimeProvider.createEvaluatorFactory(sop.getBranchingExpression().getValue(), context.getTypeEnvironment(op), inputSchemas, context);
    IBinaryIntegerInspectorFactory intInsepctorFactory = context.getBinaryIntegerInspectorFactory();
    SplitOperatorDescriptor splitOpDesc = new SplitOperatorDescriptor(spec, recDescriptor, outputArity, brachingExprEvalFactory, intInsepctorFactory, defaultBranch, propageToAllBranchAsDefault);
    contributeOpDesc(builder, (AbstractLogicalOperator) op, splitOpDesc);
    ILogicalOperator src = op.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src, 0, op, 0);
}