Examples with SplitStoreFromFile uk.gov.gchq.gaffer.operation.impl.SplitStoreFromFile used on opensource projects

Example 6 with SplitStoreFromFile

use of uk.gov.gchq.gaffer.operation.impl.SplitStoreFromFile in project Gaffer by gchq.

the class AddOperationsToChainTest method shouldHandleNestedOperationChain.

@Test
public void shouldHandleNestedOperationChain() throws SerialisationException {
    // Given
    AddOperationsToChain hook = fromJson(ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH);
    Operation discardOutput = new DiscardOutput();
    Operation splitStore = new SplitStoreFromFile();
    Operation validate = new Validate();
    Operation getAdjacentIds = new GetAdjacentIds();
    Operation count = new Count<>();
    Operation countGroups = new CountGroups();
    Operation getElements = new GetElements();
    If ifOp = new If.Builder<>().conditional(new Conditional(new Exists(), new GetElements())).then(new GetElements()).otherwise(new GetAllElements()).build();
    Operation getAllElements = new GetAllElements();
    Operation limit = new Limit<>();
    final OperationChain opChain = new OperationChain.Builder().first(getAdjacentIds).then(new OperationChain.Builder().first(getElements).then(getAllElements).build()).then(ifOp).build();
    // When
    hook.preExecute(opChain, new Context(new User()));
    // Then
    final OperationChain expectedOpChain = new OperationChain.Builder().first(discardOutput).then(splitStore).then(validate).then(getAdjacentIds).then(count).then(discardOutput).then((Operation) new OperationChain.Builder().first(countGroups).then(getElements).then(getAllElements).then(limit).then(validate).build()).then(new If.Builder<>().conditional(new Conditional(new Exists(), new OperationChain<>(new CountGroups(), new GetElements()))).then(new OperationChain<>(new CountGroups(), new GetElements())).otherwise(new OperationChain<>(new GetAllElements(), new Limit<>(), new Validate())).build()).then(new Count()).build();
    JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}

Example 7 with SplitStoreFromFile

use of uk.gov.gchq.gaffer.operation.impl.SplitStoreFromFile in project Gaffer by gchq.

the class AccumuloAddElementsFromHdfsJobFactoryTest method shouldSetNoLessThanMinNumberOfReducersSpecified.

@Test
public void shouldSetNoLessThanMinNumberOfReducersSpecified() throws IOException, StoreException, OperationException {
    // Given
    store.initialise("graphId", SCHEMA, PROPERTIES);
    final JobConf localConf = createLocalConf();
    final FileSystem fs = FileSystem.getLocal(localConf);
    fs.mkdirs(new Path(outputDir));
    fs.mkdirs(new Path(splitsDir));
    final BufferedWriter writer = new BufferedWriter(new FileWriter(splitsFile));
    for (int i = 100; i < 200; i++) {
        writer.write(i + "\n");
    }
    writer.close();
    final SplitStoreFromFile splitTable = new SplitStoreFromFile.Builder().inputPath(splitsFile).build();
    store.execute(splitTable, new Context(new User()));
    final AccumuloAddElementsFromHdfsJobFactory factory = getJobFactory();
    final Job job = Job.getInstance(localConf);
    // When
    AddElementsFromHdfs operation = new AddElementsFromHdfs.Builder().outputPath(outputDir).addInputMapperPair(inputDir, TextMapperGeneratorImpl.class.getName()).minReducers(10).splitsFilePath("target/data/splits.txt").build();
    factory.setupJob(job, operation, TextMapperGeneratorImpl.class.getName(), store);
    // Then
    assertTrue(job.getNumReduceTasks() >= 10);
    // When
    operation = new AddElementsFromHdfs.Builder().outputPath(outputDir).addInputMapperPair(inputDir, TextMapperGeneratorImpl.class.getName()).minReducers(100).splitsFilePath("target/data/splits.txt").build();
    factory.setupJob(job, operation, TextMapperGeneratorImpl.class.getName(), store);
    // Then
    assertTrue(job.getNumReduceTasks() >= 100);
    // When
    operation = new AddElementsFromHdfs.Builder().outputPath(outputDir).addInputMapperPair(inputDir, TextMapperGeneratorImpl.class.getName()).minReducers(1000).splitsFilePath("target/data/splits.txt").build();
    factory.setupJob(job, operation, TextMapperGeneratorImpl.class.getName(), store);
    // Then
    assertTrue(job.getNumReduceTasks() >= 1000);
}

Example 8 with SplitStoreFromFile

use of uk.gov.gchq.gaffer.operation.impl.SplitStoreFromFile in project Gaffer by gchq.

the class AddOperationsToChainTest method shouldFailQuietlyIfNestedOperationsCannotBeModified.

@Test
public void shouldFailQuietlyIfNestedOperationsCannotBeModified() throws SerialisationException {
    // Given
    AddOperationsToChain hook = fromJson(ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH);
    Operation discardOutput = new DiscardOutput();
    Operation splitStore = new SplitStoreFromFile();
    Operation validate = new Validate();
    Operation getAdjacentIds = new GetAdjacentIds();
    Operation count = new Count<>();
    Operation getElements = new GetElements();
    Operation getAllElements = new GetAllElements();
    TestUnmodifiableOperationsImpl nestedUnmodifiableOps = new TestUnmodifiableOperationsImpl(Arrays.asList(getAllElements, getElements));
    final OperationChain opChain = new OperationChain.Builder().first(getAdjacentIds).then(nestedUnmodifiableOps).build();
    // When
    hook.preExecute(opChain, new Context(new User()));
    // Then
    final OperationChain expectedOpChain = new OperationChain.Builder().first(discardOutput).then(splitStore).then(validate).then(getAdjacentIds).then(count).then(discardOutput).then(nestedUnmodifiableOps).then(count).build();
    JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}

Example 9 with SplitStoreFromFile

use of uk.gov.gchq.gaffer.operation.impl.SplitStoreFromFile in project Gaffer by gchq.

the class AddOperationsToChainTest method shouldHandleIfOperationWithNoConditionalOrOtherwise.

@Test
public void shouldHandleIfOperationWithNoConditionalOrOtherwise() throws SerialisationException {
    // Given
    AddOperationsToChain hook = fromJson(ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH);
    Operation discardOutput = new DiscardOutput();
    Operation splitStore = new SplitStoreFromFile();
    If ifOp = new If.Builder<>().then(new GetElements()).build();
    final OperationChain opChain = new OperationChain.Builder().first(ifOp).build();
    // When
    hook.preExecute(opChain, new Context(new User()));
    // Then
    final OperationChain expectedOpChain = new OperationChain.Builder().first(discardOutput).then(splitStore).then(new If.Builder<>().then(new OperationChain<>(new CountGroups(), new GetElements())).build()).then(new Count()).build();
    JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}

Example 10 with SplitStoreFromFile

use of uk.gov.gchq.gaffer.operation.impl.SplitStoreFromFile in project Gaffer by gchq.

the class AccumuloAddElementsFromHdfsJobFactoryTest method shouldThrowExceptionWhenMaxReducersSetOutsideOfRange.

@Test
public void shouldThrowExceptionWhenMaxReducersSetOutsideOfRange() throws IOException, StoreException, OperationException {
    // Given
    store.initialise("graphId", SCHEMA, PROPERTIES);
    final JobConf localConf = createLocalConf();
    final FileSystem fs = FileSystem.getLocal(localConf);
    fs.mkdirs(new Path(outputDir));
    fs.mkdirs(new Path(splitsDir));
    final BufferedWriter writer = new BufferedWriter(new FileWriter(splitsFile));
    for (int i = 100; i < 200; i++) {
        writer.write(i + "\n");
    }
    writer.close();
    final SplitStoreFromFile splitTable = new SplitStoreFromFile.Builder().inputPath(splitsFile).build();
    store.execute(splitTable, new Context(new User()));
    final AccumuloAddElementsFromHdfsJobFactory factory = getJobFactory();
    final Job job = Job.getInstance(localConf);
    // When
    AddElementsFromHdfs operation = new AddElementsFromHdfs.Builder().outputPath(outputDir).addInputMapperPair(inputDir, TextMapperGeneratorImpl.class.getName()).minReducers(100).maxReducers(101).splitsFilePath("target/data/splits.txt").build();
    // Then
    try {
        factory.setupJob(job, operation, TextMapperGeneratorImpl.class.getName(), store);
        fail("Exception expected");
    } catch (IllegalArgumentException e) {
        assertTrue(e.getMessage().contains("not a valid range"));
    }
}