Examples with AccumuloProperties uk.gov.gchq.gaffer.accumulostore.AccumuloProperties used on opensource projects

Example 11 with AccumuloProperties

use of uk.gov.gchq.gaffer.accumulostore.AccumuloProperties in project Gaffer by gchq.

the class TableUtilsTest method shouldCreateTableWithCorrectLocalityGroups.

@Test
public void shouldCreateTableWithCorrectLocalityGroups() throws Exception {
    final MockAccumuloStore store = new MockAccumuloStore();
    final Schema schema = new Schema.Builder().type(TestTypes.ID_STRING, String.class).type(TestTypes.DIRECTED_TRUE, Boolean.class).edge(TestGroups.EDGE, new SchemaEdgeDefinition.Builder().source(TestTypes.ID_STRING).destination(TestTypes.ID_STRING).directed(TestTypes.DIRECTED_TRUE).build()).build();
    final AccumuloProperties props = AccumuloProperties.loadStoreProperties(StreamUtil.storeProps(TableUtilsTest.class));
    props.setTable(LOCALITY_TABLE_NAME);
    store.initialise(schema, props);
    // When
    TableUtils.createTable(store);
    final Map<String, Set<Text>> localityGroups = store.getConnection().tableOperations().getLocalityGroups(LOCALITY_TABLE_NAME);
    assertEquals(1, localityGroups.size());
    Set<Text> localityGroup = localityGroups.get(TestGroups.EDGE);
    assertEquals(1, localityGroup.size());
    assertEquals(new Text(TestGroups.EDGE), localityGroup.toArray()[0]);
}

Example 12 with AccumuloProperties

use of uk.gov.gchq.gaffer.accumulostore.AccumuloProperties in project Gaffer by gchq.

the class TableUtilsTest method shouldCreateTableWithAllRequiredIterators.

@Test
public void shouldCreateTableWithAllRequiredIterators() throws Exception {
    // Given
    final MockAccumuloStore store = new MockAccumuloStore();
    final Schema schema = new Schema.Builder().type(TestTypes.ID_STRING, new TypeDefinition.Builder().aggregateFunction(new StringConcat()).clazz(String.class).build()).type(TestTypes.DIRECTED_TRUE, Boolean.class).edge(TestGroups.EDGE, new SchemaEdgeDefinition.Builder().source(TestTypes.ID_STRING).destination(TestTypes.ID_STRING).directed(TestTypes.DIRECTED_TRUE).build()).build();
    final AccumuloProperties props = AccumuloProperties.loadStoreProperties(StreamUtil.storeProps(TableUtilsTest.class));
    props.setTable(TABLE_NAME);
    store.initialise(schema, props);
    // When
    TableUtils.createTable(store);
    // Then
    final Map<String, EnumSet<IteratorScope>> itrs = store.getConnection().tableOperations().listIterators(TABLE_NAME);
    assertEquals(2, itrs.size());
    final EnumSet<IteratorScope> validator = itrs.get(AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME);
    assertEquals(EnumSet.allOf(IteratorScope.class), validator);
    final IteratorSetting validatorSetting = store.getConnection().tableOperations().getIteratorSetting(TABLE_NAME, AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME, IteratorScope.majc);
    assertEquals(AccumuloStoreConstants.VALIDATOR_ITERATOR_PRIORITY, validatorSetting.getPriority());
    assertEquals(ValidatorFilter.class.getName(), validatorSetting.getIteratorClass());
    final Map<String, String> validatorOptions = validatorSetting.getOptions();
    assertNotNull(Schema.fromJson(validatorOptions.get(AccumuloStoreConstants.SCHEMA).getBytes(CommonConstants.UTF_8)).getEdge(TestGroups.EDGE));
    assertEquals(ByteEntityAccumuloElementConverter.class.getName(), validatorOptions.get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS));
    final EnumSet<IteratorScope> aggregator = itrs.get(AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME);
    assertEquals(EnumSet.allOf(IteratorScope.class), aggregator);
    final IteratorSetting aggregatorSetting = store.getConnection().tableOperations().getIteratorSetting(TABLE_NAME, AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME, IteratorScope.majc);
    assertEquals(AccumuloStoreConstants.AGGREGATOR_ITERATOR_PRIORITY, aggregatorSetting.getPriority());
    assertEquals(AggregatorIterator.class.getName(), aggregatorSetting.getIteratorClass());
    final Map<String, String> aggregatorOptions = aggregatorSetting.getOptions();
    assertNotNull(Schema.fromJson(aggregatorOptions.get(AccumuloStoreConstants.SCHEMA).getBytes(CommonConstants.UTF_8)).getEdge(TestGroups.EDGE));
    assertEquals(ByteEntityAccumuloElementConverter.class.getName(), aggregatorOptions.get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS));
    final Map<String, String> tableProps = new HashMap<>();
    for (final Map.Entry<String, String> entry : store.getConnection().tableOperations().getProperties(TABLE_NAME)) {
        tableProps.put(entry.getKey(), entry.getValue());
    }
    assertEquals(0, Integer.parseInt(tableProps.get(Property.TABLE_FILE_REPLICATION.getKey())));
}

Example 13 with AccumuloProperties

use of uk.gov.gchq.gaffer.accumulostore.AccumuloProperties in project Gaffer by gchq.

the class TableUtilsTest method shouldCreateTableCorrectlyIfSchemaContainsNoAggregators.

@Test
public void shouldCreateTableCorrectlyIfSchemaContainsNoAggregators() throws Exception {
    // Given
    final MockAccumuloStore store = new MockAccumuloStore();
    final Schema schema = new Schema.Builder().type(TestTypes.ID_STRING, String.class).type(TestTypes.DIRECTED_TRUE, Boolean.class).edge(TestGroups.EDGE, new SchemaEdgeDefinition.Builder().source(TestTypes.ID_STRING).destination(TestTypes.ID_STRING).directed(TestTypes.DIRECTED_TRUE).build()).build();
    final AccumuloProperties props = AccumuloProperties.loadStoreProperties(StreamUtil.storeProps(TableUtilsTest.class));
    props.setTable(NO_AGGREGATORS_TABLE_NAME);
    store.initialise(schema, props);
    // When
    TableUtils.createTable(store);
    // Then
    final Map<String, EnumSet<IteratorScope>> itrs = store.getConnection().tableOperations().listIterators(NO_AGGREGATORS_TABLE_NAME);
    assertEquals(1, itrs.size());
    final EnumSet<IteratorScope> validator = itrs.get(AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME);
    assertEquals(EnumSet.allOf(IteratorScope.class), validator);
    final IteratorSetting validatorSetting = store.getConnection().tableOperations().getIteratorSetting(NO_AGGREGATORS_TABLE_NAME, AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME, IteratorScope.majc);
    assertEquals(AccumuloStoreConstants.VALIDATOR_ITERATOR_PRIORITY, validatorSetting.getPriority());
    assertEquals(ValidatorFilter.class.getName(), validatorSetting.getIteratorClass());
    final Map<String, String> validatorOptions = validatorSetting.getOptions();
    assertNotNull(Schema.fromJson(validatorOptions.get(AccumuloStoreConstants.SCHEMA).getBytes(CommonConstants.UTF_8)).getEdge(TestGroups.EDGE));
    assertEquals(ByteEntityAccumuloElementConverter.class.getName(), validatorOptions.get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS));
    final EnumSet<IteratorScope> aggregator = itrs.get(AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME);
    assertNull(aggregator);
    final IteratorSetting aggregatorSetting = store.getConnection().tableOperations().getIteratorSetting(NO_AGGREGATORS_TABLE_NAME, AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME, IteratorScope.majc);
    assertNull(aggregatorSetting);
    final Map<String, String> tableProps = new HashMap<>();
    for (final Map.Entry<String, String> entry : store.getConnection().tableOperations().getProperties(NO_AGGREGATORS_TABLE_NAME)) {
        tableProps.put(entry.getKey(), entry.getValue());
    }
    assertEquals(0, Integer.parseInt(tableProps.get(Property.TABLE_FILE_REPLICATION.getKey())));
}

Example 14 with AccumuloProperties

use of uk.gov.gchq.gaffer.accumulostore.AccumuloProperties in project Gaffer by gchq.

the class TableUtilsTest method shouldThrowExceptionIfTableNameIsNotSpecified.

@Test(expected = AccumuloRuntimeException.class)
public void shouldThrowExceptionIfTableNameIsNotSpecified() throws StoreException {
    // Given
    final Schema schema = new Schema.Builder().type("int", Integer.class).type("string", String.class).type("boolean", Boolean.class).edge("EDGE", new SchemaEdgeDefinition.Builder().source("string").destination("string").directed("boolean").build()).build();
    final AccumuloProperties properties = new AccumuloProperties();
    properties.setStoreClass(SingleUseMockAccumuloStore.class.getName());
    final AccumuloStore store = new AccumuloStore();
    store.initialise(schema, properties);
    // When
    TableUtils.ensureTableExists(store);
    fail("The expected exception was not thrown.");
}

Example 15 with AccumuloProperties

use of uk.gov.gchq.gaffer.accumulostore.AccumuloProperties in project Gaffer by gchq.

the class AccumuloAddElementsFromHdfsJobFactoryTest method shouldSetNumberOfReducersBetweenMinAndMaxSpecified.

@Test
public void shouldSetNumberOfReducersBetweenMinAndMaxSpecified() throws IOException, StoreException, OperationException {
    // Given
    final MockAccumuloStore store = new MockAccumuloStore();
    final Schema schema = Schema.fromJson(StreamUtil.schemas(AccumuloAddElementsFromHdfsJobFactoryTest.class));
    final AccumuloProperties properties = AccumuloProperties.loadStoreProperties(StreamUtil.storeProps(AccumuloAddElementsFromHdfsJobFactoryTest.class));
    store.initialise(schema, properties);
    final JobConf localConf = createLocalConf();
    final FileSystem fs = FileSystem.getLocal(localConf);
    fs.mkdirs(new Path(outputDir));
    fs.mkdirs(new Path(splitsDir));
    final BufferedWriter writer = new BufferedWriter(new FileWriter(splitsFile));
    for (int i = 100; i < 200; i++) {
        writer.write(i + "\n");
    }
    writer.close();
    final SplitTable splitTable = new SplitTable.Builder().inputPath(splitsFile).build();
    store.execute(splitTable, new User());
    final AccumuloAddElementsFromHdfsJobFactory factory = new AccumuloAddElementsFromHdfsJobFactory();
    final Job job = Job.getInstance(localConf);
    // When
    AddElementsFromHdfs operation = new AddElementsFromHdfs.Builder().outputPath(outputDir).mapperGenerator(TextMapperGeneratorImpl.class).option(AccumuloStoreConstants.OPERATION_BULK_IMPORT_MIN_REDUCERS, "10").option(AccumuloStoreConstants.OPERATION_BULK_IMPORT_MAX_REDUCERS, "20").option(AccumuloStoreConstants.OPERATION_HDFS_SPLITS_FILE_PATH, "target/data/splits.txt").build();
    factory.setupJobConf(localConf, operation, store);
    factory.setupJob(job, operation, store);
    // Then
    assertTrue(job.getNumReduceTasks() >= 10);
    assertTrue(job.getNumReduceTasks() <= 20);
    // When
    operation = new AddElementsFromHdfs.Builder().outputPath(outputDir).mapperGenerator(TextMapperGeneratorImpl.class).option(AccumuloStoreConstants.OPERATION_BULK_IMPORT_MIN_REDUCERS, "100").option(AccumuloStoreConstants.OPERATION_BULK_IMPORT_MAX_REDUCERS, "200").option(AccumuloStoreConstants.OPERATION_HDFS_SPLITS_FILE_PATH, "target/data/splits.txt").build();
    factory.setupJobConf(localConf, operation, store);
    factory.setupJob(job, operation, store);
    // Then
    assertTrue(job.getNumReduceTasks() >= 100);
    assertTrue(job.getNumReduceTasks() <= 200);
    // When
    operation = new AddElementsFromHdfs.Builder().outputPath(outputDir).mapperGenerator(TextMapperGeneratorImpl.class).option(AccumuloStoreConstants.OPERATION_BULK_IMPORT_MIN_REDUCERS, "1000").option(AccumuloStoreConstants.OPERATION_BULK_IMPORT_MAX_REDUCERS, "2000").option(AccumuloStoreConstants.OPERATION_HDFS_SPLITS_FILE_PATH, "target/data/splits.txt").build();
    factory.setupJobConf(localConf, operation, store);
    factory.setupJob(job, operation, store);
    // Then
    assertTrue(job.getNumReduceTasks() >= 1000);
    assertTrue(job.getNumReduceTasks() <= 2000);
}