Examples with BatchWriter org.apache.accumulo.core.client.BatchWriter used on opensource projects

Example 1 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project Gaffer by gchq.

the class AccumuloStore method insertGraphElements.

protected void insertGraphElements(final Iterable<Element> elements) throws StoreException {
    // Create BatchWriter
    final BatchWriter writer = TableUtils.createBatchWriter(this);
    // too high a latency, etc.
    if (elements != null) {
        for (final Element element : elements) {
            final Pair<Key> keys;
            try {
                keys = keyPackage.getKeyConverter().getKeysFromElement(element);
            } catch (final AccumuloElementConversionException e) {
                LOGGER.error("Failed to create an accumulo key from element of type " + element.getGroup() + " when trying to insert elements");
                continue;
            }
            final Value value;
            try {
                value = keyPackage.getKeyConverter().getValueFromElement(element);
            } catch (final AccumuloElementConversionException e) {
                LOGGER.error("Failed to create an accumulo value from element of type " + element.getGroup() + " when trying to insert elements");
                continue;
            }
            final Mutation m = new Mutation(keys.getFirst().getRow());
            m.put(keys.getFirst().getColumnFamily(), keys.getFirst().getColumnQualifier(), new ColumnVisibility(keys.getFirst().getColumnVisibility()), keys.getFirst().getTimestamp(), value);
            try {
                writer.addMutation(m);
            } catch (final MutationsRejectedException e) {
                LOGGER.error("Failed to create an accumulo key mutation");
                continue;
            }
            // If the GraphElement is an Edge then there will be 2 keys.
            if (keys.getSecond() != null) {
                final Mutation m2 = new Mutation(keys.getSecond().getRow());
                m2.put(keys.getSecond().getColumnFamily(), keys.getSecond().getColumnQualifier(), new ColumnVisibility(keys.getSecond().getColumnVisibility()), keys.getSecond().getTimestamp(), value);
                try {
                    writer.addMutation(m2);
                } catch (final MutationsRejectedException e) {
                    LOGGER.error("Failed to create an accumulo key mutation");
                }
            }
        }
    } else {
        throw new GafferRuntimeException("Could not find any elements to add to graph.", Status.BAD_REQUEST);
    }
    try {
        writer.close();
    } catch (final MutationsRejectedException e) {
        LOGGER.warn("Accumulo batch writer failed to close", e);
    }
}

Example 2 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class SplitRecoveryIT method test.

@Test
public void test() throws Exception {
    String tableName = getUniqueNames(1)[0];
    for (int tn = 0; tn < 2; tn++) {
        Connector connector = getConnector();
        // create a table and put some data in it
        connector.tableOperations().create(tableName);
        BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
        bw.addMutation(m("a"));
        bw.addMutation(m("b"));
        bw.addMutation(m("c"));
        bw.close();
        // take the table offline
        connector.tableOperations().offline(tableName);
        while (!isOffline(tableName, connector)) sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
        // poke a partial split into the metadata table
        connector.securityOperations().grantTablePermission(getAdminPrincipal(), MetadataTable.NAME, TablePermission.WRITE);
        Table.ID tableId = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName));
        KeyExtent extent = new KeyExtent(tableId, null, new Text("b"));
        Mutation m = extent.getPrevRowUpdateMutation();
        TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(0.5).getBytes()));
        TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(null));
        bw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
        bw.addMutation(m);
        if (tn == 1) {
            bw.flush();
            try (Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
                scanner.setRange(extent.toMetadataRange());
                scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
                KeyExtent extent2 = new KeyExtent(tableId, new Text("b"), null);
                m = extent2.getPrevRowUpdateMutation();
                TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value("/t2".getBytes()));
                TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value("M0".getBytes()));
                for (Entry<Key, Value> entry : scanner) {
                    m.put(DataFileColumnFamily.NAME, entry.getKey().getColumnQualifier(), entry.getValue());
                }
                bw.addMutation(m);
            }
        }
        bw.close();
        // bring the table online
        connector.tableOperations().online(tableName);
        // verify the tablets went online
        try (Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY)) {
            int i = 0;
            String[] expected = { "a", "b", "c" };
            for (Entry<Key, Value> entry : scanner) {
                assertEquals(expected[i], entry.getKey().getRow().toString());
                i++;
            }
            assertEquals(3, i);
            connector.tableOperations().delete(tableName);
        }
    }
}

Example 3 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class TableOperationsIT method getDiskUsage.

@Test
public void getDiskUsage() throws TableExistsException, AccumuloException, AccumuloSecurityException, TableNotFoundException, TException {
    final String[] names = getUniqueNames(2);
    String tableName = names[0];
    connector.tableOperations().create(tableName);
    // verify 0 disk usage
    List<DiskUsage> diskUsages = connector.tableOperations().getDiskUsage(Collections.singleton(tableName));
    assertEquals(1, diskUsages.size());
    assertEquals(1, diskUsages.get(0).getTables().size());
    assertEquals(Long.valueOf(0), diskUsages.get(0).getUsage());
    assertEquals(tableName, diskUsages.get(0).getTables().first());
    // add some data
    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
    Mutation m = new Mutation("a");
    m.put("b", "c", new Value("abcde".getBytes()));
    bw.addMutation(m);
    bw.flush();
    bw.close();
    connector.tableOperations().compact(tableName, new Text("A"), new Text("z"), true, true);
    // verify we have usage
    diskUsages = connector.tableOperations().getDiskUsage(Collections.singleton(tableName));
    assertEquals(1, diskUsages.size());
    assertEquals(1, diskUsages.get(0).getTables().size());
    assertTrue(diskUsages.get(0).getUsage() > 0);
    assertEquals(tableName, diskUsages.get(0).getTables().first());
    String newTable = names[1];
    // clone table
    connector.tableOperations().clone(tableName, newTable, false, null, null);
    // verify tables are exactly the same
    Set<String> tables = new HashSet<>();
    tables.add(tableName);
    tables.add(newTable);
    diskUsages = connector.tableOperations().getDiskUsage(tables);
    assertEquals(1, diskUsages.size());
    assertEquals(2, diskUsages.get(0).getTables().size());
    assertTrue(diskUsages.get(0).getUsage() > 0);
    connector.tableOperations().compact(tableName, new Text("A"), new Text("z"), true, true);
    connector.tableOperations().compact(newTable, new Text("A"), new Text("z"), true, true);
    // verify tables have differences
    diskUsages = connector.tableOperations().getDiskUsage(tables);
    assertEquals(2, diskUsages.size());
    assertEquals(1, diskUsages.get(0).getTables().size());
    assertEquals(1, diskUsages.get(1).getTables().size());
    assertTrue(diskUsages.get(0).getUsage() > 0);
    assertTrue(diskUsages.get(1).getUsage() > 0);
    connector.tableOperations().delete(tableName);
}

Example 4 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class TableOperationsIT method createMergeClonedTable.

@Test
public void createMergeClonedTable() throws Exception {
    String[] names = getUniqueNames(2);
    String originalTable = names[0];
    TableOperations tops = connector.tableOperations();
    TreeSet<Text> splits = Sets.newTreeSet(Arrays.asList(new Text("a"), new Text("b"), new Text("c"), new Text("d")));
    tops.create(originalTable);
    tops.addSplits(originalTable, splits);
    BatchWriter bw = connector.createBatchWriter(originalTable, new BatchWriterConfig());
    for (Text row : splits) {
        Mutation m = new Mutation(row);
        for (int i = 0; i < 10; i++) {
            for (int j = 0; j < 10; j++) {
                m.put(Integer.toString(i), Integer.toString(j), Integer.toString(i + j));
            }
        }
        bw.addMutation(m);
    }
    bw.close();
    String clonedTable = names[1];
    tops.clone(originalTable, clonedTable, true, null, null);
    tops.merge(clonedTable, null, new Text("b"));
    Map<String, Integer> rowCounts = new HashMap<>();
    try (Scanner s = connector.createScanner(clonedTable, new Authorizations())) {
        for (Entry<Key, Value> entry : s) {
            final Key key = entry.getKey();
            String row = key.getRow().toString();
            String cf = key.getColumnFamily().toString(), cq = key.getColumnQualifier().toString();
            String value = entry.getValue().toString();
            if (rowCounts.containsKey(row)) {
                rowCounts.put(row, rowCounts.get(row) + 1);
            } else {
                rowCounts.put(row, 1);
            }
            Assert.assertEquals(Integer.parseInt(cf) + Integer.parseInt(cq), Integer.parseInt(value));
        }
    }
    Collection<Text> clonedSplits = tops.listSplits(clonedTable);
    Set<Text> expectedSplits = Sets.newHashSet(new Text("b"), new Text("c"), new Text("d"));
    for (Text clonedSplit : clonedSplits) {
        Assert.assertTrue("Encountered unexpected split on the cloned table: " + clonedSplit, expectedSplits.remove(clonedSplit));
    }
    Assert.assertTrue("Did not find all expected splits on the cloned table: " + expectedSplits, expectedSplits.isEmpty());
}

Example 5 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class TestIngest method ingest.

public static void ingest(Connector connector, FileSystem fs, Opts opts, BatchWriterOpts bwOpts) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException, TableExistsException {
    long stopTime;
    byte[][] bytevals = generateValues(opts.dataSize);
    byte[] randomValue = new byte[opts.dataSize];
    Random random = new Random();
    long bytesWritten = 0;
    createTable(connector, opts);
    BatchWriter bw = null;
    FileSKVWriter writer = null;
    if (opts.outputFile != null) {
        Configuration conf = CachedConfiguration.getInstance();
        writer = FileOperations.getInstance().newWriterBuilder().forFile(opts.outputFile + "." + RFile.EXTENSION, fs, conf).withTableConfiguration(DefaultConfiguration.getInstance()).build();
        writer.startDefaultLocalityGroup();
    } else {
        bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
        connector.securityOperations().changeUserAuthorizations(opts.getPrincipal(), AUTHS);
    }
    Text labBA = new Text(opts.columnVisibility.getExpression());
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < opts.rows; i++) {
        int rowid;
        if (opts.stride > 0) {
            rowid = ((i % opts.stride) * (opts.rows / opts.stride)) + (i / opts.stride);
        } else {
            rowid = i;
        }
        Text row = generateRow(rowid, opts.startRow);
        Mutation m = new Mutation(row);
        for (int j = 0; j < opts.cols; j++) {
            Text colf = new Text(opts.columnFamily);
            Text colq = new Text(FastFormat.toZeroPaddedString(j, 7, 10, COL_PREFIX));
            if (writer != null) {
                Key key = new Key(row, colf, colq, labBA);
                if (opts.timestamp >= 0) {
                    key.setTimestamp(opts.timestamp);
                } else {
                    key.setTimestamp(startTime);
                }
                if (opts.delete) {
                    key.setDeleted(true);
                } else {
                    key.setDeleted(false);
                }
                bytesWritten += key.getSize();
                if (opts.delete) {
                    writer.append(key, new Value(new byte[0]));
                } else {
                    byte[] value;
                    if (opts.random != null) {
                        value = genRandomValue(random, randomValue, opts.random, rowid + opts.startRow, j);
                    } else {
                        value = bytevals[j % bytevals.length];
                    }
                    Value v = new Value(value);
                    writer.append(key, v);
                    bytesWritten += v.getSize();
                }
            } else {
                Key key = new Key(row, colf, colq, labBA);
                bytesWritten += key.getSize();
                if (opts.delete) {
                    if (opts.timestamp >= 0)
                        m.putDelete(colf, colq, opts.columnVisibility, opts.timestamp);
                    else
                        m.putDelete(colf, colq, opts.columnVisibility);
                } else {
                    byte[] value;
                    if (opts.random != null) {
                        value = genRandomValue(random, randomValue, opts.random, rowid + opts.startRow, j);
                    } else {
                        value = bytevals[j % bytevals.length];
                    }
                    bytesWritten += value.length;
                    if (opts.timestamp >= 0) {
                        m.put(colf, colq, opts.columnVisibility, opts.timestamp, new Value(value, true));
                    } else {
                        m.put(colf, colq, opts.columnVisibility, new Value(value, true));
                    }
                }
            }
        }
        if (bw != null)
            bw.addMutation(m);
    }
    if (writer != null) {
        writer.close();
    } else if (bw != null) {
        try {
            bw.close();
        } catch (MutationsRejectedException e) {
            if (e.getSecurityErrorCodes().size() > 0) {
                for (Entry<TabletId, Set<SecurityErrorCode>> entry : e.getSecurityErrorCodes().entrySet()) {
                    System.err.println("ERROR : Not authorized to write to : " + entry.getKey() + " due to " + entry.getValue());
                }
            }
            if (e.getConstraintViolationSummaries().size() > 0) {
                for (ConstraintViolationSummary cvs : e.getConstraintViolationSummaries()) {
                    System.err.println("ERROR : Constraint violates : " + cvs);
                }
            }
            throw e;
        }
    }
    stopTime = System.currentTimeMillis();
    int totalValues = opts.rows * opts.cols;
    double elapsed = (stopTime - startTime) / 1000.0;
    System.out.printf("%,12d records written | %,8d records/sec | %,12d bytes written | %,8d bytes/sec | %6.3f secs   %n", totalValues, (int) (totalValues / elapsed), bytesWritten, (int) (bytesWritten / elapsed), elapsed);
}