Examples with LoadPlan org.apache.accumulo.core.data.LoadPlan used on opensource projects

Example 1 with LoadPlan

use of org.apache.accumulo.core.data.LoadPlan in project accumulo by apache.

the class BulkNewIT method testBadLoadPlans.

@Test
public void testBadLoadPlans() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        addSplits(c, tableName, "0333 0666 0999 1333 1666");
        String dir = getDir("/testBulkFile-");
        writeData(dir + "/f1.", aconf, 0, 333);
        writeData(dir + "/f2.", aconf, 0, 666);
        final var importMappingOptions = c.tableOperations().importDirectory(dir).to(tableName);
        // Create a plan with more files than exists in dir
        LoadPlan loadPlan = LoadPlan.builder().loadFileTo("f1.rf", RangeType.TABLE, null, row(333)).loadFileTo("f2.rf", RangeType.TABLE, null, row(666)).loadFileTo("f3.rf", RangeType.TABLE, null, row(666)).build();
        final var tooManyFiles = importMappingOptions.plan(loadPlan);
        assertThrows(IllegalArgumentException.class, tooManyFiles::load);
        // Create a plan with fewer files than exists in dir
        loadPlan = LoadPlan.builder().loadFileTo("f1.rf", RangeType.TABLE, null, row(333)).build();
        final var tooFewFiles = importMappingOptions.plan(loadPlan);
        assertThrows(IllegalArgumentException.class, tooFewFiles::load);
        // Create a plan with tablet boundary that does not exist
        loadPlan = LoadPlan.builder().loadFileTo("f1.rf", RangeType.TABLE, null, row(555)).loadFileTo("f2.rf", RangeType.TABLE, null, row(555)).build();
        final var nonExistentBoundary = importMappingOptions.plan(loadPlan);
        assertThrows(AccumuloException.class, nonExistentBoundary::load);
    }
}

Example 2 with LoadPlan

use of org.apache.accumulo.core.data.LoadPlan in project accumulo by apache.

the class BulkNewIT method testBulkFileMax.

private void testBulkFileMax(boolean usePlan) throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        addSplits(c, tableName, "0333 0666 0999 1333 1666");
        String dir = getDir("/testBulkFileMax-");
        Map<String, Set<String>> hashes = new HashMap<>();
        for (String endRow : Arrays.asList("0333 0666 0999 1333 1666 null".split(" "))) {
            hashes.put(endRow, new HashSet<>());
        }
        // Add a junk file, should be ignored
        FSDataOutputStream out = fs.create(new Path(dir, "junk"));
        out.writeChars("ABCDEFG\n");
        out.close();
        // 1 Tablet 0333-null
        String h1 = writeData(dir + "/f1.", aconf, 0, 333);
        hashes.get("0333").add(h1);
        // 3 Tablets 0666-0334, 0999-0667, 1333-1000
        String h2 = writeData(dir + "/bad-file.", aconf, 334, 1333);
        hashes.get("0666").add(h2);
        hashes.get("0999").add(h2);
        hashes.get("1333").add(h2);
        // 1 Tablet 1666-1334
        String h3 = writeData(dir + "/f3.", aconf, 1334, 1499);
        hashes.get("1666").add(h3);
        // 2 Tablets 1666-1334, >1666
        String h4 = writeData(dir + "/f4.", aconf, 1500, 1999);
        hashes.get("1666").add(h4);
        hashes.get("null").add(h4);
        if (usePlan) {
            LoadPlan loadPlan = LoadPlan.builder().loadFileTo("f1.rf", RangeType.TABLE, null, row(333)).loadFileTo("bad-file.rf", RangeType.TABLE, row(333), row(1333)).loadFileTo("f3.rf", RangeType.FILE, row(1334), row(1499)).loadFileTo("f4.rf", RangeType.FILE, row(1500), row(1999)).build();
            c.tableOperations().importDirectory(dir).to(tableName).plan(loadPlan).load();
        } else {
            c.tableOperations().importDirectory(dir).to(tableName).load();
        }
        verifyData(c, tableName, 0, 1999, false);
        verifyMetadata(c, tableName, hashes);
    }
}

Example 3 with LoadPlan

use of org.apache.accumulo.core.data.LoadPlan in project accumulo by apache.

the class BulkNewIT method testBulkFile.

private void testBulkFile(boolean offline, boolean usePlan) throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        addSplits(c, tableName, "0333 0666 0999 1333 1666");
        if (offline) {
            c.tableOperations().offline(tableName);
        }
        String dir = getDir("/testBulkFile-");
        Map<String, Set<String>> hashes = new HashMap<>();
        for (String endRow : Arrays.asList("0333 0666 0999 1333 1666 null".split(" "))) {
            hashes.put(endRow, new HashSet<>());
        }
        // Add a junk file, should be ignored
        FSDataOutputStream out = fs.create(new Path(dir, "junk"));
        out.writeChars("ABCDEFG\n");
        out.close();
        // 1 Tablet 0333-null
        String h1 = writeData(dir + "/f1.", aconf, 0, 333);
        hashes.get("0333").add(h1);
        // 2 Tablets 0666-0334, 0999-0667
        String h2 = writeData(dir + "/f2.", aconf, 334, 999);
        hashes.get("0666").add(h2);
        hashes.get("0999").add(h2);
        // 2 Tablets 1333-1000, 1666-1334
        String h3 = writeData(dir + "/f3.", aconf, 1000, 1499);
        hashes.get("1333").add(h3);
        hashes.get("1666").add(h3);
        // 2 Tablets 1666-1334, >1666
        String h4 = writeData(dir + "/f4.", aconf, 1500, 1999);
        hashes.get("1666").add(h4);
        hashes.get("null").add(h4);
        if (usePlan) {
            LoadPlan loadPlan = LoadPlan.builder().loadFileTo("f1.rf", RangeType.TABLE, null, row(333)).loadFileTo("f2.rf", RangeType.TABLE, row(333), row(999)).loadFileTo("f3.rf", RangeType.FILE, row(1000), row(1499)).loadFileTo("f4.rf", RangeType.FILE, row(1500), row(1999)).build();
            c.tableOperations().importDirectory(dir).to(tableName).plan(loadPlan).load();
        } else {
            c.tableOperations().importDirectory(dir).to(tableName).load();
        }
        if (offline) {
            c.tableOperations().online(tableName);
        }
        verifyData(c, tableName, 0, 1999, false);
        verifyMetadata(c, tableName, hashes);
    }
}