Examples with LoadMappingIterator org.apache.accumulo.core.clientImpl.bulk.LoadMappingIterator used on opensource projects

Example 1 with LoadMappingIterator

use of org.apache.accumulo.core.clientImpl.bulk.LoadMappingIterator in project accumulo by apache.

the class PrepBulkImport method checkForMerge.

private void checkForMerge(final long tid, final Manager manager) throws Exception {
    VolumeManager fs = manager.getVolumeManager();
    final Path bulkDir = new Path(bulkInfo.sourceDir);
    int maxTablets = Integer.parseInt(manager.getContext().getTableConfiguration(bulkInfo.tableId).get(Property.TABLE_BULK_MAX_TABLETS));
    try (LoadMappingIterator lmi = BulkSerialize.readLoadMapping(bulkDir.toString(), bulkInfo.tableId, fs::open)) {
        TabletIterFactory tabletIterFactory = startRow -> TabletsMetadata.builder(manager.getContext()).forTable(bulkInfo.tableId).overlapping(startRow, null).checkConsistency().fetch(PREV_ROW).build().stream().map(TabletMetadata::getExtent).iterator();
        sanityCheckLoadMapping(bulkInfo.tableId.canonical(), lmi, tabletIterFactory, maxTablets, tid);
    }
}

Example 2 with LoadMappingIterator

use of org.apache.accumulo.core.clientImpl.bulk.LoadMappingIterator in project accumulo by apache.

the class PrepBulkImportTest method createLoadMappingIter.

private LoadMappingIterator createLoadMappingIter(Map<KeyExtent, String> loadRanges) throws IOException {
    SortedMap<KeyExtent, Bulk.Files> mapping = new TreeMap<>();
    loadRanges.forEach((extent, files) -> {
        Bulk.Files testFiles = new Bulk.Files();
        long c = 0L;
        for (String f : files.split(" ")) {
            c++;
            testFiles.add(new Bulk.FileInfo(f, c, c));
        }
        mapping.put(extent, testFiles);
    });
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    BulkSerialize.writeLoadMapping(mapping, "/some/dir", p -> baos);
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    LoadMappingIterator lmi = BulkSerialize.readLoadMapping("/some/dir", TableId.of("1"), p -> bais);
    return lmi;
}

Example 3 with LoadMappingIterator

use of org.apache.accumulo.core.clientImpl.bulk.LoadMappingIterator in project accumulo by apache.

the class LoadFiles method isReady.

@Override
public long isReady(long tid, Manager manager) throws Exception {
    if (manager.onlineTabletServers().isEmpty()) {
        log.warn("There are no tablet server to process bulkDir import, waiting (tid = " + FateTxId.formatTid(tid) + ")");
        return 100;
    }
    VolumeManager fs = manager.getVolumeManager();
    final Path bulkDir = new Path(bulkInfo.bulkDir);
    manager.updateBulkImportStatus(bulkInfo.sourceDir, BulkImportState.LOADING);
    try (LoadMappingIterator lmi = BulkSerialize.getUpdatedLoadMapping(bulkDir.toString(), bulkInfo.tableId, fs::open)) {
        return loadFiles(bulkInfo.tableId, bulkDir, lmi, manager, tid);
    }
}