Examples with StoredTabletFile org.apache.accumulo.core.metadata.StoredTabletFile used on opensource projects

Example 6 with StoredTabletFile

use of org.apache.accumulo.core.metadata.StoredTabletFile in project accumulo by apache.

the class ManagerMetadataUtil method fixSplit.

private static KeyExtent fixSplit(ServerContext context, TableId tableId, Text metadataEntry, Text metadataPrevEndRow, Text oper, double splitRatio, ServiceLock lock) throws AccumuloException {
    if (metadataPrevEndRow == null)
        // prev end row....
        throw new AccumuloException("Split tablet does not have prev end row, something is amiss, extent = " + metadataEntry);
    // check to see if prev tablet exist in metadata tablet
    Key prevRowKey = new Key(new Text(TabletsSection.encodeRow(tableId, metadataPrevEndRow)));
    try (ScannerImpl scanner2 = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY)) {
        scanner2.setRange(new Range(prevRowKey, prevRowKey.followingKey(PartialKey.ROW)));
        if (scanner2.iterator().hasNext()) {
            log.info("Finishing incomplete split {} {}", metadataEntry, metadataPrevEndRow);
            List<StoredTabletFile> highDatafilesToRemove = new ArrayList<>();
            SortedMap<StoredTabletFile, DataFileValue> origDatafileSizes = new TreeMap<>();
            SortedMap<StoredTabletFile, DataFileValue> highDatafileSizes = new TreeMap<>();
            SortedMap<StoredTabletFile, DataFileValue> lowDatafileSizes = new TreeMap<>();
            Key rowKey = new Key(metadataEntry);
            try (Scanner scanner3 = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY)) {
                scanner3.fetchColumnFamily(DataFileColumnFamily.NAME);
                scanner3.setRange(new Range(rowKey, rowKey.followingKey(PartialKey.ROW)));
                for (Entry<Key, Value> entry : scanner3) {
                    if (entry.getKey().compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
                        StoredTabletFile stf = new StoredTabletFile(entry.getKey().getColumnQualifierData().toString());
                        origDatafileSizes.put(stf, new DataFileValue(entry.getValue().get()));
                    }
                }
            }
            MetadataTableUtil.splitDatafiles(metadataPrevEndRow, splitRatio, new HashMap<>(), origDatafileSizes, lowDatafileSizes, highDatafileSizes, highDatafilesToRemove);
            MetadataTableUtil.finishSplit(metadataEntry, highDatafileSizes, highDatafilesToRemove, context, lock);
            return KeyExtent.fromMetaRow(rowKey.getRow(), metadataPrevEndRow);
        } else {
            log.info("Rolling back incomplete split {} {}", metadataEntry, metadataPrevEndRow);
            MetadataTableUtil.rollBackSplit(metadataEntry, oper, context, lock);
            return KeyExtent.fromMetaRow(metadataEntry, oper);
        }
    }
}

Example 7 with StoredTabletFile

use of org.apache.accumulo.core.metadata.StoredTabletFile in project accumulo by apache.

the class ManagerMetadataUtil method addNewTablet.

public static void addNewTablet(ServerContext context, KeyExtent extent, String dirName, TServerInstance location, Map<StoredTabletFile, DataFileValue> datafileSizes, Map<Long, ? extends Collection<TabletFile>> bulkLoadedFiles, MetadataTime time, long lastFlushID, long lastCompactID, ServiceLock zooLock) {
    TabletMutator tablet = context.getAmple().mutateTablet(extent);
    tablet.putPrevEndRow(extent.prevEndRow());
    tablet.putZooLock(zooLock);
    tablet.putDirName(dirName);
    tablet.putTime(time);
    if (lastFlushID > 0)
        tablet.putFlushId(lastFlushID);
    if (lastCompactID > 0)
        tablet.putCompactionId(lastCompactID);
    if (location != null) {
        tablet.putLocation(location, LocationType.CURRENT);
        tablet.deleteLocation(location, LocationType.FUTURE);
    }
    datafileSizes.forEach(tablet::putFile);
    for (Entry<Long, ? extends Collection<TabletFile>> entry : bulkLoadedFiles.entrySet()) {
        for (TabletFile ref : entry.getValue()) {
            tablet.putBulkFile(ref, entry.getKey());
        }
    }
    tablet.mutate();
}

Example 8 with StoredTabletFile

use of org.apache.accumulo.core.metadata.StoredTabletFile in project accumulo by apache.

the class BasicCompactionStrategyTest method testLargeCompaction.

@Test
public void testLargeCompaction() {
    ttcs.init(opts);
    conf = DefaultConfiguration.getInstance();
    KeyExtent ke = new KeyExtent(TableId.of("0"), null, null);
    mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf, getServerContext());
    Map<StoredTabletFile, DataFileValue> fileMap = createFileMap("f1", "2G", "f2", "2G", "f3", "2G", "f4", "2G");
    mcr.setFiles(fileMap);
    assertTrue(ttcs.shouldCompact(mcr));
    assertEquals(4, mcr.getFiles().size());
    List<StoredTabletFile> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
    assertEquals(fileMap.keySet(), new HashSet<>(filesToCompact));
    assertEquals(4, filesToCompact.size());
    assertEquals(largeCompressionType, ttcs.getCompactionPlan(mcr).writeParameters.getCompressType());
}

Example 9 with StoredTabletFile

use of org.apache.accumulo.core.metadata.StoredTabletFile in project accumulo by apache.

the class BasicCompactionStrategyTest method testFileSubsetCompaction.

@Test
public void testFileSubsetCompaction() {
    ttcs.init(opts);
    conf = DefaultConfiguration.getInstance();
    KeyExtent ke = new KeyExtent(TableId.of("0"), null, null);
    mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf, getServerContext());
    Map<StoredTabletFile, DataFileValue> fileMap = createFileMap("f1", "1G", "f2", "10M", "f3", "10M", "f4", "10M", "f5", "10M", "f6", "10M", "f7", "10M");
    Map<StoredTabletFile, DataFileValue> filesToCompactMap = createFileMap("f2", "10M", "f3", "10M", "f4", "10M", "f5", "10M", "f6", "10M", "f7", "10M");
    mcr.setFiles(fileMap);
    assertTrue(ttcs.shouldCompact(mcr));
    assertEquals(7, mcr.getFiles().size());
    List<StoredTabletFile> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
    assertEquals(filesToCompactMap.keySet(), new HashSet<>(filesToCompact));
    assertEquals(6, filesToCompact.size());
    assertNull(ttcs.getCompactionPlan(mcr).writeParameters);
}

Example 10 with StoredTabletFile

use of org.apache.accumulo.core.metadata.StoredTabletFile in project accumulo by apache.

the class BasicCompactionStrategyTest method testDefaultCompaction.

@Test
public void testDefaultCompaction() {
    ttcs.init(opts);
    conf = DefaultConfiguration.getInstance();
    KeyExtent ke = new KeyExtent(TableId.of("0"), null, null);
    mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf, InMemoryMapTest.getServerContext());
    Map<StoredTabletFile, DataFileValue> fileMap = createFileMap("f1", "10M", "f2", "10M", "f3", "10M", "f4", "10M", "f5", "100M", "f6", "100M", "f7", "100M", "f8", "100M");
    mcr.setFiles(fileMap);
    assertTrue(ttcs.shouldCompact(mcr));
    assertEquals(8, mcr.getFiles().size());
    List<StoredTabletFile> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
    assertEquals(fileMap.keySet(), new HashSet<>(filesToCompact));
    assertEquals(8, filesToCompact.size());
    assertNull(ttcs.getCompactionPlan(mcr).writeParameters);
}