Examples with DataFileValue org.apache.accumulo.core.metadata.schema.DataFileValue used on opensource projects

Example 26 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class Tablet method computeNumEntries.

synchronized void computeNumEntries() {
    Collection<DataFileValue> vals = getDatafileManager().getDatafileSizes().values();
    long numEntries = 0;
    for (DataFileValue tableValue : vals) {
        numEntries += tableValue.getNumEntries();
    }
    this.numEntriesInMemory = getTabletMemory().getNumEntries();
    numEntries += getTabletMemory().getNumEntries();
    this.numEntries = numEntries;
}

Example 27 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class Tablet method getFirstAndLastKeys.

private Map<FileRef, Pair<Key, Key>> getFirstAndLastKeys(SortedMap<FileRef, DataFileValue> allFiles) throws IOException {
    Map<FileRef, Pair<Key, Key>> result = new HashMap<>();
    FileOperations fileFactory = FileOperations.getInstance();
    VolumeManager fs = getTabletServer().getFileSystem();
    for (Entry<FileRef, DataFileValue> entry : allFiles.entrySet()) {
        FileRef file = entry.getKey();
        FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
        try (FileSKVIterator openReader = fileFactory.newReaderBuilder().forFile(file.path().toString(), ns, ns.getConf()).withTableConfiguration(this.getTableConfiguration()).seekToBeginning().build()) {
            Key first = openReader.getFirstKey();
            Key last = openReader.getLastKey();
            result.put(file, new Pair<>(first, last));
        }
    }
    return result;
}

Example 28 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class DatafileManagerTest method testReserveMergingMinorCompactionFileDisabled.

/*
   * Test disabled max file size for merging minor compaction
   */
@Test
public void testReserveMergingMinorCompactionFileDisabled() throws IOException {
    String maxMergeFileSize = "0";
    EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
    EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
    EasyMock.replay(tablet, tableConf);
    SortedMap<FileRef, DataFileValue> testFiles = createFileMap("smallishfile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
    DatafileManager dfm = new DatafileManager(tablet, testFiles);
    FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
    EasyMock.verify(tablet, tableConf);
    assertEquals("smallishfile", mergeFile.path().getName());
}

Example 29 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class TwoTierCompactionStrategyTest method testLargeCompaction.

@Test
public void testLargeCompaction() throws IOException {
    ttcs.init(opts);
    conf = DefaultConfiguration.getInstance();
    KeyExtent ke = new KeyExtent(Table.ID.of("0"), null, null);
    mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf);
    Map<FileRef, DataFileValue> fileMap = createFileMap("f1", "2G", "f2", "2G", "f3", "2G", "f4", "2G");
    mcr.setFiles(fileMap);
    Assert.assertTrue(ttcs.shouldCompact(mcr));
    Assert.assertEquals(4, mcr.getFiles().size());
    List<FileRef> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
    Assert.assertEquals(fileMap.keySet(), new HashSet<>(filesToCompact));
    Assert.assertEquals(4, filesToCompact.size());
    Assert.assertEquals(largeCompressionType, ttcs.getCompactionPlan(mcr).writeParameters.getCompressType());
}

Example 30 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class TwoTierCompactionStrategyTest method testFileSubsetCompaction.

@Test
public void testFileSubsetCompaction() throws IOException {
    ttcs.init(opts);
    conf = DefaultConfiguration.getInstance();
    KeyExtent ke = new KeyExtent(Table.ID.of("0"), null, null);
    mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf);
    Map<FileRef, DataFileValue> fileMap = createFileMap("f1", "1G", "f2", "10M", "f3", "10M", "f4", "10M", "f5", "10M", "f6", "10M", "f7", "10M");
    Map<FileRef, DataFileValue> filesToCompactMap = createFileMap("f2", "10M", "f3", "10M", "f4", "10M", "f5", "10M", "f6", "10M", "f7", "10M");
    mcr.setFiles(fileMap);
    Assert.assertTrue(ttcs.shouldCompact(mcr));
    Assert.assertEquals(7, mcr.getFiles().size());
    List<FileRef> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
    Assert.assertEquals(filesToCompactMap.keySet(), new HashSet<>(filesToCompact));
    Assert.assertEquals(6, filesToCompact.size());
    Assert.assertEquals(null, ttcs.getCompactionPlan(mcr).writeParameters.getCompressType());
}