Examples with FileSKVIterator org.apache.accumulo.core.file.FileSKVIterator used on opensource projects

Example 6 with FileSKVIterator

use of org.apache.accumulo.core.file.FileSKVIterator in project accumulo by apache.

the class RFileTest method test17.

@Test
public void test17() throws IOException {
    // add alot of the same keys to rfile that cover multiple blocks...
    // this should cause the keys in the index to be exactly the same...
    // ensure seeks work correctly
    TestRFile trf = new TestRFile(conf);
    trf.openWriter(false);
    trf.writer.startDefaultLocalityGroup();
    for (int i = 0; i < 2048; i++) {
        trf.writer.append(newKey("r0000", "cf1", "cq1", "", 1), newValue("" + i));
    }
    for (int i = 2048; i < 4096; i++) {
        trf.writer.append(newKey("r0001", "cf1", "cq1", "", 1), newValue("" + i));
    }
    trf.writer.close();
    trf.openReader();
    FileSKVIterator indexIter = trf.reader.getIndex();
    int count = 0;
    while (indexIter.hasTop()) {
        count++;
        indexIter.next();
    }
    assertTrue(count > 4);
    trf.iter.seek(new Range(newKey("r0000", "cf1", "cq1", "", 1), true, newKey("r0001", "cf1", "cq1", "", 1), false), EMPTY_COL_FAMS, false);
    for (int i = 0; i < 2048; i++) {
        assertTrue(trf.iter.hasTop());
        assertEquals(newKey("r0000", "cf1", "cq1", "", 1), trf.iter.getTopKey());
        assertEquals(newValue("" + i), trf.iter.getTopValue());
        trf.iter.next();
    }
    assertFalse(trf.iter.hasTop());
    trf.iter.seek(new Range(newKey("r0000", "cf1", "cq1", "", 1), false, newKey("r0001", "cf1", "cq1", "", 1), true), EMPTY_COL_FAMS, false);
    for (int i = 2048; i < 4096; i++) {
        assertTrue(trf.iter.hasTop());
        assertEquals(newKey("r0001", "cf1", "cq1", "", 1), trf.iter.getTopKey());
        assertEquals(newValue("" + i), trf.iter.getTopValue());
        trf.iter.next();
    }
    assertFalse(trf.iter.hasTop());
    trf.iter.seek(new Range(newKey("r0001", "cf1", "cq1", "", 1), true, newKey("r0001", "cf1", "cq1", "", 1), true), EMPTY_COL_FAMS, false);
    for (int i = 2048; i < 4096; i++) {
        assertTrue(trf.iter.hasTop());
        assertEquals(newKey("r0001", "cf1", "cq1", "", 1), trf.iter.getTopKey());
        assertEquals(newValue("" + i), trf.iter.getTopValue());
        trf.iter.next();
    }
    assertFalse(trf.iter.hasTop());
    trf.iter.seek(new Range(newKey("r0002", "cf1", "cq1", "", 1), true, newKey("r0002", "cf1", "cq1", "", 1), true), EMPTY_COL_FAMS, false);
    assertFalse(trf.iter.hasTop());
    trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
    for (int i = 0; i < 2048; i++) {
        assertTrue(trf.iter.hasTop());
        assertEquals(newKey("r0000", "cf1", "cq1", "", 1), trf.iter.getTopKey());
        assertEquals(newValue("" + i), trf.iter.getTopValue());
        trf.iter.next();
    }
    for (int i = 2048; i < 4096; i++) {
        assertTrue(trf.iter.hasTop());
        assertEquals(newKey("r0001", "cf1", "cq1", "", 1), trf.iter.getTopKey());
        assertEquals(newValue("" + i), trf.iter.getTopValue());
        trf.iter.next();
    }
    assertFalse(trf.iter.hasTop());
    trf.closeReader();
}

Example 7 with FileSKVIterator

use of org.apache.accumulo.core.file.FileSKVIterator in project accumulo by apache.

the class RFileTest method testWriterTableProperties.

@Test
public void testWriterTableProperties() throws Exception {
    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    Map<String, String> props = new HashMap<>();
    props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1K");
    props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX.getKey(), "1K");
    RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).withTableProperties(props).build();
    SortedMap<Key, Value> testData1 = createTestData(10, 10, 10);
    writer.append(testData1.entrySet());
    writer.close();
    Reader reader = getReader(localFs, testFile);
    FileSKVIterator iiter = reader.getIndex();
    int count = 0;
    while (iiter.hasTop()) {
        count++;
        iiter.next();
    }
    // if settings are used then should create multiple index entries
    Assert.assertTrue(count > 10);
    reader.close();
    Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build();
    Assert.assertEquals(testData1, toMap(scanner));
    scanner.close();
}

Example 8 with FileSKVIterator

use of org.apache.accumulo.core.file.FileSKVIterator in project accumulo by apache.

the class AccumuloFileOutputFormatIT method handleWriteTests.

private void handleWriteTests(boolean content) throws Exception {
    File f = folder.newFile(testName.getMethodName());
    assertTrue(f.delete());
    MRTester.main(new String[] { content ? TEST_TABLE : EMPTY_TABLE, f.getAbsolutePath() });
    assertTrue(f.exists());
    File[] files = f.listFiles(new FileFilter() {

        @Override
        public boolean accept(File file) {
            return file.getName().startsWith("part-m-");
        }
    });
    assertNotNull(files);
    if (content) {
        assertEquals(1, files.length);
        assertTrue(files[0].exists());
        Configuration conf = CachedConfiguration.getInstance();
        DefaultConfiguration acuconf = DefaultConfiguration.getInstance();
        FileSKVIterator sample = RFileOperations.getInstance().newReaderBuilder().forFile(files[0].toString(), FileSystem.get(conf), conf).withTableConfiguration(acuconf).build().getSample(new SamplerConfigurationImpl(SAMPLER_CONFIG));
        assertNotNull(sample);
    } else {
        assertEquals(0, files.length);
    }
}

Example 9 with FileSKVIterator

use of org.apache.accumulo.core.file.FileSKVIterator in project accumulo by apache.

the class Compactor method compactLocalityGroup.

private void compactLocalityGroup(String lgName, Set<ByteSequence> columnFamilies, boolean inclusive, FileSKVWriter mfw, CompactionStats majCStats) throws IOException, CompactionCanceledException {
    ArrayList<FileSKVIterator> readers = new ArrayList<>(filesToCompact.size());
    Span span = Trace.start("compact");
    try {
        long entriesCompacted = 0;
        List<SortedKeyValueIterator<Key, Value>> iters = openMapDataFiles(lgName, readers);
        if (imm != null) {
            iters.add(imm.compactionIterator());
        }
        CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()), entriesRead);
        DeletingIterator delIter = new DeletingIterator(citr, propogateDeletes);
        ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
        // if(env.getIteratorScope() )
        TabletIteratorEnvironment iterEnv;
        if (env.getIteratorScope() == IteratorScope.majc)
            iterEnv = new TabletIteratorEnvironment(IteratorScope.majc, !propogateDeletes, acuTableConf);
        else if (env.getIteratorScope() == IteratorScope.minc)
            iterEnv = new TabletIteratorEnvironment(IteratorScope.minc, acuTableConf);
        else
            throw new IllegalArgumentException();
        SortedKeyValueIterator<Key, Value> itr = iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(env.getIteratorScope(), cfsi, extent, acuTableConf, iterators, iterEnv));
        itr.seek(extent.toDataRange(), columnFamilies, inclusive);
        if (!inclusive) {
            mfw.startDefaultLocalityGroup();
        } else {
            mfw.startNewLocalityGroup(lgName, columnFamilies);
        }
        Span write = Trace.start("write");
        try {
            while (itr.hasTop() && env.isCompactionEnabled()) {
                mfw.append(itr.getTopKey(), itr.getTopValue());
                itr.next();
                entriesCompacted++;
                if (entriesCompacted % 1024 == 0) {
                    // Periodically update stats, do not want to do this too often since its volatile
                    entriesWritten.addAndGet(1024);
                }
            }
            if (itr.hasTop() && !env.isCompactionEnabled()) {
                // cancel major compaction operation
                try {
                    try {
                        mfw.close();
                    } catch (IOException e) {
                        log.error("{}", e.getMessage(), e);
                    }
                    fs.deleteRecursively(outputFile.path());
                } catch (Exception e) {
                    log.warn("Failed to delete Canceled compaction output file {}", outputFile, e);
                }
                throw new CompactionCanceledException();
            }
        } finally {
            CompactionStats lgMajcStats = new CompactionStats(citr.getCount(), entriesCompacted);
            majCStats.add(lgMajcStats);
            write.stop();
        }
    } finally {
        // close sequence files opened
        for (FileSKVIterator reader : readers) {
            try {
                reader.close();
            } catch (Throwable e) {
                log.warn("Failed to close map file", e);
            }
        }
        span.stop();
    }
}

Example 10 with FileSKVIterator

use of org.apache.accumulo.core.file.FileSKVIterator in project accumulo by apache.

the class OfflineIterator method createIterator.

private SortedKeyValueIterator<Key, Value> createIterator(KeyExtent extent, List<String> absFiles) throws TableNotFoundException, AccumuloException, IOException {
    // TODO share code w/ tablet - ACCUMULO-1303
    // possible race condition here, if table is renamed
    String tableName = Tables.getTableName(conn.getInstance(), tableId);
    AccumuloConfiguration acuTableConf = new ConfigurationCopy(conn.tableOperations().getProperties(tableName));
    Configuration conf = CachedConfiguration.getInstance();
    for (SortedKeyValueIterator<Key, Value> reader : readers) {
        ((FileSKVIterator) reader).close();
    }
    readers.clear();
    SamplerConfiguration scannerSamplerConfig = options.getSamplerConfiguration();
    SamplerConfigurationImpl scannerSamplerConfigImpl = scannerSamplerConfig == null ? null : new SamplerConfigurationImpl(scannerSamplerConfig);
    SamplerConfigurationImpl samplerConfImpl = SamplerConfigurationImpl.newSamplerConfig(acuTableConf);
    if (scannerSamplerConfigImpl != null && ((samplerConfImpl != null && !scannerSamplerConfigImpl.equals(samplerConfImpl)) || samplerConfImpl == null)) {
        throw new SampleNotPresentException();
    }
    // TODO need to close files - ACCUMULO-1303
    for (String file : absFiles) {
        FileSystem fs = VolumeConfiguration.getVolume(file, conf, config).getFileSystem();
        FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder().forFile(file, fs, conf).withTableConfiguration(acuTableConf).build();
        if (scannerSamplerConfigImpl != null) {
            reader = reader.getSample(scannerSamplerConfigImpl);
            if (reader == null)
                throw new SampleNotPresentException();
        }
        readers.add(reader);
    }
    MultiIterator multiIter = new MultiIterator(readers, extent);
    OfflineIteratorEnvironment iterEnv = new OfflineIteratorEnvironment(authorizations, acuTableConf, false, samplerConfImpl == null ? null : samplerConfImpl.toSamplerConfiguration());
    byte[] defaultSecurityLabel;
    ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
    defaultSecurityLabel = cv.getExpression();
    SortedKeyValueIterator<Key, Value> visFilter = IteratorUtil.setupSystemScanIterators(multiIter, new HashSet<>(options.fetchedColumns), authorizations, defaultSecurityLabel);
    return iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.serverSideIteratorList, options.serverSideIteratorOptions, iterEnv, false));
}