Examples with FileRef org.apache.accumulo.server.fs.FileRef used on opensource projects

Example 11 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class CompactionInfo method toThrift.

public ActiveCompaction toThrift() {
    CompactionType type;
    if (compactor.hasIMM())
        if (compactor.getFilesToCompact().size() > 0)
            type = CompactionType.MERGE;
        else
            type = CompactionType.MINOR;
    else if (!compactor.willPropogateDeletes())
        type = CompactionType.FULL;
    else
        type = CompactionType.MAJOR;
    CompactionReason reason;
    if (compactor.hasIMM()) {
        switch(compactor.getMinCReason()) {
            case USER:
                reason = CompactionReason.USER;
                break;
            case CLOSE:
                reason = CompactionReason.CLOSE;
                break;
            case SYSTEM:
            default:
                reason = CompactionReason.SYSTEM;
                break;
        }
    } else {
        switch(compactor.getMajorCompactionReason()) {
            case USER:
                reason = CompactionReason.USER;
                break;
            case CHOP:
                reason = CompactionReason.CHOP;
                break;
            case IDLE:
                reason = CompactionReason.IDLE;
                break;
            case NORMAL:
            default:
                reason = CompactionReason.SYSTEM;
                break;
        }
    }
    List<IterInfo> iiList = new ArrayList<>();
    Map<String, Map<String, String>> iterOptions = new HashMap<>();
    for (IteratorSetting iterSetting : compactor.getIterators()) {
        iiList.add(new IterInfo(iterSetting.getPriority(), iterSetting.getIteratorClass(), iterSetting.getName()));
        iterOptions.put(iterSetting.getName(), iterSetting.getOptions());
    }
    List<String> filesToCompact = new ArrayList<>();
    for (FileRef ref : compactor.getFilesToCompact()) filesToCompact.add(ref.toString());
    return new ActiveCompaction(compactor.extent.toThrift(), System.currentTimeMillis() - compactor.getStartTime(), filesToCompact, compactor.getOutputFile(), type, reason, localityGroup, entriesRead, entriesWritten, iiList, iterOptions);
}

Example 12 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class ScanDataSource method createIterator.

private SortedKeyValueIterator<Key, Value> createIterator() throws IOException {
    Map<FileRef, DataFileValue> files;
    SamplerConfigurationImpl samplerConfig = options.getSamplerConfigurationImpl();
    synchronized (tablet) {
        if (memIters != null)
            throw new IllegalStateException("Tried to create new scan iterator w/o releasing memory");
        if (tablet.isClosed())
            throw new TabletClosedException();
        if (interruptFlag.get())
            throw new IterationInterruptedException(tablet.getExtent().toString() + " " + interruptFlag.hashCode());
        // only acquire the file manager when we know the tablet is open
        if (fileManager == null) {
            fileManager = tablet.getTabletResources().newScanFileManager();
            tablet.addActiveScans(this);
        }
        if (fileManager.getNumOpenFiles() != 0)
            throw new IllegalStateException("Tried to create new scan iterator w/o releasing files");
        // set this before trying to get iterators in case
        // getIterators() throws an exception
        expectedDeletionCount = tablet.getDataSourceDeletions();
        memIters = tablet.getTabletMemory().getIterators(samplerConfig);
        Pair<Long, Map<FileRef, DataFileValue>> reservation = tablet.getDatafileManager().reserveFilesForScan();
        fileReservationId = reservation.getFirst();
        files = reservation.getSecond();
    }
    Collection<InterruptibleIterator> mapfiles = fileManager.openFiles(files, options.isIsolated(), samplerConfig);
    for (SortedKeyValueIterator<Key, Value> skvi : Iterables.concat(mapfiles, memIters)) ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
    List<SortedKeyValueIterator<Key, Value>> iters = new ArrayList<>(mapfiles.size() + memIters.size());
    iters.addAll(mapfiles);
    iters.addAll(memIters);
    MultiIterator multiIter = new MultiIterator(iters, tablet.getExtent());
    TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(IteratorScope.scan, tablet.getTableConfiguration(), fileManager, files, options.getAuthorizations(), samplerConfig);
    statsIterator = new StatsIterator(multiIter, TabletServer.seekCount, tablet.getScannedCounter());
    SortedKeyValueIterator<Key, Value> visFilter = IteratorUtil.setupSystemScanIterators(statsIterator, options.getColumnSet(), options.getAuthorizations(), options.getDefaultLabels());
    if (!loadIters) {
        return visFilter;
    } else {
        List<IterInfo> iterInfos;
        Map<String, Map<String, String>> iterOpts;
        ParsedIteratorConfig pic = tablet.getTableConfiguration().getParsedIteratorConfig(IteratorScope.scan);
        if (options.getSsiList().size() == 0 && options.getSsio().size() == 0) {
            // No scan time iterator options were set, so can just use the pre-parsed table iterator options.
            iterInfos = pic.getIterInfo();
            iterOpts = pic.getOpts();
        } else {
            // Scan time iterator options were set, so need to merge those with pre-parsed table iterator options.
            iterOpts = new HashMap<>(pic.getOpts().size() + options.getSsio().size());
            iterInfos = new ArrayList<>(pic.getIterInfo().size() + options.getSsiList().size());
            IteratorUtil.mergeIteratorConfig(iterInfos, iterOpts, pic.getIterInfo(), pic.getOpts(), options.getSsiList(), options.getSsio());
        }
        String context;
        if (options.getClassLoaderContext() != null) {
            log.trace("Loading iterators for scan with scan context: {}", options.getClassLoaderContext());
            context = options.getClassLoaderContext();
        } else {
            context = pic.getContext();
            if (context != null) {
                log.trace("Loading iterators for scan with table context: {}", options.getClassLoaderContext());
            } else {
                log.trace("Loading iterators for scan");
            }
        }
        return iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(visFilter, iterInfos, iterOpts, iterEnv, true, context));
    }
}

Example 13 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class Tablet method split.

public TreeMap<KeyExtent, TabletData> split(byte[] sp) throws IOException {
    if (sp != null && extent.getEndRow() != null && extent.getEndRow().equals(new Text(sp))) {
        throw new IllegalArgumentException();
    }
    if (sp != null && sp.length > tableConfiguration.getAsBytes(Property.TABLE_MAX_END_ROW_SIZE)) {
        String msg = "Cannot split tablet " + extent + ", selected split point too long.  Length :  " + sp.length;
        log.warn(msg);
        throw new IOException(msg);
    }
    if (extent.isRootTablet()) {
        String msg = "Cannot split root tablet";
        log.warn(msg);
        throw new RuntimeException(msg);
    }
    try {
        initiateClose(true, false, false);
    } catch (IllegalStateException ise) {
        log.debug("File {} not splitting : {}", extent, ise.getMessage());
        return null;
    }
    // obtain this info outside of synch block since it will involve opening
    // the map files... it is ok if the set of map files changes, because
    // this info is used for optimization... it is ok if map files are missing
    // from the set... can still query and insert into the tablet while this
    // map file operation is happening
    Map<FileRef, FileUtil.FileInfo> firstAndLastRows = FileUtil.tryToGetFirstAndLastRows(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(), getDatafileManager().getFiles());
    synchronized (this) {
        // java needs tuples ...
        TreeMap<KeyExtent, TabletData> newTablets = new TreeMap<>();
        long t1 = System.currentTimeMillis();
        // choose a split point
        SplitRowSpec splitPoint;
        if (sp == null)
            splitPoint = findSplitRow(getDatafileManager().getFiles());
        else {
            Text tsp = new Text(sp);
            splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(getTabletServer().getFileSystem(), tabletDirectory, getTabletServer().getConfiguration(), extent.getPrevEndRow(), extent.getEndRow(), FileUtil.toPathStrings(getDatafileManager().getFiles()), tsp), tsp);
        }
        if (splitPoint == null || splitPoint.row == null) {
            log.info("had to abort split because splitRow was null");
            closeState = CloseState.OPEN;
            return null;
        }
        closeState = CloseState.CLOSING;
        completeClose(true, false);
        Text midRow = splitPoint.row;
        double splitRatio = splitPoint.splitRatio;
        KeyExtent low = new KeyExtent(extent.getTableId(), midRow, extent.getPrevEndRow());
        KeyExtent high = new KeyExtent(extent.getTableId(), extent.getEndRow(), midRow);
        String lowDirectory = createTabletDirectory(getTabletServer().getFileSystem(), extent.getTableId(), midRow);
        // write new tablet information to MetadataTable
        SortedMap<FileRef, DataFileValue> lowDatafileSizes = new TreeMap<>();
        SortedMap<FileRef, DataFileValue> highDatafileSizes = new TreeMap<>();
        List<FileRef> highDatafilesToRemove = new ArrayList<>();
        MetadataTableUtil.splitDatafiles(midRow, splitRatio, firstAndLastRows, getDatafileManager().getDatafileSizes(), lowDatafileSizes, highDatafileSizes, highDatafilesToRemove);
        log.debug("Files for low split {} {}", low, lowDatafileSizes.keySet());
        log.debug("Files for high split {} {}", high, highDatafileSizes.keySet());
        String time = tabletTime.getMetadataValue();
        MetadataTableUtil.splitTablet(high, extent.getPrevEndRow(), splitRatio, getTabletServer(), getTabletServer().getLock());
        MasterMetadataUtil.addNewTablet(getTabletServer(), low, lowDirectory, getTabletServer().getTabletSession(), lowDatafileSizes, getBulkIngestedFiles(), time, lastFlushID, lastCompactID, getTabletServer().getLock());
        MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, getTabletServer(), getTabletServer().getLock());
        log.debug("TABLET_HIST {} split {} {}", extent, low, high);
        newTablets.put(high, new TabletData(tabletDirectory, highDatafileSizes, time, lastFlushID, lastCompactID, lastLocation, getBulkIngestedFiles()));
        newTablets.put(low, new TabletData(lowDirectory, lowDatafileSizes, time, lastFlushID, lastCompactID, lastLocation, getBulkIngestedFiles()));
        long t2 = System.currentTimeMillis();
        log.debug(String.format("offline split time : %6.2f secs", (t2 - t1) / 1000.0));
        closeState = CloseState.COMPLETE;
        return newTablets;
    }
}

Example 14 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class DatafileManagerTest method testReserveMergingMinorCompactionFile_MaxFilesNotReached.

/*
   * Test max files not reached (table.file.max) when calling reserveMergingMinorCompactionFile
   */
@Test
public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException {
    EasyMock.replay(tablet, tableConf);
    SortedMap<FileRef, DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M");
    DatafileManager dfm = new DatafileManager(tablet, testFiles);
    FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
    EasyMock.verify(tablet, tableConf);
    assertEquals(null, mergeFile);
}

Example 15 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class DatafileManagerTest method testReserveMergingMinorCompactionFile_MaxExceeded.

/*
   * Test max file size (table.compaction.minor.merge.file.size.max) exceeded when calling reserveMergingMinorCompactionFile
   */
@Test
public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
    String maxMergeFileSize = "1000B";
    EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
    EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
    EasyMock.replay(tablet, tableConf);
    SortedMap<FileRef, DataFileValue> testFiles = createFileMap("largefile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
    DatafileManager dfm = new DatafileManager(tablet, testFiles);
    FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
    EasyMock.verify(tablet, tableConf);
    assertEquals(null, mergeFile);
}