Examples with DataFileValue org.apache.accumulo.core.metadata.schema.DataFileValue used on opensource projects

Example 6 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class VolumeUtil method updateTabletVolumes.

/**
 * This method does two things. First, it switches any volumes a tablet is using that are configured in instance.volumes.replacements. Second, if a tablet dir
 * is no longer configured for use it chooses a new tablet directory.
 */
public static TabletFiles updateTabletVolumes(AccumuloServerContext context, ZooLock zooLock, VolumeManager vm, KeyExtent extent, TabletFiles tabletFiles, boolean replicate) throws IOException {
    List<Pair<Path, Path>> replacements = ServerConstants.getVolumeReplacements();
    log.trace("Using volume replacements: {}", replacements);
    List<LogEntry> logsToRemove = new ArrayList<>();
    List<LogEntry> logsToAdd = new ArrayList<>();
    List<FileRef> filesToRemove = new ArrayList<>();
    SortedMap<FileRef, DataFileValue> filesToAdd = new TreeMap<>();
    TabletFiles ret = new TabletFiles();
    for (LogEntry logEntry : tabletFiles.logEntries) {
        LogEntry switchedLogEntry = switchVolumes(logEntry, replacements);
        if (switchedLogEntry != null) {
            logsToRemove.add(logEntry);
            logsToAdd.add(switchedLogEntry);
            ret.logEntries.add(switchedLogEntry);
            log.debug("Replacing volume {} : {} -> {}", extent, logEntry.filename, switchedLogEntry.filename);
        } else {
            ret.logEntries.add(logEntry);
        }
    }
    if (extent.isRootTablet()) {
        ret.datafiles = tabletFiles.datafiles;
    } else {
        for (Entry<FileRef, DataFileValue> entry : tabletFiles.datafiles.entrySet()) {
            String metaPath = entry.getKey().meta().toString();
            String switchedPath = switchVolume(metaPath, FileType.TABLE, replacements);
            if (switchedPath != null) {
                filesToRemove.add(entry.getKey());
                FileRef switchedRef = new FileRef(switchedPath, new Path(switchedPath));
                filesToAdd.put(switchedRef, entry.getValue());
                ret.datafiles.put(switchedRef, entry.getValue());
                log.debug("Replacing volume {} : {} -> {}", extent, metaPath, switchedPath);
            } else {
                ret.datafiles.put(entry.getKey(), entry.getValue());
            }
        }
    }
    String tabletDir = tabletFiles.dir;
    String switchedDir = switchVolume(tabletDir, FileType.TABLE, replacements);
    if (switchedDir != null) {
        log.debug("Replacing volume {} : {} -> {}", extent, tabletDir, switchedDir);
        tabletDir = switchedDir;
    }
    if (logsToRemove.size() + filesToRemove.size() > 0 || switchedDir != null) {
        MetadataTableUtil.updateTabletVolumes(extent, logsToRemove, logsToAdd, filesToRemove, filesToAdd, switchedDir, zooLock, context);
        if (replicate) {
            Status status = StatusUtil.fileClosed();
            log.debug("Tablet directory switched, need to record old log files {} {}", logsToRemove, ProtobufUtil.toString(status));
            // Before deleting these logs, we need to mark them for replication
            for (LogEntry logEntry : logsToRemove) {
                ReplicationTableUtil.updateFiles(context, extent, logEntry.filename, status);
            }
        }
    }
    ret.dir = decommisionedTabletDir(context, zooLock, vm, extent, tabletDir);
    if (extent.isRootTablet()) {
        SortedMap<FileRef, DataFileValue> copy = ret.datafiles;
        ret.datafiles = new TreeMap<>();
        for (Entry<FileRef, DataFileValue> entry : copy.entrySet()) {
            ret.datafiles.put(new FileRef(new Path(ret.dir, entry.getKey().path().getName()).toString()), entry.getValue());
        }
    }
    // method this should return the exact strings that are in the metadata table
    return ret;
}

Example 7 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class MetadataConstraints method check.

@Override
public List<Short> check(Environment env, Mutation mutation) {
    ArrayList<Short> violations = null;
    Collection<ColumnUpdate> colUpdates = mutation.getUpdates();
    // check the row, it should contains at least one ; or end with <
    boolean containsSemiC = false;
    byte[] row = mutation.getRow();
    // always allow rows that fall within reserved areas
    if (row.length > 0 && row[0] == '~')
        return null;
    if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~')
        return null;
    for (byte b : row) {
        if (b == ';') {
            containsSemiC = true;
        }
        if (b == ';' || b == '<')
            break;
        if (!validTableNameChars[0xff & b]) {
            violations = addIfNotPresent(violations, 4);
        }
    }
    if (!containsSemiC) {
        // see if last row char is <
        if (row.length == 0 || row[row.length - 1] != '<') {
            violations = addIfNotPresent(violations, 4);
        }
    } else {
        if (row.length == 0) {
            violations = addIfNotPresent(violations, 4);
        }
    }
    if (row.length > 0 && row[0] == '!') {
        if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';')) {
            violations = addIfNotPresent(violations, 4);
        }
    }
    // ensure row is not less than Constants.METADATA_TABLE_ID
    if (new Text(row).compareTo(new Text(MetadataTable.ID.getUtf8())) < 0) {
        violations = addViolation(violations, 5);
    }
    boolean checkedBulk = false;
    for (ColumnUpdate columnUpdate : colUpdates) {
        Text columnFamily = new Text(columnUpdate.getColumnFamily());
        if (columnUpdate.isDeleted()) {
            if (!isValidColumn(columnUpdate)) {
                violations = addViolation(violations, 2);
            }
            continue;
        }
        if (columnUpdate.getValue().length == 0 && !columnFamily.equals(ScanFileColumnFamily.NAME)) {
            violations = addViolation(violations, 6);
        }
        if (columnFamily.equals(DataFileColumnFamily.NAME)) {
            try {
                DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
                if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) {
                    violations = addViolation(violations, 1);
                }
            } catch (NumberFormatException | ArrayIndexOutOfBoundsException nfe) {
                violations = addViolation(violations, 1);
            }
        } else if (columnFamily.equals(ScanFileColumnFamily.NAME)) {
        } else if (columnFamily.equals(TabletsSection.BulkFileColumnFamily.NAME)) {
            if (!columnUpdate.isDeleted() && !checkedBulk) {
                // splits, which also write the time reference, are allowed to write this reference even when
                // the transaction is not running because the other half of the tablet is holding a reference
                // to the file.
                boolean isSplitMutation = false;
                // When a tablet is assigned, it re-writes the metadata. It should probably only update the location information,
                // but it writes everything. We allow it to re-write the bulk information if it is setting the location.
                // See ACCUMULO-1230.
                boolean isLocationMutation = false;
                HashSet<Text> dataFiles = new HashSet<>();
                HashSet<Text> loadedFiles = new HashSet<>();
                String tidString = new String(columnUpdate.getValue(), UTF_8);
                int otherTidCount = 0;
                for (ColumnUpdate update : mutation.getUpdates()) {
                    if (new ColumnFQ(update).equals(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)) {
                        isSplitMutation = true;
                    } else if (new Text(update.getColumnFamily()).equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
                        isLocationMutation = true;
                    } else if (new Text(update.getColumnFamily()).equals(DataFileColumnFamily.NAME)) {
                        dataFiles.add(new Text(update.getColumnQualifier()));
                    } else if (new Text(update.getColumnFamily()).equals(TabletsSection.BulkFileColumnFamily.NAME)) {
                        loadedFiles.add(new Text(update.getColumnQualifier()));
                        if (!new String(update.getValue(), UTF_8).equals(tidString)) {
                            otherTidCount++;
                        }
                    }
                }
                if (!isSplitMutation && !isLocationMutation) {
                    long tid = Long.parseLong(tidString);
                    try {
                        if (otherTidCount > 0 || !dataFiles.equals(loadedFiles) || !getArbitrator().transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid)) {
                            violations = addViolation(violations, 8);
                        }
                    } catch (Exception ex) {
                        violations = addViolation(violations, 8);
                    }
                }
                checkedBulk = true;
            }
        } else {
            if (!isValidColumn(columnUpdate)) {
                violations = addViolation(violations, 2);
            } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0 && (violations == null || !violations.contains((short) 4))) {
                KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null);
                Text per = KeyExtent.decodePrevEndRow(new Value(columnUpdate.getValue()));
                boolean prevEndRowLessThanEndRow = per == null || ke.getEndRow() == null || per.compareTo(ke.getEndRow()) < 0;
                if (!prevEndRowLessThanEndRow) {
                    violations = addViolation(violations, 3);
                }
            } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.ServerColumnFamily.LOCK_COLUMN)) {
                if (zooCache == null) {
                    zooCache = new ZooCache();
                }
                if (zooRoot == null) {
                    zooRoot = ZooUtil.getRoot(HdfsZooInstance.getInstance());
                }
                boolean lockHeld = false;
                String lockId = new String(columnUpdate.getValue(), UTF_8);
                try {
                    lockHeld = ZooLock.isLockHeld(zooCache, new ZooUtil.LockID(zooRoot, lockId));
                } catch (Exception e) {
                    log.debug("Failed to verify lock was held {} {}", lockId, e.getMessage());
                }
                if (!lockHeld) {
                    violations = addViolation(violations, 7);
                }
            }
        }
    }
    if (violations != null) {
        log.debug("violating metadata mutation : {}", new String(mutation.getRow(), UTF_8));
        for (ColumnUpdate update : mutation.getUpdates()) {
            log.debug(" update: {}:{} value {}", new String(update.getColumnFamily(), UTF_8), new String(update.getColumnQualifier(), UTF_8), (update.isDeleted() ? "[delete]" : new String(update.getValue(), UTF_8)));
        }
    }
    return violations;
}

Example 8 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class ScanDataSource method createIterator.

private SortedKeyValueIterator<Key, Value> createIterator() throws IOException {
    Map<FileRef, DataFileValue> files;
    SamplerConfigurationImpl samplerConfig = options.getSamplerConfigurationImpl();
    synchronized (tablet) {
        if (memIters != null)
            throw new IllegalStateException("Tried to create new scan iterator w/o releasing memory");
        if (tablet.isClosed())
            throw new TabletClosedException();
        if (interruptFlag.get())
            throw new IterationInterruptedException(tablet.getExtent().toString() + " " + interruptFlag.hashCode());
        // only acquire the file manager when we know the tablet is open
        if (fileManager == null) {
            fileManager = tablet.getTabletResources().newScanFileManager();
            tablet.addActiveScans(this);
        }
        if (fileManager.getNumOpenFiles() != 0)
            throw new IllegalStateException("Tried to create new scan iterator w/o releasing files");
        // set this before trying to get iterators in case
        // getIterators() throws an exception
        expectedDeletionCount = tablet.getDataSourceDeletions();
        memIters = tablet.getTabletMemory().getIterators(samplerConfig);
        Pair<Long, Map<FileRef, DataFileValue>> reservation = tablet.getDatafileManager().reserveFilesForScan();
        fileReservationId = reservation.getFirst();
        files = reservation.getSecond();
    }
    Collection<InterruptibleIterator> mapfiles = fileManager.openFiles(files, options.isIsolated(), samplerConfig);
    for (SortedKeyValueIterator<Key, Value> skvi : Iterables.concat(mapfiles, memIters)) ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
    List<SortedKeyValueIterator<Key, Value>> iters = new ArrayList<>(mapfiles.size() + memIters.size());
    iters.addAll(mapfiles);
    iters.addAll(memIters);
    MultiIterator multiIter = new MultiIterator(iters, tablet.getExtent());
    TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(IteratorScope.scan, tablet.getTableConfiguration(), fileManager, files, options.getAuthorizations(), samplerConfig);
    statsIterator = new StatsIterator(multiIter, TabletServer.seekCount, tablet.getScannedCounter());
    SortedKeyValueIterator<Key, Value> visFilter = IteratorUtil.setupSystemScanIterators(statsIterator, options.getColumnSet(), options.getAuthorizations(), options.getDefaultLabels());
    if (!loadIters) {
        return visFilter;
    } else {
        List<IterInfo> iterInfos;
        Map<String, Map<String, String>> iterOpts;
        ParsedIteratorConfig pic = tablet.getTableConfiguration().getParsedIteratorConfig(IteratorScope.scan);
        if (options.getSsiList().size() == 0 && options.getSsio().size() == 0) {
            // No scan time iterator options were set, so can just use the pre-parsed table iterator options.
            iterInfos = pic.getIterInfo();
            iterOpts = pic.getOpts();
        } else {
            // Scan time iterator options were set, so need to merge those with pre-parsed table iterator options.
            iterOpts = new HashMap<>(pic.getOpts().size() + options.getSsio().size());
            iterInfos = new ArrayList<>(pic.getIterInfo().size() + options.getSsiList().size());
            IteratorUtil.mergeIteratorConfig(iterInfos, iterOpts, pic.getIterInfo(), pic.getOpts(), options.getSsiList(), options.getSsio());
        }
        String context;
        if (options.getClassLoaderContext() != null) {
            log.trace("Loading iterators for scan with scan context: {}", options.getClassLoaderContext());
            context = options.getClassLoaderContext();
        } else {
            context = pic.getContext();
            if (context != null) {
                log.trace("Loading iterators for scan with table context: {}", options.getClassLoaderContext());
            } else {
                log.trace("Loading iterators for scan");
            }
        }
        return iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(visFilter, iterInfos, iterOpts, iterEnv, true, context));
    }
}

Example 9 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class Tablet method split.

public TreeMap<KeyExtent, TabletData> split(byte[] sp) throws IOException {
    if (sp != null && extent.getEndRow() != null && extent.getEndRow().equals(new Text(sp))) {
        throw new IllegalArgumentException();
    }
    if (sp != null && sp.length > tableConfiguration.getAsBytes(Property.TABLE_MAX_END_ROW_SIZE)) {
        String msg = "Cannot split tablet " + extent + ", selected split point too long.  Length :  " + sp.length;
        log.warn(msg);
        throw new IOException(msg);
    }
    if (extent.isRootTablet()) {
        String msg = "Cannot split root tablet";
        log.warn(msg);
        throw new RuntimeException(msg);
    }
    try {
        initiateClose(true, false, false);
    } catch (IllegalStateException ise) {
        log.debug("File {} not splitting : {}", extent, ise.getMessage());
        return null;
    }
    // obtain this info outside of synch block since it will involve opening
    // the map files... it is ok if the set of map files changes, because
    // this info is used for optimization... it is ok if map files are missing
    // from the set... can still query and insert into the tablet while this
    // map file operation is happening
    Map<FileRef, FileUtil.FileInfo> firstAndLastRows = FileUtil.tryToGetFirstAndLastRows(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(), getDatafileManager().getFiles());
    synchronized (this) {
        // java needs tuples ...
        TreeMap<KeyExtent, TabletData> newTablets = new TreeMap<>();
        long t1 = System.currentTimeMillis();
        // choose a split point
        SplitRowSpec splitPoint;
        if (sp == null)
            splitPoint = findSplitRow(getDatafileManager().getFiles());
        else {
            Text tsp = new Text(sp);
            splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(getTabletServer().getFileSystem(), tabletDirectory, getTabletServer().getConfiguration(), extent.getPrevEndRow(), extent.getEndRow(), FileUtil.toPathStrings(getDatafileManager().getFiles()), tsp), tsp);
        }
        if (splitPoint == null || splitPoint.row == null) {
            log.info("had to abort split because splitRow was null");
            closeState = CloseState.OPEN;
            return null;
        }
        closeState = CloseState.CLOSING;
        completeClose(true, false);
        Text midRow = splitPoint.row;
        double splitRatio = splitPoint.splitRatio;
        KeyExtent low = new KeyExtent(extent.getTableId(), midRow, extent.getPrevEndRow());
        KeyExtent high = new KeyExtent(extent.getTableId(), extent.getEndRow(), midRow);
        String lowDirectory = createTabletDirectory(getTabletServer().getFileSystem(), extent.getTableId(), midRow);
        // write new tablet information to MetadataTable
        SortedMap<FileRef, DataFileValue> lowDatafileSizes = new TreeMap<>();
        SortedMap<FileRef, DataFileValue> highDatafileSizes = new TreeMap<>();
        List<FileRef> highDatafilesToRemove = new ArrayList<>();
        MetadataTableUtil.splitDatafiles(midRow, splitRatio, firstAndLastRows, getDatafileManager().getDatafileSizes(), lowDatafileSizes, highDatafileSizes, highDatafilesToRemove);
        log.debug("Files for low split {} {}", low, lowDatafileSizes.keySet());
        log.debug("Files for high split {} {}", high, highDatafileSizes.keySet());
        String time = tabletTime.getMetadataValue();
        MetadataTableUtil.splitTablet(high, extent.getPrevEndRow(), splitRatio, getTabletServer(), getTabletServer().getLock());
        MasterMetadataUtil.addNewTablet(getTabletServer(), low, lowDirectory, getTabletServer().getTabletSession(), lowDatafileSizes, getBulkIngestedFiles(), time, lastFlushID, lastCompactID, getTabletServer().getLock());
        MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, getTabletServer(), getTabletServer().getLock());
        log.debug("TABLET_HIST {} split {} {}", extent, low, high);
        newTablets.put(high, new TabletData(tabletDirectory, highDatafileSizes, time, lastFlushID, lastCompactID, lastLocation, getBulkIngestedFiles()));
        newTablets.put(low, new TabletData(lowDirectory, lowDatafileSizes, time, lastFlushID, lastCompactID, lastLocation, getBulkIngestedFiles()));
        long t2 = System.currentTimeMillis();
        log.debug(String.format("offline split time : %6.2f secs", (t2 - t1) / 1000.0));
        closeState = CloseState.COMPLETE;
        return newTablets;
    }
}

Example 10 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class DatafileManagerTest method testReserveMergingMinorCompactionFile_MaxFilesNotReached.

/*
   * Test max files not reached (table.file.max) when calling reserveMergingMinorCompactionFile
   */
@Test
public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException {
    EasyMock.replay(tablet, tableConf);
    SortedMap<FileRef, DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M");
    DatafileManager dfm = new DatafileManager(tablet, testFiles);
    FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
    EasyMock.verify(tablet, tableConf);
    assertEquals(null, mergeFile);
}