Examples with MetadataTime org.apache.accumulo.core.metadata.schema.MetadataTime used on opensource projects

Example 6 with MetadataTime

use of org.apache.accumulo.core.metadata.schema.MetadataTime in project accumulo by apache.

the class TabletTimeTest method testMaxMetadataTime_Null3.

@Test
public void testMaxMetadataTime_Null3() {
    MetadataTime nullTime = null;
    assertNull(TabletTime.maxMetadataTime(nullTime, nullTime));
}

Example 7 with MetadataTime

use of org.apache.accumulo.core.metadata.schema.MetadataTime in project accumulo by apache.

the class MetadataTableUtil method addTablet.

public static void addTablet(KeyExtent extent, String path, ServerContext context, TimeType timeType, ServiceLock zooLock) {
    TabletMutator tablet = context.getAmple().mutateTablet(extent);
    tablet.putPrevEndRow(extent.prevEndRow());
    tablet.putDirName(path);
    tablet.putTime(new MetadataTime(0, timeType));
    tablet.putZooLock(zooLock);
    tablet.mutate();
}

Example 8 with MetadataTime

use of org.apache.accumulo.core.metadata.schema.MetadataTime in project accumulo by apache.

the class Tablet method split.

public TreeMap<KeyExtent, TabletData> split(byte[] sp) throws IOException {
    if (sp != null && extent.endRow() != null && extent.endRow().equals(new Text(sp))) {
        throw new IllegalArgumentException("Attempting to split on EndRow " + extent.endRow() + " for " + extent);
    }
    if (sp != null && sp.length > tableConfiguration.getAsBytes(Property.TABLE_MAX_END_ROW_SIZE)) {
        String msg = "Cannot split tablet " + extent + ", selected split point too long.  Length :  " + sp.length;
        log.warn(msg);
        throw new IOException(msg);
    }
    if (extent.isRootTablet()) {
        String msg = "Cannot split root tablet";
        log.warn(msg);
        throw new RuntimeException(msg);
    }
    try {
        initiateClose(true);
    } catch (IllegalStateException ise) {
        log.debug("File {} not splitting : {}", extent, ise.getMessage());
        return null;
    }
    // obtain this info outside of synch block since it will involve opening
    // the map files... it is ok if the set of map files changes, because
    // this info is used for optimization... it is ok if map files are missing
    // from the set... can still query and insert into the tablet while this
    // map file operation is happening
    Map<TabletFile, FileUtil.FileInfo> firstAndLastRows = FileUtil.tryToGetFirstAndLastRows(context, getDatafileManager().getFiles());
    synchronized (this) {
        // java needs tuples ...
        TreeMap<KeyExtent, TabletData> newTablets = new TreeMap<>();
        long t1 = System.currentTimeMillis();
        // choose a split point
        SplitRowSpec splitPoint;
        if (sp == null) {
            splitPoint = findSplitRow(getDatafileManager().getFiles());
        } else {
            Text tsp = new Text(sp);
            splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(context, chooseTabletDir(), extent.prevEndRow(), extent.endRow(), getDatafileManager().getFiles(), tsp), tsp);
        }
        if (splitPoint == null || splitPoint.row == null) {
            log.info("had to abort split because splitRow was null");
            closeState = CloseState.OPEN;
            return null;
        }
        closeState = CloseState.CLOSING;
        completeClose(true, false);
        Text midRow = splitPoint.row;
        double splitRatio = splitPoint.splitRatio;
        KeyExtent low = new KeyExtent(extent.tableId(), midRow, extent.prevEndRow());
        KeyExtent high = new KeyExtent(extent.tableId(), extent.endRow(), midRow);
        String lowDirectoryName = createTabletDirectoryName(context, midRow);
        // write new tablet information to MetadataTable
        SortedMap<StoredTabletFile, DataFileValue> lowDatafileSizes = new TreeMap<>();
        SortedMap<StoredTabletFile, DataFileValue> highDatafileSizes = new TreeMap<>();
        List<StoredTabletFile> highDatafilesToRemove = new ArrayList<>();
        MetadataTableUtil.splitDatafiles(midRow, splitRatio, firstAndLastRows, getDatafileManager().getDatafileSizes(), lowDatafileSizes, highDatafileSizes, highDatafilesToRemove);
        log.debug("Files for low split {} {}", low, lowDatafileSizes.keySet());
        log.debug("Files for high split {} {}", high, highDatafileSizes.keySet());
        MetadataTime time = tabletTime.getMetadataTime();
        HashSet<ExternalCompactionId> ecids = new HashSet<>();
        compactable.getExternalCompactionIds(ecids::add);
        MetadataTableUtil.splitTablet(high, extent.prevEndRow(), splitRatio, getTabletServer().getContext(), getTabletServer().getLock(), ecids);
        ManagerMetadataUtil.addNewTablet(getTabletServer().getContext(), low, lowDirectoryName, getTabletServer().getTabletSession(), lowDatafileSizes, bulkImported, time, lastFlushID, lastCompactID, getTabletServer().getLock());
        MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, getTabletServer().getContext(), getTabletServer().getLock());
        TabletLogger.split(extent, low, high, getTabletServer().getTabletSession());
        newTablets.put(high, new TabletData(dirName, highDatafileSizes, time, lastFlushID, lastCompactID, lastLocation, bulkImported));
        newTablets.put(low, new TabletData(lowDirectoryName, lowDatafileSizes, time, lastFlushID, lastCompactID, lastLocation, bulkImported));
        long t2 = System.currentTimeMillis();
        log.debug(String.format("offline split time : %6.2f secs", (t2 - t1) / 1000.0));
        closeState = CloseState.COMPLETE;
        return newTablets;
    }
}

Example 9 with MetadataTime

use of org.apache.accumulo.core.metadata.schema.MetadataTime in project accumulo by apache.

the class Upgrader9to10 method computeRootTabletTime.

MetadataTime computeRootTabletTime(ServerContext context, Collection<String> goodPaths) {
    try {
        context.setupCrypto();
        long rtime = Long.MIN_VALUE;
        for (String good : goodPaths) {
            Path path = new Path(good);
            FileSystem ns = context.getVolumeManager().getFileSystemByPath(path);
            long maxTime = -1;
            try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder().forFile(path.toString(), ns, ns.getConf(), context.getCryptoService()).withTableConfiguration(context.getTableConfiguration(RootTable.ID)).seekToBeginning().build()) {
                while (reader.hasTop()) {
                    maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
                    reader.next();
                }
            }
            if (maxTime > rtime) {
                rtime = maxTime;
            }
        }
        if (rtime < 0) {
            throw new IllegalStateException("Unexpected root tablet logical time " + rtime);
        }
        return new MetadataTime(rtime, TimeType.LOGICAL);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}

Example 10 with MetadataTime

use of org.apache.accumulo.core.metadata.schema.MetadataTime in project accumulo by apache.

the class SplitRecoveryIT method splitPartiallyAndRecover.

private void splitPartiallyAndRecover(ServerContext context, KeyExtent extent, KeyExtent high, KeyExtent low, double splitRatio, SortedMap<StoredTabletFile, DataFileValue> mapFiles, Text midRow, String location, int steps, ServiceLock zl) throws Exception {
    SortedMap<StoredTabletFile, DataFileValue> lowDatafileSizes = new TreeMap<>();
    SortedMap<StoredTabletFile, DataFileValue> highDatafileSizes = new TreeMap<>();
    List<StoredTabletFile> highDatafilesToRemove = new ArrayList<>();
    MetadataTableUtil.splitDatafiles(midRow, splitRatio, new HashMap<>(), mapFiles, lowDatafileSizes, highDatafileSizes, highDatafilesToRemove);
    MetadataTableUtil.splitTablet(high, extent.prevEndRow(), splitRatio, context, zl, Set.of());
    TServerInstance instance = new TServerInstance(location, zl.getSessionId());
    Assignment assignment = new Assignment(high, instance);
    TabletMutator tabletMutator = context.getAmple().mutateTablet(extent);
    tabletMutator.putLocation(assignment.server, LocationType.FUTURE);
    tabletMutator.mutate();
    if (steps >= 1) {
        Map<Long, List<TabletFile>> bulkFiles = getBulkFilesLoaded(context, high);
        ManagerMetadataUtil.addNewTablet(context, low, "lowDir", instance, lowDatafileSizes, bulkFiles, new MetadataTime(0, TimeType.LOGICAL), -1L, -1L, zl);
    }
    if (steps >= 2) {
        MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, context, zl);
    }
    TabletMetadata meta = context.getAmple().readTablet(high);
    KeyExtent fixedExtent = ManagerMetadataUtil.fixSplit(context, meta, zl);
    if (steps < 2)
        assertEquals(splitRatio, meta.getSplitRatio(), 0.0);
    if (steps >= 1) {
        assertEquals(high, fixedExtent);
        ensureTabletHasNoUnexpectedMetadataEntries(context, low, lowDatafileSizes);
        ensureTabletHasNoUnexpectedMetadataEntries(context, high, highDatafileSizes);
        Map<Long, ? extends Collection<TabletFile>> lowBulkFiles = getBulkFilesLoaded(context, low);
        Map<Long, ? extends Collection<TabletFile>> highBulkFiles = getBulkFilesLoaded(context, high);
        if (!lowBulkFiles.equals(highBulkFiles)) {
            throw new Exception(" " + lowBulkFiles + " != " + highBulkFiles + " " + low + " " + high);
        }
        if (lowBulkFiles.isEmpty()) {
            throw new Exception(" no bulk files " + low);
        }
    } else {
        assertEquals(extent, fixedExtent);
        ensureTabletHasNoUnexpectedMetadataEntries(context, extent, mapFiles);
    }
}