Examples with MapFileInfo org.apache.accumulo.core.dataImpl.thrift.MapFileInfo used on opensource projects

Example 1 with MapFileInfo

use of org.apache.accumulo.core.dataImpl.thrift.MapFileInfo in project accumulo by apache.

the class ThriftClientHandler method bulkImport.

@Override
public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, final long tid, final Map<TKeyExtent, Map<String, MapFileInfo>> files, final boolean setTime) throws ThriftSecurityException {
    if (!security.canPerformSystemActions(credentials)) {
        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
    }
    try {
        return transactionWatcher.run(Constants.BULK_ARBITRATOR_TYPE, tid, () -> {
            List<TKeyExtent> failures = new ArrayList<>();
            for (Entry<TKeyExtent, Map<String, MapFileInfo>> entry : files.entrySet()) {
                TKeyExtent tke = entry.getKey();
                Map<String, MapFileInfo> fileMap = entry.getValue();
                Map<TabletFile, MapFileInfo> fileRefMap = new HashMap<>();
                for (Entry<String, MapFileInfo> mapping : fileMap.entrySet()) {
                    Path path = new Path(mapping.getKey());
                    FileSystem ns = context.getVolumeManager().getFileSystemByPath(path);
                    path = ns.makeQualified(path);
                    fileRefMap.put(new TabletFile(path), mapping.getValue());
                }
                Tablet importTablet = server.getOnlineTablet(KeyExtent.fromThrift(tke));
                if (importTablet == null) {
                    failures.add(tke);
                } else {
                    try {
                        importTablet.importMapFiles(tid, fileRefMap, setTime);
                    } catch (IOException ioe) {
                        log.info("files {} not imported to {}: {}", fileMap.keySet(), KeyExtent.fromThrift(tke), ioe.getMessage());
                        failures.add(tke);
                    }
                }
            }
            return failures;
        });
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with MapFileInfo

use of org.apache.accumulo.core.dataImpl.thrift.MapFileInfo in project accumulo by apache.

the class ThriftClientHandler method loadFiles.

@Override
public void loadFiles(TInfo tinfo, TCredentials credentials, long tid, String dir, Map<TKeyExtent, Map<String, MapFileInfo>> tabletImports, boolean setTime) throws ThriftSecurityException {
    if (!security.canPerformSystemActions(credentials)) {
        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
    }
    transactionWatcher.runQuietly(Constants.BULK_ARBITRATOR_TYPE, tid, () -> {
        tabletImports.forEach((tke, fileMap) -> {
            Map<TabletFile, MapFileInfo> newFileMap = new HashMap<>();
            for (Entry<String, MapFileInfo> mapping : fileMap.entrySet()) {
                Path path = new Path(dir, mapping.getKey());
                FileSystem ns = context.getVolumeManager().getFileSystemByPath(path);
                path = ns.makeQualified(path);
                newFileMap.put(new TabletFile(path), mapping.getValue());
            }
            var files = newFileMap.keySet().stream().map(TabletFile::getPathStr).collect(toList());
            server.updateBulkImportState(files, BulkImportState.INITIAL);
            Tablet importTablet = server.getOnlineTablet(KeyExtent.fromThrift(tke));
            if (importTablet != null) {
                try {
                    server.updateBulkImportState(files, BulkImportState.PROCESSING);
                    importTablet.importMapFiles(tid, newFileMap, setTime);
                } catch (IOException ioe) {
                    log.debug("files {} not imported to {}: {}", fileMap.keySet(), KeyExtent.fromThrift(tke), ioe.getMessage());
                } finally {
                    server.removeBulkImportState(files);
                }
            }
        });
    });
}

Example 3 with MapFileInfo

use of org.apache.accumulo.core.dataImpl.thrift.MapFileInfo in project accumulo by apache.

the class Tablet method importMapFiles.

public void importMapFiles(long tid, Map<TabletFile, MapFileInfo> fileMap, boolean setTime) throws IOException {
    Map<TabletFile, DataFileValue> entries = new HashMap<>(fileMap.size());
    List<String> files = new ArrayList<>();
    for (Entry<TabletFile, MapFileInfo> entry : fileMap.entrySet()) {
        entries.put(entry.getKey(), new DataFileValue(entry.getValue().estimatedSize, 0L));
        files.add(entry.getKey().getPathStr());
    }
    // Clients timeout and will think that this operation failed.
    // Don't do it if we spent too long waiting for the lock
    long now = System.currentTimeMillis();
    synchronized (this) {
        if (isClosed()) {
            throw new IOException("tablet " + extent + " is closed");
        }
        // TODO check seems unneeded now - ACCUMULO-1291
        long lockWait = System.currentTimeMillis() - now;
        if (lockWait > getTabletServer().getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT)) {
            throw new IOException("Timeout waiting " + (lockWait / 1000.) + " seconds to get tablet lock for " + extent);
        }
        List<TabletFile> alreadyImported = bulkImported.get(tid);
        if (alreadyImported != null) {
            for (TabletFile entry : alreadyImported) {
                if (fileMap.remove(entry) != null) {
                    log.trace("Ignoring import of bulk file already imported: {}", entry);
                }
            }
        }
        fileMap.keySet().removeIf(file -> {
            if (bulkImporting.contains(file)) {
                log.info("Ignoring import of bulk file currently importing: " + file);
                return true;
            }
            return false;
        });
        if (fileMap.isEmpty()) {
            return;
        }
        incrementWritesInProgress();
        // prevent other threads from processing this file while its added to the metadata table.
        bulkImporting.addAll(fileMap.keySet());
    }
    try {
        tabletServer.updateBulkImportState(files, BulkImportState.LOADING);
        var storedTabletFile = getDatafileManager().importMapFiles(tid, entries, setTime);
        lastMapFileImportTime = System.currentTimeMillis();
        if (needsSplit()) {
            getTabletServer().executeSplit(this);
        } else {
            compactable.filesAdded(false, storedTabletFile);
        }
    } finally {
        synchronized (this) {
            decrementWritesInProgress();
            if (!bulkImporting.removeAll(fileMap.keySet())) {
                throw new AssertionError("Likely bug in code, always expect to remove something.  Please open an Accumulo issue.");
            }
            try {
                bulkImported.computeIfAbsent(tid, k -> new ArrayList<>()).addAll(fileMap.keySet());
            } catch (Exception ex) {
                log.info(ex.toString(), ex);
            }
            tabletServer.removeBulkImportState(files);
        }
    }
}

Example 4 with MapFileInfo

use of org.apache.accumulo.core.dataImpl.thrift.MapFileInfo in project accumulo by apache.

the class BulkFailureIT method oldLoad.

private static void oldLoad(long txid, ClientContext context, KeyExtent extent, Path path, long size, boolean expectFailure) throws Exception {
    TabletClientService.Iface client = getClient(context, extent);
    try {
        Map<String, MapFileInfo> val = Map.of(path.toString(), new MapFileInfo(size));
        Map<KeyExtent, Map<String, MapFileInfo>> files = Map.of(extent, val);
        client.bulkImport(TraceUtil.traceInfo(), context.rpcCreds(), txid, files.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toThrift(), Entry::getValue)), false);
        if (expectFailure) {
            fail("Expected RPC to fail");
        }
    } catch (TApplicationException tae) {
        if (!expectFailure) {
            throw tae;
        }
    } finally {
        ThriftUtil.returnClient((TServiceClient) client, context);
    }
}

Example 5 with MapFileInfo

use of org.apache.accumulo.core.dataImpl.thrift.MapFileInfo in project accumulo by apache.

the class BulkFailureIT method newLoad.

private static void newLoad(long txid, ClientContext context, KeyExtent extent, Path path, long size, boolean expectFailure) throws Exception {
    TabletClientService.Iface client = getClient(context, extent);
    try {
        Map<String, MapFileInfo> val = Map.of(path.getName(), new MapFileInfo(size));
        Map<KeyExtent, Map<String, MapFileInfo>> files = Map.of(extent, val);
        client.loadFiles(TraceUtil.traceInfo(), context.rpcCreds(), txid, path.getParent().toString(), files.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toThrift(), Entry::getValue)), false);
        if (!expectFailure) {
            while (!getLoaded(context, extent).contains(path)) {
                Thread.sleep(100);
            }
        }
    } finally {
        ThriftUtil.returnClient((TServiceClient) client, context);
    }
}