Examples with ObsoleteVersionException voldemort.versioning.ObsoleteVersionException used on opensource projects

Example 31 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class BdbBuildPerformanceTest method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    if (args.length != 3)
        Utils.croak("USAGE: java " + BdbBuildPerformanceTest.class.getName() + "serverPropsFile storeName jsonSequenceDataFile");
    String serverPropsFile = args[0];
    String storeName = args[1];
    String jsonDataFile = args[2];
    final Store<ByteArray, byte[], byte[]> store = new BdbStorageConfiguration(new VoldemortConfig(new Props(new File(serverPropsFile)))).getStore(TestUtils.makeStoreDefinition(storeName), TestUtils.makeSingleNodeRoutingStrategy());
    final AtomicInteger obsoletes = new AtomicInteger(0);
    Path jsonFilePath = new Path(jsonDataFile);
    FileStatus jsonFileStatus = jsonFilePath.getFileSystem(new Configuration()).listStatus(jsonFilePath)[0];
    final SequenceFileRecordReader<BytesWritable, BytesWritable> reader = new SequenceFileRecordReader<BytesWritable, BytesWritable>(new Configuration(), new FileSplit(jsonFilePath, 0, jsonFileStatus.getLen(), (String[]) null));
    PerformanceTest readWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int index) throws Exception {
            try {
                BytesWritable key = new BytesWritable();
                BytesWritable value = new BytesWritable();
                reader.next(key, value);
                store.put(new ByteArray(ByteUtils.copy(key.get(), 0, key.getSize())), Versioned.value(ByteUtils.copy(value.get(), 0, value.getSize())), null);
            } catch (ObsoleteVersionException e) {
                obsoletes.incrementAndGet();
            }
        }
    };
    readWriteTest.run(30 * 1000 * 1000, 1);
    System.out.println("Bdb write throuhput with one thread:");
    readWriteTest.printStats();
}

Example 32 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class MysqlBuildPerformanceTest method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    if (args.length != 3)
        Utils.croak("USAGE: java " + MysqlBuildPerformanceTest.class.getName() + "serverPropsFile storeName jsonSequenceDataFile");
    String serverPropsFile = args[0];
    String storeName = args[1];
    String jsonDataFile = args[2];
    final Store<ByteArray, byte[], byte[]> store = new MysqlStorageConfiguration(new VoldemortConfig(new Props(new File(serverPropsFile)))).getStore(TestUtils.makeStoreDefinition(storeName), TestUtils.makeSingleNodeRoutingStrategy());
    final AtomicInteger obsoletes = new AtomicInteger(0);
    Path jsonFilePath = new Path(jsonDataFile);
    FileStatus jsonFileStatus = jsonFilePath.getFileSystem(new Configuration()).listStatus(jsonFilePath)[0];
    final SequenceFileRecordReader<BytesWritable, BytesWritable> reader = new SequenceFileRecordReader<BytesWritable, BytesWritable>(new Configuration(), new FileSplit(jsonFilePath, 0, jsonFileStatus.getLen(), (String[]) null));
    PerformanceTest readWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int index) throws Exception {
            try {
                BytesWritable key = new BytesWritable();
                BytesWritable value = new BytesWritable();
                reader.next(key, value);
                store.put(new ByteArray(ByteUtils.copy(key.get(), 0, key.getSize())), Versioned.value(ByteUtils.copy(value.get(), 0, value.getSize())), null);
            } catch (ObsoleteVersionException e) {
                obsoletes.incrementAndGet();
            }
        }
    };
    readWriteTest.run(1000, 1);
    System.out.println("MySQl write throuhput with one thread:");
    readWriteTest.printStats();
}

Example 33 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class ZonedRebalanceNonContiguousZonesTest method populateData.

private void populateData(Cluster cluster, StoreDefinition storeDef) throws Exception {
    // Create SocketStores for each Node first
    Map<Integer, Store<ByteArray, byte[], byte[]>> storeMap = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    for (Node node : cluster.getNodes()) {
        storeMap.put(node.getId(), getSocketStore(storeDef.getName(), node.getHost(), node.getSocketPort()));
    }
    BaseStoreRoutingPlan storeInstance = new BaseStoreRoutingPlan(cluster, storeDef);
    for (Entry<String, String> entry : testEntries.entrySet()) {
        ByteArray keyBytes = new ByteArray(ByteUtils.getBytes(entry.getKey(), "UTF-8"));
        List<Integer> preferenceNodes = storeInstance.getReplicationNodeList(keyBytes.get());
        // Go over every node
        for (int nodeId : preferenceNodes) {
            try {
                storeMap.get(nodeId).put(keyBytes, new Versioned<byte[]>(ByteUtils.getBytes(entry.getValue(), "UTF-8")), null);
            } catch (ObsoleteVersionException e) {
                logger.info("Why are we seeing this at all here ?? ");
                e.printStackTrace();
            }
        }
    }
    // close all socket stores
    for (Store<ByteArray, byte[], byte[]> store : storeMap.values()) {
        store.close();
    }
}

Example 34 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleFetchAndUpdate.

public VAdminProto.AsyncOperationStatusResponse handleFetchAndUpdate(VAdminProto.InitiateFetchAndUpdateRequest request) {
    final int nodeId = request.getNodeId();
    final List<Integer> partitionIds = request.getPartitionIdsList();
    final VoldemortFilter filter = request.hasFilter() ? getFilterFromRequest(request.getFilter(), voldemortConfig, networkClassLoader) : new DefaultVoldemortFilter();
    final String storeName = request.getStore();
    final Cluster initialCluster = request.hasInitialCluster() ? new ClusterMapper().readCluster(new StringReader(request.getInitialCluster())) : null;
    int requestId = asyncService.getUniqueRequestId();
    VAdminProto.AsyncOperationStatusResponse.Builder response = VAdminProto.AsyncOperationStatusResponse.newBuilder().setRequestId(requestId).setComplete(false).setDescription("Fetch and update").setStatus("Started");
    final StoreDefinition storeDef = metadataStore.getStoreDef(storeName);
    final boolean isReadOnlyStore = storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0;
    final StreamingStats streamingStats = voldemortConfig.isJmxEnabled() ? storeRepository.getStreamingStats(storeName) : null;
    try {
        asyncService.submitOperation(requestId, new AsyncOperation(requestId, "Fetch and Update") {

            private final AtomicBoolean running = new AtomicBoolean(true);

            @Override
            public void stop() {
                running.set(false);
                logger.info("Stopping fetch and update for store " + storeName + " from node " + nodeId + "( " + partitionIds + " )");
            }

            @Override
            public void operate() {
                AdminClient adminClient = AdminClient.createTempAdminClient(voldemortConfig, metadataStore.getCluster(), voldemortConfig.getClientMaxConnectionsPerNode());
                try {
                    StorageEngine<ByteArray, byte[], byte[]> storageEngine = getStorageEngine(storeRepository, storeName);
                    EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxWriteBytesPerSec());
                    if (isReadOnlyStore) {
                        ReadOnlyStorageEngine readOnlyStorageEngine = ((ReadOnlyStorageEngine) storageEngine);
                        String destinationDir = readOnlyStorageEngine.getCurrentDirPath();
                        logger.info("Fetching files for RO store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        updateStatus("Fetching files for RO store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        adminClient.readonlyOps.fetchPartitionFiles(nodeId, storeName, partitionIds, destinationDir, readOnlyStorageEngine.getChunkedFileSet().getChunkIdToNumChunks().keySet(), running);
                    } else {
                        logger.info("Fetching entries for RW store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        updateStatus("Fetching entries for RW store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " ) ");
                        if (partitionIds.size() > 0) {
                            Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = adminClient.bulkFetchOps.fetchEntries(nodeId, storeName, partitionIds, filter, false, initialCluster, 0);
                            long numTuples = 0;
                            long startTime = System.currentTimeMillis();
                            long startNs = System.nanoTime();
                            while (running.get() && entriesIterator.hasNext()) {
                                Pair<ByteArray, Versioned<byte[]>> entry = entriesIterator.next();
                                if (streamingStats != null) {
                                    streamingStats.reportNetworkTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
                                }
                                ByteArray key = entry.getFirst();
                                Versioned<byte[]> value = entry.getSecond();
                                startNs = System.nanoTime();
                                try {
                                    /**
                                         * TODO This also needs to be fixed to
                                         * use the atomic multi version puts
                                         */
                                    storageEngine.put(key, value, null);
                                } catch (ObsoleteVersionException e) {
                                    // log and ignore
                                    if (logger.isDebugEnabled()) {
                                        logger.debug("Fetch and update threw Obsolete version exception. Ignoring");
                                    }
                                } finally {
                                    if (streamingStats != null) {
                                        streamingStats.reportStreamingPut(Operation.UPDATE_ENTRIES);
                                        streamingStats.reportStorageTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
                                    }
                                }
                                long totalTime = (System.currentTimeMillis() - startTime) / 1000;
                                throttler.maybeThrottle(key.length() + valueSize(value));
                                if ((numTuples % 100000) == 0 && numTuples > 0) {
                                    logger.info(numTuples + " entries copied from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                                    updateStatus(numTuples + " entries copied from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                                }
                                numTuples++;
                            }
                            long totalTime = (System.currentTimeMillis() - startTime) / 1000;
                            if (running.get()) {
                                logger.info("Completed fetching " + numTuples + " entries from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                            } else {
                                logger.info("Fetch and update stopped after fetching " + numTuples + " entries for node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                            }
                        } else {
                            logger.info("No entries to fetch from node " + nodeId + " for store '" + storeName + "'");
                        }
                    }
                } finally {
                    adminClient.close();
                }
            }
        });
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleFetchAndUpdate failed for request(" + request.toString() + ")", e);
    }
    return response.build();
}

Example 35 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class UpdatePartitionEntriesStreamRequestHandler method handleRequest.

@Override
public StreamRequestHandlerState handleRequest(DataInputStream inputStream, DataOutputStream outputStream) throws IOException {
    if (!metadataStore.getPartitionStreamingEnabledUnlocked()) {
        throw new VoldemortException("Partition streaming is disabled on node " + metadataStore.getNodeId() + " under " + metadataStore.getServerStateUnlocked() + " state.");
    }
    long startNs = System.nanoTime();
    if (request == null) {
        int size = 0;
        try {
            size = inputStream.readInt();
        } catch (EOFException e) {
            if (logger.isTraceEnabled())
                logger.trace("Incomplete read for message size");
            if (streamStats != null)
                streamStats.reportNetworkTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
            return StreamRequestHandlerState.INCOMPLETE_READ;
        }
        if (size == -1) {
            long totalTime = (System.currentTimeMillis() - startTime) / 1000;
            logger.info(getHandlerName() + " successfully updated " + counter + " entries for store '" + storageEngine.getName() + "' in " + totalTime + " s");
            if (logger.isTraceEnabled())
                logger.trace("Message size -1, completed partition update");
            if (streamStats != null)
                streamStats.reportNetworkTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
            return StreamRequestHandlerState.COMPLETE;
        }
        if (logger.isTraceEnabled())
            logger.trace("UpdatePartitionEntriesRequest message size: " + size);
        byte[] input = new byte[size];
        try {
            ByteUtils.read(inputStream, input);
        } catch (EOFException e) {
            if (logger.isTraceEnabled())
                logger.trace("Incomplete read for message");
            return StreamRequestHandlerState.INCOMPLETE_READ;
        } finally {
            if (streamStats != null)
                streamStats.reportNetworkTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
        }
        VAdminProto.UpdatePartitionEntriesRequest.Builder builder = VAdminProto.UpdatePartitionEntriesRequest.newBuilder();
        builder.mergeFrom(input);
        request = builder.build();
    }
    VAdminProto.PartitionEntry partitionEntry = request.getPartitionEntry();
    ByteArray key = ProtoUtils.decodeBytes(partitionEntry.getKey());
    Versioned<byte[]> value = ProtoUtils.decodeVersioned(partitionEntry.getVersioned());
    if (filter.accept(key, value)) {
        startNs = System.nanoTime();
        try {
            processEntry(key, value);
            if (logger.isTraceEnabled())
                logger.trace(getHandlerName() + " (Streaming put) successful");
        } catch (ObsoleteVersionException e) {
            // log and ignore
            if (logger.isDebugEnabled())
                logger.debug(getHandlerName() + " (Streaming put) threw ObsoleteVersionException, Ignoring.");
        } finally {
            if (streamStats != null) {
                streamStats.reportStreamingPut(Operation.UPDATE_ENTRIES);
                streamStats.reportStorageTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
            }
        }
        throttler.maybeThrottle(key.length() + AdminServiceRequestHandler.valueSize(value));
    }
    // log progress
    counter++;
    if (0 == counter % STAT_RECORDS_INTERVAL) {
        long totalTime = (System.currentTimeMillis() - startTime) / Time.MS_PER_SECOND;
        logger.info(getHandlerName() + " updated " + counter + " entries for store '" + storageEngine.getName() + "' in " + totalTime + " s");
    }
    request = null;
    return StreamRequestHandlerState.READING;
}