Examples with MessageFormatWriteSet com.github.ambry.messageformat.MessageFormatWriteSet used on opensource projects

Example 11 with MessageFormatWriteSet

use of com.github.ambry.messageformat.MessageFormatWriteSet in project ambry by linkedin.

the class AmbryRequests method handlePutRequest.

public void handlePutRequest(Request request) throws IOException, InterruptedException {
    PutRequest.ReceivedPutRequest receivedRequest = PutRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    metrics.putBlobRequestQueueTimeInMs.update(requestQueueTime);
    metrics.putBlobRequestRate.mark();
    long startTime = SystemTime.getInstance().milliseconds();
    PutResponse response = null;
    try {
        ServerErrorCode error = validateRequest(receivedRequest.getBlobId().getPartition(), RequestOrResponseType.PutRequest);
        if (error != ServerErrorCode.No_Error) {
            logger.error("Validating put request failed with error {} for request {}", error, receivedRequest);
            response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), error);
        } else {
            MessageFormatInputStream stream = new PutMessageFormatInputStream(receivedRequest.getBlobId(), receivedRequest.getBlobEncryptionKey(), receivedRequest.getBlobProperties(), receivedRequest.getUsermetadata(), receivedRequest.getBlobStream(), receivedRequest.getBlobSize(), receivedRequest.getBlobType());
            MessageInfo info = new MessageInfo(receivedRequest.getBlobId(), stream.getSize(), false, Utils.addSecondsToEpochTime(receivedRequest.getBlobProperties().getCreationTimeInMs(), receivedRequest.getBlobProperties().getTimeToLiveInSeconds()), receivedRequest.getCrc(), receivedRequest.getBlobProperties().getAccountId(), receivedRequest.getBlobProperties().getContainerId(), receivedRequest.getBlobProperties().getCreationTimeInMs());
            ArrayList<MessageInfo> infoList = new ArrayList<MessageInfo>();
            infoList.add(info);
            MessageFormatWriteSet writeset = new MessageFormatWriteSet(stream, infoList, false);
            Store storeToPut = storageManager.getStore(receivedRequest.getBlobId().getPartition());
            storeToPut.put(writeset);
            response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.No_Error);
            metrics.blobSizeInBytes.update(receivedRequest.getBlobSize());
            metrics.blobUserMetadataSizeInBytes.update(receivedRequest.getUsermetadata().limit());
            if (notification != null) {
                notification.onBlobReplicaCreated(currentNode.getHostname(), currentNode.getPort(), receivedRequest.getBlobId().getID(), BlobReplicaSourceType.PRIMARY);
            }
        }
    } catch (StoreException e) {
        logger.error("Store exception on a put with error code " + e.getErrorCode() + " for request " + receivedRequest, e);
        if (e.getErrorCode() == StoreErrorCodes.Already_Exist) {
            metrics.idAlreadyExistError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.IOError) {
            metrics.storeIOError.inc();
        } else {
            metrics.unExpectedStorePutError.inc();
        }
        response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
    } catch (Exception e) {
        logger.error("Unknown exception on a put for request " + receivedRequest, e);
        response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.Unknown_Error);
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", receivedRequest, response, processingTime);
        metrics.putBlobProcessingTimeInMs.update(processingTime);
        metrics.updatePutBlobProcessingTimeBySize(receivedRequest.getBlobSize(), processingTime);
    }
    sendPutResponse(requestResponseChannel, response, request, metrics.putBlobResponseQueueTimeInMs, metrics.putBlobSendTimeInMs, metrics.putBlobTotalTimeInMs, totalTimeSpent, receivedRequest.getBlobSize(), metrics);
}

Example 12 with MessageFormatWriteSet

use of com.github.ambry.messageformat.MessageFormatWriteSet in project ambry by linkedin.

the class AmbryRequests method handlePutRequest.

@Override
public void handlePutRequest(NetworkRequest request) throws IOException, InterruptedException {
    PutRequest receivedRequest;
    if (request instanceof LocalChannelRequest) {
        // This is a case where handlePutRequest is called when frontends are writing to Azure. In this case, this method
        // is called by request handler threads running within the frontend router itself. So, the request can be directly
        // referenced as java objects without any need for deserialization.
        PutRequest sentRequest = (PutRequest) ((LocalChannelRequest) request).getRequestInfo().getRequest();
        // However, we will create a new PutRequest object to represent the received Put request since the blob content
        // 'buffer' in PutRequest is accessed as 'stream' while writing to Store. Also, crc value for this request
        // would be null since it is only calculated (on the fly) when sending the request to network. It might be okay to
        // use null crc here since the scenario for which we are using crc (i.e. possibility of collisions due to fast
        // replication) as described in this PR https://github.com/linkedin/ambry/pull/549 might not be applicable when
        // frontends are talking to Azure.
        receivedRequest = new PutRequest(sentRequest.getCorrelationId(), sentRequest.getClientId(), sentRequest.getBlobId(), sentRequest.getBlobProperties(), sentRequest.getUsermetadata(), sentRequest.getBlobSize(), sentRequest.getBlobType(), sentRequest.getBlobEncryptionKey(), new ByteBufInputStream(sentRequest.getBlob()), null);
    } else {
        InputStream is = request.getInputStream();
        DataInputStream dis = is instanceof DataInputStream ? (DataInputStream) is : new DataInputStream(is);
        receivedRequest = PutRequest.readFrom(dis, clusterMap);
    }
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    metrics.putBlobRequestQueueTimeInMs.update(requestQueueTime);
    metrics.putBlobRequestRate.mark();
    long startTime = SystemTime.getInstance().milliseconds();
    PutResponse response = null;
    try {
        ServerErrorCode error = validateRequest(receivedRequest.getBlobId().getPartition(), RequestOrResponseType.PutRequest, false);
        if (error != ServerErrorCode.No_Error) {
            logger.error("Validating put request failed with error {} for request {}", error, receivedRequest);
            response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), error);
        } else {
            MessageFormatInputStream stream = new PutMessageFormatInputStream(receivedRequest.getBlobId(), receivedRequest.getBlobEncryptionKey(), receivedRequest.getBlobProperties(), receivedRequest.getUsermetadata(), receivedRequest.getBlobStream(), receivedRequest.getBlobSize(), receivedRequest.getBlobType());
            BlobProperties properties = receivedRequest.getBlobProperties();
            long expirationTime = Utils.addSecondsToEpochTime(receivedRequest.getBlobProperties().getCreationTimeInMs(), properties.getTimeToLiveInSeconds());
            MessageInfo info = new MessageInfo.Builder(receivedRequest.getBlobId(), stream.getSize(), properties.getAccountId(), properties.getContainerId(), properties.getCreationTimeInMs()).expirationTimeInMs(expirationTime).crc(receivedRequest.getCrc()).lifeVersion(MessageInfo.LIFE_VERSION_FROM_FRONTEND).build();
            ArrayList<MessageInfo> infoList = new ArrayList<>();
            infoList.add(info);
            MessageFormatWriteSet writeset = new MessageFormatWriteSet(stream, infoList, false);
            Store storeToPut = storeManager.getStore(receivedRequest.getBlobId().getPartition());
            storeToPut.put(writeset);
            response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.No_Error);
            metrics.blobSizeInBytes.update(receivedRequest.getBlobSize());
            metrics.blobUserMetadataSizeInBytes.update(receivedRequest.getUsermetadata().limit());
            if (notification != null) {
                notification.onBlobReplicaCreated(currentNode.getHostname(), currentNode.getPort(), receivedRequest.getBlobId().getID(), BlobReplicaSourceType.PRIMARY);
            }
        }
    } catch (StoreException e) {
        logger.error("Store exception on a put with error code {} for request {}", e.getErrorCode(), receivedRequest, e);
        if (e.getErrorCode() == StoreErrorCodes.Already_Exist) {
            metrics.idAlreadyExistError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.IOError) {
            metrics.storeIOError.inc();
        } else {
            metrics.unExpectedStorePutError.inc();
        }
        response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
    } catch (Exception e) {
        logger.error("Unknown exception on a put for request {}", receivedRequest, e);
        response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.Unknown_Error);
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", receivedRequest, response, processingTime);
        metrics.putBlobProcessingTimeInMs.update(processingTime);
        metrics.updatePutBlobProcessingTimeBySize(receivedRequest.getBlobSize(), processingTime);
    }
    sendPutResponse(requestResponseChannel, response, request, metrics.putBlobResponseQueueTimeInMs, metrics.putBlobSendTimeInMs, metrics.putBlobTotalTimeInMs, totalTimeSpent, receivedRequest.getBlobSize(), metrics);
}

Example 13 with MessageFormatWriteSet

use of com.github.ambry.messageformat.MessageFormatWriteSet in project ambry by linkedin.

the class InMemoryStore method undelete.

@Override
public short undelete(MessageInfo info) throws StoreException {
    StoreKey key = info.getStoreKey();
    MessageInfo deleteInfo = getMessageInfo(key, messageInfos, true, false, false);
    if (info.getLifeVersion() == -1 && deleteInfo == null) {
        throw new StoreException("Key " + key + " not delete yet", StoreErrorCodes.ID_Not_Deleted);
    }
    short lifeVersion = info.getLifeVersion();
    MessageInfo latestInfo = deleteInfo;
    if (info.getLifeVersion() == MessageInfo.LIFE_VERSION_FROM_FRONTEND) {
        if (deleteInfo == null) {
            throw new StoreException("Id " + key + " requires first value to be a put and last value to be a delete", StoreErrorCodes.ID_Not_Deleted);
        }
        lifeVersion = (short) (deleteInfo.getLifeVersion() + 1);
    } else {
        if (deleteInfo == null) {
            latestInfo = getMergedMessageInfo(key, messageInfos);
        }
    }
    try {
        MessageFormatInputStream stream = new UndeleteMessageFormatInputStream(key, info.getAccountId(), info.getContainerId(), info.getOperationTimeMs(), lifeVersion);
        // Update info to add stream size;
        info = new MessageInfo(key, stream.getSize(), false, latestInfo.isTtlUpdated(), true, latestInfo.getExpirationTimeInMs(), null, info.getAccountId(), info.getContainerId(), info.getOperationTimeMs(), lifeVersion);
        MessageFormatWriteSet writeSet = new MessageFormatWriteSet(stream, Collections.singletonList(info), false);
        writeSet.writeTo(log);
        messageInfos.add(info);
        return lifeVersion;
    } catch (Exception e) {
        throw new StoreException("Unknown error while trying to undelete blobs from store", e, StoreErrorCodes.Unknown_Error);
    }
}

Example 14 with MessageFormatWriteSet

use of com.github.ambry.messageformat.MessageFormatWriteSet in project ambry by linkedin.

the class StoreCopierTest method addMessage.

/**
 * Adds a message to the given {@code store}.
 * @param store the {@link Store} to add the message to.
 * @param key the {@link StoreKey} associated with the message.
 * @param expiryTimeMs the expiry time associated with the message.
 * @param isDelete {@code true} if this is a delete message, {@code false} otherwise.
 * @param isTtlUpdate {@code true} if this is a ttl update message or if {@code isDelete} is true and the created
 *                    {@link MessageInfo} needs to indicate that a TTL update had occurred, {@code false} otherwise.
 * @param accountId accountId of the blob
 * @param containerId containerId of the blob
 * @param operationTimeMs operationTime in ms of put or delete
 * @return the message that was written.
 * @throws IOException
 * @throws StoreException
 */
private byte[] addMessage(Store store, StoreKey key, long expiryTimeMs, boolean isDelete, boolean isTtlUpdate, short accountId, short containerId, long operationTimeMs) throws IOException, StoreException {
    int size = PUT_RECORD_SIZE;
    MessageInfo messageInfo = new MessageInfo(key, size, isDelete, isTtlUpdate, expiryTimeMs, accountId, containerId, operationTimeMs);
    byte[] data = TestUtils.getRandomBytes(size);
    MessageFormatWriteSet writeSet = new MessageFormatWriteSet(new ByteArrayInputStream(data), Collections.singletonList(messageInfo), false);
    if (isDelete) {
        store.delete(writeSet.getMessageSetInfo());
    } else if (isTtlUpdate) {
        store.updateTtl(writeSet.getMessageSetInfo());
    } else {
        store.put(writeSet);
    }
    return data;
}