Examples with MessageInfo com.github.ambry.store.MessageInfo used on opensource projects

Example 21 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class MessageInfoAndMetadataListSerde method getMessageInfoAndMetadataListSize.

/**
 * @return the size in bytes that serialization of this MessageInfoAndMetadataList will result in.
 */
int getMessageInfoAndMetadataListSize() {
    int size = 0;
    if (version >= VERSION_5) {
        // version
        size += Short.BYTES;
    }
    // num elements in list
    size += Integer.BYTES;
    if (messageInfoList == null) {
        return size;
    }
    ListIterator<MessageInfo> infoListIterator = messageInfoList.listIterator();
    ListIterator<MessageMetadata> metadataListIterator = messageMetadataList.listIterator();
    while (infoListIterator.hasNext()) {
        MessageInfo messageInfo = infoListIterator.next();
        MessageMetadata messageMetadata = metadataListIterator.next();
        size += messageInfo.getStoreKey().sizeInBytes();
        // message size
        size += Long.BYTES;
        // expiration time
        size += Long.BYTES;
        // whether deleted
        size += Byte.BYTES;
        if (version < VERSION_1 || version > VERSION_MAX) {
            throw new IllegalArgumentException("Unknown version in MessageInfoList " + version);
        }
        if (version >= VERSION_5) {
            // whether ttl updated
            size += Byte.BYTES;
        }
        if (version >= VERSION_6) {
            // whether undelete
            size += Byte.BYTES;
        }
        if (version > VERSION_1) {
            // whether crc is present
            size += Byte.BYTES;
            if (messageInfo.getCrc() != null) {
                // crc
                size += Long.BYTES;
            }
        }
        if (version > VERSION_2) {
            // accountId
            size += Short.BYTES;
            // containerId
            size += Short.BYTES;
            // operationTime
            size += Long.BYTES;
        }
        if (version >= VERSION_6) {
            // lifeVersion
            size += Short.BYTES;
        }
        if (version > VERSION_3) {
            // whether message metadata is present.
            size += Byte.BYTES;
            if (messageMetadata != null) {
                size += messageMetadata.sizeInBytes();
            }
        }
    }
    return size;
}

Example 22 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class MessageInfoAndMetadataListSerde method serializeMessageInfoAndMetadataList.

/**
 * Serialize this object into the given buffer.
 * @param outputBuf the {@link ByteBuf} to serialize into.
 */
void serializeMessageInfoAndMetadataList(ByteBuf outputBuf) {
    if (version >= VERSION_5) {
        outputBuf.writeShort(version);
    }
    outputBuf.writeInt(messageInfoList == null ? 0 : messageInfoList.size());
    if (messageInfoList != null) {
        ListIterator<MessageInfo> infoListIterator = messageInfoList.listIterator();
        ListIterator<MessageMetadata> metadataListIterator = messageMetadataList.listIterator();
        while (infoListIterator.hasNext()) {
            MessageInfo messageInfo = infoListIterator.next();
            MessageMetadata messageMetadata = metadataListIterator.next();
            outputBuf.writeBytes(messageInfo.getStoreKey().toBytes());
            outputBuf.writeLong(messageInfo.getSize());
            outputBuf.writeLong(messageInfo.getExpirationTimeInMs());
            outputBuf.writeByte(messageInfo.isDeleted() ? UPDATED : (byte) ~UPDATED);
            if (version < VERSION_1 || version > VERSION_MAX) {
                throw new IllegalArgumentException("Unknown version in MessageInfoList " + version);
            }
            if (version >= VERSION_5) {
                outputBuf.writeByte(messageInfo.isTtlUpdated() ? UPDATED : (byte) ~UPDATED);
            }
            if (version >= VERSION_6) {
                outputBuf.writeByte(messageInfo.isUndeleted() ? UPDATED : (byte) ~UPDATED);
            }
            if (version > VERSION_1) {
                Long crc = messageInfo.getCrc();
                if (crc != null) {
                    outputBuf.writeByte(FIELD_PRESENT);
                    outputBuf.writeLong(crc);
                } else {
                    outputBuf.writeByte((byte) ~FIELD_PRESENT);
                }
            }
            if (version > VERSION_2) {
                outputBuf.writeShort(messageInfo.getAccountId());
                outputBuf.writeShort(messageInfo.getContainerId());
                outputBuf.writeLong(messageInfo.getOperationTimeMs());
            }
            if (version >= VERSION_6) {
                outputBuf.writeShort(messageInfo.getLifeVersion());
            }
            if (version > VERSION_3) {
                if (messageMetadata != null) {
                    outputBuf.writeByte(FIELD_PRESENT);
                    messageMetadata.serializeMessageMetadata(outputBuf);
                } else {
                    outputBuf.writeByte((byte) ~FIELD_PRESENT);
                }
            }
        }
    }
}

Example 23 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class MessageInfoAndMetadataListSerde method deserializeMessageInfoAndMetadataList.

/**
 * Deserialize the given stream and return the MessageInfo and Metadata lists.
 * @param stream the stream to deserialize from.
 * @param map the clustermap to use.
 * @param versionToDeserializeIn the SerDe version to use to deserialize.
 * @return the deserialized {@link MessageInfoAndMetadataListSerde}.
 * @throws IOException if an I/O error occurs while reading from the stream.
 */
static MessageInfoAndMetadataListSerde deserializeMessageInfoAndMetadataList(DataInputStream stream, ClusterMap map, short versionToDeserializeIn) throws IOException {
    if (versionToDeserializeIn >= VERSION_5) {
        short versionFromStream = stream.readShort();
        if (versionFromStream != versionToDeserializeIn) {
            throw new IllegalArgumentException("Argument provided [" + versionToDeserializeIn + "] and stream [" + versionFromStream + "] disagree on version");
        }
    } else {
        versionToDeserializeIn = versionToDeserializeIn == DETERMINE_VERSION ? stream.readShort() : versionToDeserializeIn;
    }
    int messageCount = stream.readInt();
    ArrayList<MessageInfo> messageInfoList = new ArrayList<>(messageCount);
    ArrayList<MessageMetadata> messageMetadataList = new ArrayList<>(messageCount);
    for (int i = 0; i < messageCount; i++) {
        BlobId id = new BlobId(stream, map);
        long size = stream.readLong();
        long ttl = stream.readLong();
        boolean isDeleted = stream.readByte() == UPDATED;
        boolean isTtlUpdated = false;
        boolean isUndeleted = false;
        short lifeVersion = 0;
        Long crc = null;
        short accountId = Account.UNKNOWN_ACCOUNT_ID;
        short containerId = Container.UNKNOWN_CONTAINER_ID;
        long operationTime = Utils.Infinite_Time;
        if (versionToDeserializeIn < VERSION_1 || versionToDeserializeIn > VERSION_MAX) {
            throw new IllegalArgumentException("Unknown version to deserialize MessageInfoList " + versionToDeserializeIn);
        }
        if (versionToDeserializeIn >= VERSION_5) {
            isTtlUpdated = stream.readByte() == UPDATED;
        }
        if (versionToDeserializeIn >= VERSION_6) {
            isUndeleted = stream.readByte() == UPDATED;
        }
        if (versionToDeserializeIn > VERSION_1) {
            crc = stream.readByte() == FIELD_PRESENT ? stream.readLong() : null;
        }
        if (versionToDeserializeIn > VERSION_2) {
            accountId = stream.readShort();
            containerId = stream.readShort();
            operationTime = stream.readLong();
        }
        if (versionToDeserializeIn >= VERSION_6) {
            lifeVersion = stream.readShort();
        }
        messageInfoList.add(new MessageInfo(id, size, isDeleted, isTtlUpdated, isUndeleted, ttl, crc, accountId, containerId, operationTime, lifeVersion));
        if (versionToDeserializeIn > VERSION_3) {
            MessageMetadata messageMetadata = stream.readByte() == FIELD_PRESENT ? MessageMetadata.deserializeMessageMetadata(stream) : null;
            messageMetadataList.add(messageMetadata);
        } else {
            messageMetadataList.add(null);
        }
    }
    return new MessageInfoAndMetadataListSerde(messageInfoList, messageMetadataList, versionToDeserializeIn);
}

Example 24 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class AmbryRequests method handleUndeleteRequest.

@Override
public void handleUndeleteRequest(NetworkRequest request) throws IOException, InterruptedException {
    UndeleteRequest undeleteRequest;
    if (request instanceof LocalChannelRequest) {
        // This is a case where handleUndeleteRequest is called when frontends are talking to Azure. In this case, this method
        // is called by request handler threads running within the frontend router itself. So, the request can be directly
        // referenced as java objects without any need for deserialization.
        undeleteRequest = (UndeleteRequest) ((LocalChannelRequest) request).getRequestInfo().getRequest();
    } else {
        undeleteRequest = UndeleteRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
    }
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    metrics.undeleteBlobRequestQueueTimeInMs.update(requestQueueTime);
    metrics.undeleteBlobRequestRate.mark();
    long startTime = SystemTime.getInstance().milliseconds();
    UndeleteResponse response = null;
    Store storeToUndelete;
    StoreKey convertedStoreKey;
    try {
        convertedStoreKey = getConvertedStoreKeys(Collections.singletonList(undeleteRequest.getBlobId())).get(0);
        ServerErrorCode error = validateRequest(undeleteRequest.getBlobId().getPartition(), RequestOrResponseType.UndeleteRequest, false);
        if (error != ServerErrorCode.No_Error) {
            logger.error("Validating undelete request failed with error {} for request {}", error, undeleteRequest);
            response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), error);
        } else {
            BlobId convertedBlobId = (BlobId) convertedStoreKey;
            MessageInfo info = new MessageInfo.Builder(convertedBlobId, -1, convertedBlobId.getAccountId(), convertedBlobId.getContainerId(), undeleteRequest.getOperationTimeMs()).isUndeleted(true).lifeVersion(MessageInfo.LIFE_VERSION_FROM_FRONTEND).build();
            storeToUndelete = storeManager.getStore(undeleteRequest.getBlobId().getPartition());
            short lifeVersion = storeToUndelete.undelete(info);
            response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), lifeVersion);
            if (notification != null) {
                notification.onBlobReplicaUndeleted(currentNode.getHostname(), currentNode.getPort(), convertedStoreKey.getID(), BlobReplicaSourceType.PRIMARY);
            }
        }
    } catch (StoreException e) {
        boolean logInErrorLevel = false;
        if (e.getErrorCode() == StoreErrorCodes.ID_Not_Found) {
            metrics.idNotFoundError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.TTL_Expired) {
            metrics.ttlExpiredError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.ID_Deleted_Permanently) {
            metrics.idDeletedError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.Life_Version_Conflict) {
            metrics.lifeVersionConflictError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.ID_Not_Deleted) {
            metrics.idNotDeletedError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.ID_Undeleted) {
            metrics.idUndeletedError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.Authorization_Failure) {
            metrics.undeleteAuthorizationFailure.inc();
        } else {
            logInErrorLevel = true;
            metrics.unExpectedStoreUndeleteError.inc();
        }
        if (logInErrorLevel) {
            logger.error("Store exception on a undelete with error code {} for request {}", e.getErrorCode(), undeleteRequest, e);
        } else {
            logger.trace("Store exception on a undelete with error code {} for request {}", e.getErrorCode(), undeleteRequest, e);
        }
        if (e.getErrorCode() == StoreErrorCodes.ID_Undeleted) {
            if (e instanceof IdUndeletedStoreException) {
                response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), ((IdUndeletedStoreException) e).getLifeVersion(), ServerErrorCode.Blob_Already_Undeleted);
            } else {
                response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), MessageInfo.LIFE_VERSION_FROM_FRONTEND, ServerErrorCode.Blob_Already_Undeleted);
            }
        } else {
            response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
        }
    } catch (Exception e) {
        logger.error("Unknown exception for undelete request {}", undeleteRequest, e);
        response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), ServerErrorCode.Unknown_Error);
        metrics.unExpectedStoreUndeleteError.inc();
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", undeleteRequest, response, processingTime);
        metrics.undeleteBlobProcessingTimeInMs.update(processingTime);
    }
    requestResponseChannel.sendResponse(response, request, new ServerNetworkResponseMetrics(metrics.undeleteBlobResponseQueueTimeInMs, metrics.undeleteBlobSendTimeInMs, metrics.undeleteBlobTotalTimeInMs, null, null, totalTimeSpent));
}

Example 25 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class MessageInfoAndMetadataListSerDeTest method testSerDe.

@Test
public void testSerDe() throws Exception {
    MockClusterMap mockMap = new MockClusterMap();
    MockPartitionId partitionId = new MockPartitionId();
    short[] accountIds = { 100, 101, 102, 103 };
    short[] containerIds = { 10, 11, 12, 13 };
    boolean[] isDeletedVals = { false, true, false, true };
    boolean[] isTtlUpdatedVals = { true, false, false, true };
    boolean[] isUndeletedVals = { true, false, false, true };
    short[] lifeVersionVals = { 1, 2, 3, 4 };
    Long[] crcs = { null, 100L, Long.MIN_VALUE, Long.MAX_VALUE };
    StoreKey[] keys = { new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[0], containerIds[0], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[1], containerIds[1], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[2], containerIds[2], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[3], containerIds[3], partitionId, false, BlobId.BlobDataType.DATACHUNK) };
    long[] blobSizes = { 1024, 2048, 4096, 8192 };
    long[] times = { SystemTime.getInstance().milliseconds(), SystemTime.getInstance().milliseconds() - 1, SystemTime.getInstance().milliseconds() + TimeUnit.SECONDS.toMillis(5), Utils.Infinite_Time };
    MessageMetadata[] messageMetadata = new MessageMetadata[4];
    messageMetadata[0] = new MessageMetadata(ByteBuffer.wrap(getRandomBytes(100)));
    messageMetadata[1] = new MessageMetadata(null);
    messageMetadata[2] = null;
    messageMetadata[3] = new MessageMetadata(ByteBuffer.wrap(getRandomBytes(200)));
    List<MessageInfo> messageInfoList = new ArrayList<>(4);
    List<MessageMetadata> messageMetadataList = new ArrayList<>(4);
    for (int i = 0; i < 4; i++) {
        messageInfoList.add(new MessageInfo(keys[i], blobSizes[i], isDeletedVals[i], isTtlUpdatedVals[i], isUndeletedVals[i], times[i], crcs[i], accountIds[i], containerIds[i], times[i], lifeVersionVals[i]));
        messageMetadataList.add(messageMetadata[i]);
    }
    // Serialize and then deserialize
    MessageInfoAndMetadataListSerde messageInfoAndMetadataListSerde = new MessageInfoAndMetadataListSerde(messageInfoList, messageMetadataList, serDeVersion);
    ByteBuf buffer = Unpooled.buffer(messageInfoAndMetadataListSerde.getMessageInfoAndMetadataListSize());
    messageInfoAndMetadataListSerde.serializeMessageInfoAndMetadataList(buffer);
    if (serDeVersion >= MessageInfoAndMetadataListSerde.VERSION_5) {
        // should fail if the wrong version is provided
        try {
            MessageInfoAndMetadataListSerde.deserializeMessageInfoAndMetadataList(new NettyByteBufDataInputStream(buffer), mockMap, (short) (serDeVersion - 1));
            Assert.fail("Should have failed to deserialize");
        } catch (IllegalArgumentException e) {
        // expected. Nothing to do
        }
    }
    buffer.readerIndex(0);
    MessageInfoAndMetadataListSerde messageInfoAndMetadataList = MessageInfoAndMetadataListSerde.deserializeMessageInfoAndMetadataList(new NettyByteBufDataInputStream(buffer), mockMap, serDeVersion);
    // Verify
    List<MessageInfo> responseMessageInfoList = messageInfoAndMetadataList.getMessageInfoList();
    List<MessageMetadata> responseMessageMetadataList = messageInfoAndMetadataList.getMessageMetadataList();
    Assert.assertEquals(4, responseMessageInfoList.size());
    Assert.assertEquals(4, responseMessageMetadataList.size());
    for (int i = 0; i < 4; i++) {
        assertMessageInfoEquality(messageInfoList.get(i), responseMessageInfoList.get(i));
        assertMessageMetadataEquality(messageMetadataList.get(i), responseMessageMetadataList.get(i));
    }
}