Examples with DataLoadTargetPb com.alibaba.maxgraph.proto.DataLoadTargetPb used on opensource projects

Example 1 with DataLoadTargetPb

use of com.alibaba.maxgraph.proto.DataLoadTargetPb in project GraphScope by alibaba.

the class PrepareDataLoadExecutor method execute.

@Override
public DdlResult execute(ByteString ddlBlob, GraphDef graphDef, int partitionCount) throws InvalidProtocolBufferException {
    DataLoadTargetPb dataLoadTargetPb = DataLoadTargetPb.parseFrom(ddlBlob);
    DataLoadTarget dataLoadTarget = DataLoadTarget.parseProto(dataLoadTargetPb);
    String label = dataLoadTarget.getLabel();
    String srcLabel = dataLoadTarget.getSrcLabel();
    String dstLabel = dataLoadTarget.getDstLabel();
    long version = graphDef.getSchemaVersion();
    if (!graphDef.hasLabel(label)) {
        throw new DdlException("label [" + label + "] not exists, schema version [" + version + "]");
    }
    GraphDef.Builder graphDefBuilder = GraphDef.newBuilder(graphDef);
    TypeDef typeDef = graphDef.getTypeDef(label);
    long tableIdx = graphDef.getTableIdx();
    tableIdx++;
    DataLoadTarget.Builder targetBuilder = DataLoadTarget.newBuilder(dataLoadTarget);
    if (srcLabel == null || srcLabel.isEmpty()) {
        // Vertex type
        if (typeDef.getTypeEnum() != TypeEnum.VERTEX) {
            throw new DdlException("invalid data load target [" + dataLoadTarget + "], label is not a vertex");
        }
        graphDefBuilder.putVertexTableId(typeDef.getTypeLabelId(), tableIdx);
        targetBuilder.setLabelId(typeDef.getLabelId());
    } else {
        // Edge kind
        if (typeDef.getTypeEnum() != TypeEnum.EDGE) {
            throw new DdlException("invalid data load target [" + dataLoadTarget + "], label is not an edge");
        }
        EdgeKind.Builder edgeKindBuilder = EdgeKind.newBuilder();
        LabelId edgeLabelId = graphDef.getLabelId(label);
        if (edgeLabelId == null) {
            throw new DdlException("invalid edgeLabel [" + label + "], schema version [" + version + "]");
        }
        edgeKindBuilder.setEdgeLabelId(edgeLabelId);
        targetBuilder.setLabelId(edgeLabelId.getId());
        LabelId srcVertexLabelId = graphDef.getLabelId(srcLabel);
        if (srcVertexLabelId == null) {
            throw new DdlException("invalid srcVertexLabel [" + srcLabel + "], schema version [" + version + "]");
        }
        edgeKindBuilder.setSrcVertexLabelId(srcVertexLabelId);
        targetBuilder.setSrcLabelId(srcVertexLabelId.getId());
        LabelId dstVertexLabelId = graphDef.getLabelId(dstLabel);
        if (dstVertexLabelId == null) {
            throw new DdlException("invalid dstVertexLabel [" + dstLabel + "], schema version [" + version + "]");
        }
        edgeKindBuilder.setDstVertexLabelId(dstVertexLabelId);
        targetBuilder.setDstLabelId(dstVertexLabelId.getId());
        EdgeKind edgeKind = edgeKindBuilder.build();
        if (!graphDef.hasEdgeKind(edgeKind)) {
            throw new DdlException("invalid data load target [" + dataLoadTarget + "], edgeKind not exists");
        }
        graphDefBuilder.putEdgeTableId(edgeKind, tableIdx);
    }
    version++;
    graphDefBuilder.setTableIdx(tableIdx);
    graphDefBuilder.setVersion(version);
    GraphDef newGraphDef = graphDefBuilder.build();
    List<Operation> operations = new ArrayList<>(partitionCount);
    for (int i = 0; i < partitionCount; i++) {
        operations.add(new PrepareDataLoadOperation(i, version, targetBuilder.build(), tableIdx));
    }
    return new DdlResult(newGraphDef, operations);
}

Example 2 with DataLoadTargetPb

use of com.alibaba.maxgraph.proto.DataLoadTargetPb in project GraphScope by alibaba.

the class ClientService method prepareDataLoad.

@Override
public void prepareDataLoad(PrepareDataLoadRequest request, StreamObserver<PrepareDataLoadResponse> responseObserver) {
    DdlRequestBatch.Builder builder = DdlRequestBatch.newBuilder();
    for (DataLoadTargetPb dataLoadTargetPb : request.getDataLoadTargetsList()) {
        DataLoadTarget dataLoadTarget = DataLoadTarget.parseProto(dataLoadTargetPb);
        builder.addDdlRequest(new com.alibaba.graphscope.groot.schema.request.PrepareDataLoadRequest(dataLoadTarget));
    }
    DdlRequestBatch batch = builder.build();
    try {
        long snapshotId = this.batchDdlClient.batchDdl(batch);
        this.snapshotCache.addListener(snapshotId, () -> {
            responseObserver.onNext(PrepareDataLoadResponse.newBuilder().setGraphDef(this.snapshotCache.getSnapshotWithSchema().getGraphDef().toProto()).build());
            responseObserver.onCompleted();
        });
    } catch (Exception e) {
        logger.error("prepare data load failed", e);
        responseObserver.onError(Status.INTERNAL.withDescription(e.getMessage()).asRuntimeException());
    }
}

Example 3 with DataLoadTargetPb

use of com.alibaba.maxgraph.proto.DataLoadTargetPb in project GraphScope by alibaba.

the class ClientService method commitDataLoad.

@Override
public void commitDataLoad(CommitDataLoadRequest request, StreamObserver<CommitDataLoadResponse> responseObserver) {
    DdlRequestBatch.Builder builder = DdlRequestBatch.newBuilder();
    Map<Long, DataLoadTargetPb> tableToTarget = request.getTableToTargetMap();
    tableToTarget.forEach((tableId, targetPb) -> {
        DataLoadTarget dataLoadTarget = DataLoadTarget.parseProto(targetPb);
        builder.addDdlRequest(new com.alibaba.graphscope.groot.schema.request.CommitDataLoadRequest(dataLoadTarget, tableId));
    });
    DdlRequestBatch batch = builder.build();
    try {
        long snapshotId = this.batchDdlClient.batchDdl(batch);
        this.snapshotCache.addListener(snapshotId, () -> {
            responseObserver.onNext(CommitDataLoadResponse.newBuilder().build());
            responseObserver.onCompleted();
        });
    } catch (Exception e) {
        logger.error("commit data load failed", e);
        responseObserver.onError(Status.INTERNAL.withDescription(e.getMessage()).asRuntimeException());
    }
}

Example 4 with DataLoadTargetPb

use of com.alibaba.maxgraph.proto.DataLoadTargetPb in project GraphScope by alibaba.

the class CommitDataLoadExecutor method execute.

@Override
public DdlResult execute(ByteString ddlBlob, GraphDef graphDef, int partitionCount) throws InvalidProtocolBufferException {
    CommitDataLoadPb commitDataLoadPb = CommitDataLoadPb.parseFrom(ddlBlob);
    DataLoadTargetPb dataLoadTargetPb = commitDataLoadPb.getTarget();
    DataLoadTarget dataLoadTarget = DataLoadTarget.parseProto(dataLoadTargetPb);
    String label = dataLoadTarget.getLabel();
    String srcLabel = dataLoadTarget.getSrcLabel();
    String dstLabel = dataLoadTarget.getDstLabel();
    long version = graphDef.getSchemaVersion();
    if (!graphDef.hasLabel(label)) {
        throw new DdlException("label [" + label + "] not exists, schema version [" + version + "]");
    }
    GraphDef.Builder graphDefBuilder = GraphDef.newBuilder(graphDef);
    TypeDef typeDef = graphDef.getTypeDef(label);
    DataLoadTarget.Builder targetBuilder = DataLoadTarget.newBuilder(dataLoadTarget);
    if (srcLabel == null || srcLabel.isEmpty()) {
        // Vertex type
        if (typeDef.getTypeEnum() != TypeEnum.VERTEX) {
            throw new DdlException("invalid data load target [" + dataLoadTarget + "], label is not a vertex");
        }
        targetBuilder.setLabelId(typeDef.getLabelId());
    } else {
        // Edge kind
        if (typeDef.getTypeEnum() != TypeEnum.EDGE) {
            throw new DdlException("invalid data load target [" + dataLoadTarget + "], label is not an edge");
        }
        EdgeKind.Builder edgeKindBuilder = EdgeKind.newBuilder();
        LabelId edgeLabelId = graphDef.getLabelId(label);
        if (edgeLabelId == null) {
            throw new DdlException("invalid edgeLabel [" + label + "], schema version [" + version + "]");
        }
        edgeKindBuilder.setEdgeLabelId(edgeLabelId);
        targetBuilder.setLabelId(edgeLabelId.getId());
        LabelId srcVertexLabelId = graphDef.getLabelId(srcLabel);
        if (srcVertexLabelId == null) {
            throw new DdlException("invalid srcVertexLabel [" + srcLabel + "], schema version [" + version + "]");
        }
        edgeKindBuilder.setSrcVertexLabelId(srcVertexLabelId);
        targetBuilder.setSrcLabelId(srcVertexLabelId.getId());
        LabelId dstVertexLabelId = graphDef.getLabelId(dstLabel);
        if (dstVertexLabelId == null) {
            throw new DdlException("invalid dstVertexLabel [" + dstLabel + "], schema version [" + version + "]");
        }
        edgeKindBuilder.setDstVertexLabelId(dstVertexLabelId);
        targetBuilder.setDstLabelId(dstVertexLabelId.getId());
        EdgeKind edgeKind = edgeKindBuilder.build();
        if (!graphDef.hasEdgeKind(edgeKind)) {
            throw new DdlException("invalid data load target [" + dataLoadTarget + "], edgeKind not exists");
        }
    }
    version++;
    graphDefBuilder.setVersion(version);
    GraphDef newGraphDef = graphDefBuilder.build();
    List<Operation> operations = new ArrayList<>(partitionCount);
    for (int i = 0; i < partitionCount; i++) {
        operations.add(new CommitDataLoadOperation(i, version, CommitDataLoadPb.newBuilder().setTableIdx(commitDataLoadPb.getTableIdx()).setTarget(targetBuilder.build().toProto()).build()));
    }
    return new DdlResult(newGraphDef, operations);
}