Examples with RelationIdentifier org.apache.carbondata.core.metadata.schema.table.RelationIdentifier used on opensource projects

Example 6 with RelationIdentifier

use of org.apache.carbondata.core.metadata.schema.table.RelationIdentifier in project carbondata by apache.

the class ThriftWrapperSchemaConverterImpl method fromWrapperToExternalRI.

private List<org.apache.carbondata.format.RelationIdentifier> fromWrapperToExternalRI(List<RelationIdentifier> relationIdentifiersList) {
    List<org.apache.carbondata.format.RelationIdentifier> thriftRelationIdentifierList = new ArrayList<>();
    for (RelationIdentifier relationIdentifier : relationIdentifiersList) {
        org.apache.carbondata.format.RelationIdentifier thriftRelationIdentifier = new org.apache.carbondata.format.RelationIdentifier();
        thriftRelationIdentifier.setDatabaseName(relationIdentifier.getDatabaseName());
        thriftRelationIdentifier.setTableName(relationIdentifier.getTableName());
        thriftRelationIdentifier.setTableId(relationIdentifier.getTableId());
        thriftRelationIdentifierList.add(thriftRelationIdentifier);
    }
    return thriftRelationIdentifierList;
}

Example 7 with RelationIdentifier

use of org.apache.carbondata.core.metadata.schema.table.RelationIdentifier in project carbondata by apache.

the class MVProvider method isViewCanBeEnabled.

/**
 * This method checks if main table and mv table are synchronised or not. If synchronised
 * return true to enable the mv
 *
 * @param schema of mv to be disabled or enabled
 * @return flag to enable or disable mv
 * @throws IOException
 */
public boolean isViewCanBeEnabled(MVSchema schema, boolean ignoreDeferredCheck) throws IOException {
    if (!ignoreDeferredCheck) {
        if (!schema.isRefreshIncremental()) {
            return true;
        }
    }
    boolean isViewCanBeEnabled = true;
    String viewMetadataPath = CarbonTablePath.getMetadataPath(schema.getIdentifier().getTablePath());
    LoadMetadataDetails[] viewLoadMetadataDetails = SegmentStatusManager.readLoadMetadata(viewMetadataPath);
    Map<String, List<String>> viewSegmentMap = new HashMap<>();
    for (LoadMetadataDetails loadMetadataDetail : viewLoadMetadataDetails) {
        if (loadMetadataDetail.getSegmentStatus() == SegmentStatus.SUCCESS) {
            Map<String, List<String>> segmentMap = new Gson().fromJson(loadMetadataDetail.getExtraInfo(), Map.class);
            if (viewSegmentMap.isEmpty()) {
                viewSegmentMap.putAll(segmentMap);
            } else {
                for (Map.Entry<String, List<String>> entry : segmentMap.entrySet()) {
                    if (null != viewSegmentMap.get(entry.getKey())) {
                        viewSegmentMap.get(entry.getKey()).addAll(entry.getValue());
                    }
                }
            }
        }
    }
    List<RelationIdentifier> relatedTables = schema.getRelatedTables();
    for (RelationIdentifier relatedTable : relatedTables) {
        SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = SegmentStatusManager.getValidAndInvalidSegmentsInfo(relatedTable);
        List<String> relatedTableSegmentList = SegmentStatusManager.getValidSegmentList(validAndInvalidSegmentsInfo);
        if (!relatedTableSegmentList.isEmpty()) {
            if (viewSegmentMap.isEmpty()) {
                isViewCanBeEnabled = false;
            } else {
                String tableUniqueName = relatedTable.getDatabaseName() + CarbonCommonConstants.POINT + relatedTable.getTableName();
                isViewCanBeEnabled = viewSegmentMap.get(tableUniqueName).containsAll(relatedTableSegmentList);
                if (!isViewCanBeEnabled) {
                    // in case if main table is compacted and mv table mapping is not updated,
                    // check from merged Load Mapping
                    isViewCanBeEnabled = viewSegmentMap.get(tableUniqueName).containsAll(relatedTableSegmentList.stream().map(validAndInvalidSegmentsInfo.getMergedLoadMapping()::get).flatMap(Collection::stream).collect(Collectors.toList()));
                }
            }
        }
    }
    if (!isViewCanBeEnabled) {
        LOG.error("MV `" + schema.getIdentifier().getTableName() + "` is not in Sync with its related tables. Refresh MV to sync it.");
    }
    return isViewCanBeEnabled;
}

Example 8 with RelationIdentifier

use of org.apache.carbondata.core.metadata.schema.table.RelationIdentifier in project carbondata by apache.

the class MVManager method getUpdatedSegmentMap.

/**
 * In case of compaction on mv table,this method will merge the segment list of main table
 * and return updated segment mapping
 *
 * @param mergedLoadName      to find which all segments are merged to new compacted segment
 * @param viewSchema       of mv table
 * @param viewLoadMetadataDetails of mv table
 * @return updated segment map after merging segment list
 */
@SuppressWarnings("unchecked")
public static String getUpdatedSegmentMap(String mergedLoadName, MVSchema viewSchema, LoadMetadataDetails[] viewLoadMetadataDetails) {
    Map<String, List<String>> segmentMapping = new HashMap<>();
    List<RelationIdentifier> relationIdentifiers = viewSchema.getRelatedTables();
    for (RelationIdentifier relationIdentifier : relationIdentifiers) {
        for (LoadMetadataDetails loadMetadataDetail : viewLoadMetadataDetails) {
            if (loadMetadataDetail.getSegmentStatus() == SegmentStatus.COMPACTED) {
                if (mergedLoadName.equalsIgnoreCase(loadMetadataDetail.getMergedLoadName())) {
                    Map segmentMap = new Gson().fromJson(loadMetadataDetail.getExtraInfo(), Map.class);
                    if (segmentMapping.isEmpty()) {
                        segmentMapping.putAll(segmentMap);
                    } else {
                        segmentMapping.get(relationIdentifier.getDatabaseName() + CarbonCommonConstants.POINT + relationIdentifier.getTableName()).addAll((List<String>) segmentMap.get(relationIdentifier.getDatabaseName() + CarbonCommonConstants.POINT + relationIdentifier.getTableName()));
                    }
                }
            }
        }
    }
    Gson gson = new Gson();
    return gson.toJson(segmentMapping);
}

Example 9 with RelationIdentifier

use of org.apache.carbondata.core.metadata.schema.table.RelationIdentifier in project carbondata by apache.

the class MVManager method onTruncate.

/**
 * This method will remove all segments of MV table in case of Insert-Overwrite/Update/Delete
 * operations on main table
 *
 * @param schemas mv schemas
 */
public void onTruncate(List<MVSchema> schemas) throws IOException {
    for (MVSchema schema : schemas) {
        if (!schema.isRefreshOnManual()) {
            setStatus(schema.identifier, MVStatus.DISABLED);
        }
        RelationIdentifier relationIdentifier = schema.getIdentifier();
        SegmentStatusManager segmentStatusManager = new SegmentStatusManager(AbsoluteTableIdentifier.from(relationIdentifier.getTablePath(), relationIdentifier.getDatabaseName(), relationIdentifier.getTableName()));
        ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
        try {
            if (carbonLock.lockWithRetries()) {
                LOGGER.info("Acquired lock for table" + relationIdentifier.getDatabaseName() + "." + relationIdentifier.getTableName() + " for table status update");
                String metaDataPath = CarbonTablePath.getMetadataPath(relationIdentifier.getTablePath());
                LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager.readLoadMetadata(metaDataPath);
                for (LoadMetadataDetails entry : loadMetadataDetails) {
                    entry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
                }
                SegmentStatusManager.writeLoadDetailsIntoFile(CarbonTablePath.getTableStatusFilePath(relationIdentifier.getTablePath()), loadMetadataDetails);
            } else {
                LOGGER.error("Not able to acquire the lock for Table status update for table " + relationIdentifier.getDatabaseName() + "." + relationIdentifier.getTableName());
            }
        } finally {
            if (carbonLock.unlock()) {
                LOGGER.info("Table unlocked successfully after table status update" + relationIdentifier.getDatabaseName() + "." + relationIdentifier.getTableName());
            } else {
                LOGGER.error("Unable to unlock Table lock for table" + relationIdentifier.getDatabaseName() + "." + relationIdentifier.getTableName() + " during table status update");
            }
        }
    }
}

Example 10 with RelationIdentifier

use of org.apache.carbondata.core.metadata.schema.table.RelationIdentifier in project carbondata by apache.

the class DataMapStoreManager method getAllDataMapSchemas.

/**
 * It gives all datamap schemas.
 *
 * @return
 */
public List<DataMapSchema> getAllDataMapSchemas(CarbonTable carbonTable) {
    // TODO cache all schemas and update only when datamap status file updates
    List<DataMapSchema> dataMapSchemas = getAllDataMapSchemas();
    List<DataMapSchema> dataMaps = new ArrayList<>();
    if (dataMapSchemas != null) {
        for (DataMapSchema dataMapSchema : dataMapSchemas) {
            RelationIdentifier identifier = dataMapSchema.getParentTables().get(0);
            if (dataMapSchema.isIndexDataMap() && identifier.getTableName().equals(carbonTable.getTableName()) && identifier.getDatabaseName().equals(carbonTable.getDatabaseName())) {
                dataMaps.add(dataMapSchema);
            }
        }
    }
    return dataMaps;
}