Examples with IndexMetadata org.apache.carbondata.core.metadata.schema.indextable.IndexMetadata used on opensource projects

Example 1 with IndexMetadata

use of org.apache.carbondata.core.metadata.schema.indextable.IndexMetadata in project carbondata by apache.

the class CarbonTableReader method refreshIndexInfo.

private void refreshIndexInfo(CarbonTable carbonTable, Configuration config) {
    Map<String, Map<String, Map<String, String>>> indexTableMap = new ConcurrentHashMap<>();
    String indexInfo = config.get("indexInfo", IndexTableInfo.toGson(new IndexTableInfo[0]));
    String parentTableName = config.get("parentTableName", "");
    String parentTableId = config.get("parentTableId", "");
    String parentTablePath = config.get("parentTablePath", "");
    boolean isIndexTable = Boolean.getBoolean(config.get("isIndexTable", "false"));
    IndexTableInfo[] indexTableInfos = IndexTableInfo.fromGson(indexInfo);
    for (IndexTableInfo indexTableInfo : indexTableInfos) {
        Map<String, String> indexProperties = indexTableInfo.getIndexProperties();
        String indexProvider;
        if (indexProperties != null) {
            indexProvider = indexProperties.get(CarbonCommonConstants.INDEX_PROVIDER);
        } else {
            // in case if SI table has been created before the change CARBONDATA-3765,
            // indexProperties variable will not be present. On direct upgrade of SI store,
            // indexProperties will be null, in that case, create indexProperties from indexCols
            // For details, refer
            // {@link org.apache.spark.sql.secondaryindex.hive.CarbonInternalMetastore#refreshIndexInfo}
            indexProperties = new HashMap<>();
            indexProperties.put(CarbonCommonConstants.INDEX_COLUMNS, String.join(",", indexTableInfo.getIndexCols()));
            indexProvider = IndexType.SI.getIndexProviderName();
            indexProperties.put(CarbonCommonConstants.INDEX_PROVIDER, indexProvider);
            indexProperties.put(CarbonCommonConstants.INDEX_STATUS, IndexStatus.DISABLED.name());
        }
        if (indexTableMap.get(indexProvider) == null) {
            Map<String, Map<String, String>> indexTableInfoMap = new HashMap<>();
            indexTableInfoMap.put(indexTableInfo.getTableName(), indexProperties);
            indexTableMap.put(indexProvider, indexTableInfoMap);
        } else {
            indexTableMap.get(indexProvider).put(indexTableInfo.getTableName(), indexProperties);
        }
    }
    IndexMetadata indexMetadata = new IndexMetadata(indexTableMap, parentTableName, isIndexTable, parentTablePath, parentTableId);
    try {
        carbonTable.getTableInfo().getFactTable().getTableProperties().put(carbonTable.getCarbonTableIdentifier().getTableId(), indexMetadata.serialize());
    } catch (IOException e) {
        LOGGER.error("Error serializing index metadata", e);
    }
}

Example 2 with IndexMetadata

use of org.apache.carbondata.core.metadata.schema.indextable.IndexMetadata in project carbondata by apache.

the class IndexTableUtil method addIndexInfoToParentTable.

public static void addIndexInfoToParentTable(CarbonTable parentTable, String indexProviderName, String indexName, Map<String, String> indexProperties) throws IOException {
    // set index information in parent table
    IndexMetadata parentIndexMetadata;
    Map<String, String> tableProperties = parentTable.getTableInfo().getFactTable().getTableProperties();
    if (tableProperties.get(parentTable.getCarbonTableIdentifier().getTableId()) != null) {
        parentIndexMetadata = parentTable.getIndexMetadata();
    } else {
        parentIndexMetadata = new IndexMetadata(false);
    }
    parentIndexMetadata.addIndexTableInfo(indexProviderName, indexName, indexProperties);
    tableProperties.put(parentTable.getCarbonTableIdentifier().getTableId(), parentIndexMetadata.serialize());
}

Example 3 with IndexMetadata

use of org.apache.carbondata.core.metadata.schema.indextable.IndexMetadata in project carbondata by apache.

the class CarbonCostBasedOptimizer method getSecondaryIndexes.

private static Map<String, List<String>> getSecondaryIndexes(TableInfo tableInfo) {
    Map<String, List<String>> indexes = new HashMap<>();
    String indexMeta = tableInfo.getFactTable().getTableProperties().get(tableInfo.getFactTable().getTableId());
    IndexMetadata indexMetadata = null;
    if (null != indexMeta) {
        try {
            indexMetadata = IndexMetadata.deserialize(indexMeta);
        } catch (IOException e) {
            LOGGER.error("Error deserializing index metadata", e);
        }
    }
    if (indexMetadata != null) {
        String provider = IndexType.SI.getIndexProviderName();
        if (!indexMetadata.isIndexTable() && (null != indexMetadata.getIndexesMap().get(provider))) {
            for (Map.Entry<String, Map<String, String>> entry : indexMetadata.getIndexesMap().get(provider).entrySet()) {
                Map<String, String> indexProperties = entry.getValue();
                indexes.put(entry.getKey(), Arrays.asList(indexProperties.get(CarbonCommonConstants.INDEX_COLUMNS).split(",")));
            }
        }
    }
    return indexes;
}

Example 4 with IndexMetadata

use of org.apache.carbondata.core.metadata.schema.indextable.IndexMetadata in project carbondata by apache.

the class CarbonIndexFileMergeWriter method writeMergeIndexFileBasedOnSegmentFile.

public String writeMergeIndexFileBasedOnSegmentFile(String segmentId, List<String> indexFileNamesTobeAdded, boolean isOldStoreIndexFilesPresent, SegmentFileStore segmentFileStore, CarbonFile[] indexFiles, String uuid, String partitionPath) throws IOException {
    SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
    String newSegmentFileName = null;
    // in case of partition table, merge index file to be created for each partition
    if (null != partitionPath) {
        for (CarbonFile indexFile : indexFiles) {
            fileStore.readIndexFile(indexFile);
        }
    } else {
        fileStore.readAllIIndexOfSegment(segmentFileStore.getSegmentFile(), segmentFileStore.getTablePath(), SegmentStatus.SUCCESS, true);
    }
    Map<String, Map<String, byte[]>> indexLocationMap = groupIndexesBySegment(fileStore.getCarbonIndexMapWithFullPath());
    List<PartitionSpec> partitionSpecs = SegmentFileStore.getPartitionSpecs(segmentId, table.getTablePath(), SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(table.getTablePath())));
    List<String> mergeIndexFiles = new ArrayList<>();
    for (Map.Entry<String, Map<String, byte[]>> entry : indexLocationMap.entrySet()) {
        String mergeIndexFile = writeMergeIndexFile(indexFileNamesTobeAdded, entry.getKey(), entry.getValue(), segmentId, uuid);
        for (Map.Entry<String, SegmentFileStore.FolderDetails> segment : segmentFileStore.getLocationMap().entrySet()) {
            String location = segment.getKey();
            if (segment.getValue().isRelative()) {
                location = segmentFileStore.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + location;
            }
            if (FileFactory.getCarbonFile(entry.getKey()).equals(FileFactory.getCarbonFile(location))) {
                segment.getValue().setMergeFileName(mergeIndexFile);
                mergeIndexFiles.add(entry.getKey() + CarbonCommonConstants.FILE_SEPARATOR + mergeIndexFile);
                segment.getValue().setFiles(new HashSet<>());
                break;
            }
        }
        if (table.isHivePartitionTable()) {
            for (PartitionSpec partitionSpec : partitionSpecs) {
                if (partitionSpec.getLocation().toString().equals(partitionPath)) {
                    try {
                        SegmentFileStore.writeSegmentFileForPartitionTable(table.getTablePath(), mergeIndexFile, partitionPath, segmentId + CarbonCommonConstants.UNDERSCORE + uuid + "", partitionSpec.getPartitions(), true);
                    } catch (Exception ex) {
                        // delete merge index file if created,
                        // keep only index files as segment file writing is failed
                        FileFactory.getCarbonFile(mergeIndexFile).delete();
                        LOGGER.error("unable to write segment file during merge index writing: " + ex.getMessage());
                        throw ex;
                    }
                }
            }
        }
    }
    if (table.isIndexTable()) {
        // To maintain same segment file name mapping between parent and SI table.
        IndexMetadata indexMetadata = table.getIndexMetadata();
        LoadMetadataDetails[] loadDetails = SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(indexMetadata.getParentTablePath()));
        LoadMetadataDetails loadMetaDetail = Arrays.stream(loadDetails).filter(loadDetail -> loadDetail.getLoadName().equals(segmentId)).findFirst().get();
        newSegmentFileName = loadMetaDetail.getSegmentFile();
    } else {
        // Instead of modifying existing segment file, generate uuid to write into new segment file.
        uuid = String.valueOf(System.currentTimeMillis());
        newSegmentFileName = SegmentFileStore.genSegmentFileName(segmentId, uuid) + CarbonTablePath.SEGMENT_EXT;
    }
    String path = CarbonTablePath.getSegmentFilesLocation(table.getTablePath()) + CarbonCommonConstants.FILE_SEPARATOR + newSegmentFileName;
    if (!table.isHivePartitionTable()) {
        try {
            SegmentFileStore.writeSegmentFile(segmentFileStore.getSegmentFile(), path);
        } catch (Exception ex) {
            // keep only index files as segment file writing is failed
            for (String mergeIndexFile : mergeIndexFiles) {
                FileFactory.getCarbonFile(mergeIndexFile).delete();
            }
            LOGGER.error("unable to write segment file during merge index writing: " + ex.getMessage());
            throw ex;
        }
        boolean status = SegmentFileStore.updateTableStatusFile(table, segmentId, newSegmentFileName, table.getCarbonTableIdentifier().getTableId(), segmentFileStore);
        if (!status) {
            throw new IOException("Table status update with mergeIndex file has failed");
        }
    }
    // do not delete index files immediately to avoid index file not found during concurrent queries
    if (!isOldStoreIndexFilesPresent) {
        for (CarbonFile file : indexFiles) {
            file.delete();
        }
    }
    return uuid;
}