Examples with CarbonTable org.apache.carbondata.core.metadata.schema.table.CarbonTable used on opensource projects

Example 71 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class CarbonTableReader method parseCarbonMetadata.

/**
 * Read the metadata of the given table and cache it in this.carbonCache (CarbonTableReader cache).
 *
 * @param table name of the given table.
 * @return the CarbonTable instance which contains all the needed metadata for a table.
 */
private CarbonTable parseCarbonMetadata(SchemaTableName table) {
    CarbonTable result = null;
    try {
        CarbonTableCacheModel cache = carbonCache.get().get(table);
        if (cache == null) {
            cache = new CarbonTableCacheModel();
        }
        if (cache.isValid())
            return cache.carbonTable;
        // If table is not previously cached, then:
        // Step 1: get store path of the table and cache it.
        // create table identifier. the table id is randomly generated.
        CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier(table.getSchemaName(), table.getTableName(), UUID.randomUUID().toString());
        String storePath = config.getStorePath();
        String tablePath = storePath + "/" + carbonTableIdentifier.getDatabaseName() + "/" + carbonTableIdentifier.getTableName();
        // Step 2: read the metadata (tableInfo) of the table.
        ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {

            // TBase is used to read and write thrift objects.
            // TableInfo is a kind of TBase used to read and write table information.
            // TableInfo is generated by thrift, see schema.thrift under format/src/main/thrift for details.
            public TBase create() {
                return new org.apache.carbondata.format.TableInfo();
            }
        };
        ThriftReader thriftReader = new ThriftReader(CarbonTablePath.getSchemaFilePath(tablePath), createTBase);
        thriftReader.open();
        org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
        thriftReader.close();
        // Step 3: convert format level TableInfo to code level TableInfo
        SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
        // wrapperTableInfo is the code level information of a table in carbondata core, different from the Thrift TableInfo.
        TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo, table.getSchemaName(), table.getTableName(), tablePath);
        // Step 4: Load metadata info into CarbonMetadata
        CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
        cache.carbonTable = CarbonMetadata.getInstance().getCarbonTable(table.getSchemaName(), table.getTableName());
        // cache the table
        carbonCache.get().put(table, cache);
        result = cache.carbonTable;
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
    return result;
}

Example 72 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class IndexStoreManager method clearIndex.

/**
 * Clear the index/indexes of a table from memory
 *
 * @param identifier Table identifier
 */
public void clearIndex(AbsoluteTableIdentifier identifier) {
    CarbonTable carbonTable = getCarbonTable(identifier);
    boolean launchJob = false;
    try {
        // launchJob will be true if either the table has a CGIndex or index server is enabled for
        // the specified table.
        launchJob = hasCGIndex(carbonTable) || CarbonProperties.getInstance().isDistributedPruningEnabled(identifier.getDatabaseName(), identifier.getTableName());
    } catch (IOException e) {
        LOGGER.warn("Unable to launch job to clear indexes.", e);
    }
    clearIndexCache(identifier, launchJob);
}

Example 73 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class IndexStoreManager method clearIndexCache.

/**
 * Clear the index/indexes of a table from memory
 *
 * @param identifier Table identifier
 */
public void clearIndexCache(AbsoluteTableIdentifier identifier, boolean clearInAllWorkers) {
    String tableId = identifier.getCarbonTableIdentifier().getTableId();
    if (clearInAllWorkers) {
        // carbon table need to lookup only if launch job is set.
        CarbonTable carbonTable = getCarbonTable(identifier);
        if (null != carbonTable) {
            String jobClassName;
            if (CarbonProperties.getInstance().isDistributedPruningEnabled(identifier.getDatabaseName(), identifier.getTableName())) {
                jobClassName = IndexUtil.DISTRIBUTED_JOB_NAME;
            } else {
                jobClassName = IndexUtil.EMBEDDED_JOB_NAME;
            }
            try {
                IndexUtil.executeClearIndexJob(carbonTable, jobClassName);
            } catch (IOException e) {
                LOGGER.error("clear index job failed", e);
            // ignoring the exception
            }
        }
    } else {
        // remove carbon table from meta cache if launchJob is false as this would be called in
        // executor side.
        CarbonMetadata.getInstance().removeTable(identifier.getDatabaseName(), identifier.getTableName());
    }
    List<TableIndex> tableIndices = allIndexes.get(identifier.getCarbonTableIdentifier().getTableId());
    if (tableIndices == null) {
        String keyUsingTablePath = getKeyUsingTablePath(identifier.getTablePath());
        if (keyUsingTablePath != null) {
            tableId = keyUsingTablePath;
        }
    }
    segmentRefreshMap.remove(tableId);
    clearIndex(tableId);
    allIndexes.remove(tableId);
    tablePathMap.remove(tableId);
}

Example 74 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class StoreCreator method createTableAndLoadModel.

public CarbonLoadModel createTableAndLoadModel(boolean deleteOldStore) throws Exception {
    if (deleteOldStore) {
        File storeDir = new File(storePath);
        CarbonUtil.deleteFoldersAndFiles(storeDir);
    }
    CarbonTable table = createTable(absoluteTableIdentifier);
    return buildCarbonLoadModel(table, csvPath, absoluteTableIdentifier);
}

Example 75 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class CarbonOutputCommitter method commitJob.

/**
 * Update the tablestatus as success after job is success
 */
@Override
public void commitJob(JobContext context) throws IOException {
    // comma separated partitions
    String partitionPath = context.getConfiguration().get("carbon.output.partitions.name");
    long t1 = System.currentTimeMillis();
    try {
        super.commitJob(context);
    } catch (IOException e) {
        // ignore, in case of concurrent load it try to remove temporary folders by other load may
        // cause file not found exception. This will not impact carbon load,
        LOGGER.warn(e.getMessage());
    }
    LOGGER.info("$$$ Time taken for the super.commitJob in ms: " + (System.currentTimeMillis() - t1));
    boolean overwriteSet = CarbonTableOutputFormat.isOverwriteSet(context.getConfiguration());
    CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
    List<PartitionSpec> currentPartitionsOfTable = (List<PartitionSpec>) ObjectSerializationUtil.convertStringToObject(context.getConfiguration().get("carbon.currentpartition"));
    if (loadModel.getCarbonDataLoadSchema().getCarbonTable().isHivePartitionTable()) {
        try {
            commitJobForPartition(context, overwriteSet, loadModel, partitionPath, currentPartitionsOfTable);
        } catch (Exception e) {
            CarbonLoaderUtil.updateTableStatusForFailure(loadModel);
            LOGGER.error("commit job failed", e);
            throw new IOException(e.getMessage());
        } finally {
            if (segmentLock != null) {
                segmentLock.unlock();
            }
        }
        return;
    }
    LoadMetadataDetails newMetaEntry = loadModel.getCurrentLoadMetadataDetail();
    String readPath = CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath()) + CarbonCommonConstants.FILE_SEPARATOR + loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp() + ".tmp";
    // Merge all partition files into a single file.
    String segmentFileName = SegmentFileStore.genSegmentFileName(loadModel.getSegmentId(), String.valueOf(loadModel.getFactTimeStamp()));
    SegmentFileStore.SegmentFile segmentFile = SegmentFileStore.mergeSegmentFiles(readPath, segmentFileName, CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath()));
    if (segmentFile != null) {
        if (null == newMetaEntry) {
            throw new RuntimeException("Internal Error");
        }
        // Move all files from temp directory of each segment to partition directory
        SegmentFileStore.moveFromTempFolder(segmentFile, loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp() + ".tmp", loadModel.getTablePath());
        newMetaEntry.setSegmentFile(segmentFileName + CarbonTablePath.SEGMENT_EXT);
    }
    OperationContext operationContext = (OperationContext) getOperationContext();
    CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    String uuid = "";
    newMetaEntry.setSegmentFile(segmentFileName + CarbonTablePath.SEGMENT_EXT);
    CarbonLoaderUtil.populateNewLoadMetaEntry(newMetaEntry, SegmentStatus.SUCCESS, loadModel.getFactTimeStamp(), true);
    long segmentSize = CarbonLoaderUtil.addDataIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(), carbonTable);
    if (segmentSize > 0 || overwriteSet) {
        if (operationContext != null) {
            operationContext.setProperty(CarbonCommonConstants.CURRENT_SEGMENTFILE, newMetaEntry.getSegmentFile());
            LoadEvents.LoadTablePreStatusUpdateEvent event = new LoadEvents.LoadTablePreStatusUpdateEvent(carbonTable.getCarbonTableIdentifier(), loadModel);
            try {
                OperationListenerBus.getInstance().fireEvent(event, operationContext);
            } catch (Exception e) {
                throw new IOException(e);
            }
        }
        // After merging index, update newMetaEntry with updated merge index size
        boolean isMergeIndexEnabled = Boolean.parseBoolean(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT, CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT));
        if (isMergeIndexEnabled) {
            CarbonLoaderUtil.addIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(), carbonTable);
        }
        String uniqueId = null;
        if (overwriteSet) {
            if (!loadModel.isCarbonTransactionalTable()) {
                CarbonLoaderUtil.deleteNonTransactionalTableForInsertOverwrite(loadModel);
            } else {
                if (segmentSize == 0) {
                    newMetaEntry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
                }
                List<String> partitionList = (List<String>) ObjectSerializationUtil.convertStringToObject(partitionPath);
                uniqueId = overwritePartitions(loadModel, newMetaEntry, uuid, partitionList, currentPartitionsOfTable);
            }
        } else {
            CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, false, uuid, false);
        }
        commitJobFinal(context, loadModel, operationContext, carbonTable, uniqueId);
    } else {
        CarbonLoaderUtil.updateTableStatusForFailure(loadModel);
    }
    if (segmentLock != null) {
        segmentLock.unlock();
    }
}