Examples with TableSchema org.apache.carbondata.core.metadata.schema.table.TableSchema used on opensource projects

Example 6 with TableSchema

use of org.apache.carbondata.core.metadata.schema.table.TableSchema in project carbondata by apache.

the class ThriftWrapperSchemaConverterImplTest method testFromExternalToWrapperTableSchema.

@Test
public void testFromExternalToWrapperTableSchema() {
    String tableId = "1";
    String tableName = "tableName";
    TableSchema actualResult = thriftWrapperSchemaConverter.fromExternalToWrapperTableSchema(tabSchema, "tableName");
    assertEquals(tableId, actualResult.getTableId());
    assertEquals(tableName, actualResult.getTableName());
}

Example 7 with TableSchema

use of org.apache.carbondata.core.metadata.schema.table.TableSchema in project carbondata by apache.

the class ThriftWrapperSchemaConverterImplTest method testFromWrapperToExternalTableSchema.

@Test
public void testFromWrapperToExternalTableSchema() {
    TableSchema wrapperTableSchema = new TableSchema();
    new MockUp<ColumnSchema>() {

        @Mock
        public List<Encoding> getEncodingList() {
            return encodings;
        }

        @Mock
        public int getSchemaOrdinal() {
            return 1;
        }

        @Mock
        public DataType getDataType() {
            return DataType.BOOLEAN;
        }

        @Mock
        public String getColumnName() {
            return "columnName";
        }

        @Mock
        public String getColumnUniqueId() {
            return "1";
        }

        @Mock
        public boolean isColumnar() {
            return true;
        }

        @Mock
        public boolean isDimensionColumn() {
            return true;
        }

        @Mock
        public int getColumnGroupId() {
            return 1;
        }

        @Mock
        public int getScale() {
            return 1;
        }

        @Mock
        public int getPrecision() {
            return 1;
        }

        @Mock
        public int getNumberOfChild() {
            return 1;
        }

        @Mock
        public byte[] getDefaultValue() {
            return new byte[] { 1, 2 };
        }

        @Mock
        public Map<String, String> getColumnProperties() {
            return columnPropertyMap;
        }

        @Mock
        public boolean isInvisible() {
            return true;
        }

        @Mock
        public String getColumnReferenceId() {
            return "1";
        }
    };
    new MockUp<SchemaEvolutionEntry>() {

        @Mock
        public List<ColumnSchema> getAdded() {
            return columnSchemas;
        }

        @Mock
        public List<ColumnSchema> getRemoved() {
            return columnSchemas;
        }
    };
    final SchemaEvolution schemaEvolution = new SchemaEvolution();
    final Map mapTableProperties = new HashMap<String, String>();
    new MockUp<TableSchema>() {

        @Mock
        public List<ColumnSchema> getListOfColumns() {
            return columnSchemas;
        }

        @Mock
        public SchemaEvolution getSchemaEvalution() {
            return schemaEvolution;
        }

        @Mock
        public String getTableId() {
            return "tableId";
        }

        @Mock
        public Map<String, String> getTableProperties() {
            return mapTableProperties;
        }
    };
    org.apache.carbondata.format.TableSchema expectedResult = new org.apache.carbondata.format.TableSchema("tableId", thriftColumnSchemas, schemaEvol);
    org.apache.carbondata.format.TableSchema actualResult = thriftWrapperSchemaConverter.fromWrapperToExternalTableSchema(wrapperTableSchema);
    assertEquals(expectedResult, actualResult);
}

Example 8 with TableSchema

use of org.apache.carbondata.core.metadata.schema.table.TableSchema in project carbondata by apache.

the class ThriftWrapperSchemaConverterImplTest method testFromWrapperToExternalTableInfo.

@Test
public void testFromWrapperToExternalTableInfo() {
    TableInfo wrapperTableInfo = new TableInfo();
    String dbName = "dbName";
    String tableName = "TableName";
    final TableSchema wrapperTableSchema = new TableSchema();
    final List<TableSchema> tableSchemas = new ArrayList<>();
    tableSchemas.add(wrapperTableSchema);
    new MockUp<SchemaEvolutionEntry>() {

        @Mock
        public List<ColumnSchema> getAdded() {
            return columnSchemas;
        }

        @Mock
        public List<ColumnSchema> getRemoved() {
            return columnSchemas;
        }
    };
    new MockUp<ColumnSchema>() {

        @Mock
        public List<Encoding> getEncodingList() {
            return encodings;
        }

        @Mock
        public int getSchemaOrdinal() {
            return 1;
        }

        @Mock
        public DataType getDataType() {
            return DataType.BOOLEAN;
        }

        @Mock
        public String getColumnName() {
            return "columnName";
        }

        @Mock
        public String getColumnUniqueId() {
            return "1";
        }

        @Mock
        public boolean isColumnar() {
            return true;
        }

        @Mock
        public boolean isDimensionColumn() {
            return true;
        }

        @Mock
        public int getColumnGroupId() {
            return 1;
        }

        @Mock
        public int getScale() {
            return 1;
        }

        @Mock
        public int getPrecision() {
            return 1;
        }

        @Mock
        public int getNumberOfChild() {
            return 1;
        }

        @Mock
        public byte[] getDefaultValue() {
            return new byte[] { 1, 2 };
        }

        @Mock
        public Map<String, String> getColumnProperties() {
            return columnPropertyMap;
        }

        @Mock
        public boolean isInvisible() {
            return true;
        }

        @Mock
        public String getColumnReferenceId() {
            return "1";
        }
    };
    new MockUp<TableInfo>() {

        @Mock
        public TableSchema getFactTable() {
            return wrapperTableSchema;
        }

        @Mock
        public List<TableSchema> getAggregateTableList() {
            return tableSchemas;
        }
    };
    new MockUp<TableSchema>() {

        @Mock
        public List<ColumnSchema> getListOfColumns() {
            return columnSchemas;
        }

        final SchemaEvolution schemaEvolution = new SchemaEvolution();

        final Map mapTableProperties = new HashMap<String, String>();

        @Mock
        public SchemaEvolution getSchemaEvalution() {
            return schemaEvolution;
        }

        @Mock
        public String getTableId() {
            return "tableId";
        }

        @Mock
        public Map<String, String> getTableProperties() {
            return mapTableProperties;
        }
    };
    org.apache.carbondata.format.TableSchema thriftFactTable = new org.apache.carbondata.format.TableSchema("tableId", thriftColumnSchemas, schemaEvol);
    List<org.apache.carbondata.format.TableSchema> thriftAggTables = new ArrayList<>();
    thriftAggTables.add(thriftFactTable);
    org.apache.carbondata.format.TableInfo actualResult = thriftWrapperSchemaConverter.fromWrapperToExternalTableInfo(wrapperTableInfo, dbName, tableName);
    org.apache.carbondata.format.TableInfo expectedResult = new org.apache.carbondata.format.TableInfo(thriftFactTable, thriftAggTables);
    assertEquals(expectedResult, actualResult);
}

Example 9 with TableSchema

use of org.apache.carbondata.core.metadata.schema.table.TableSchema in project carbondata by apache.

the class StoreCreator method createTable.

private static CarbonTable createTable() throws IOException {
    TableInfo tableInfo = new TableInfo();
    tableInfo.setStorePath(absoluteTableIdentifier.getStorePath());
    tableInfo.setDatabaseName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName());
    TableSchema tableSchema = new TableSchema();
    tableSchema.setTableName(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
    List<ColumnSchema> columnSchemas = new ArrayList<ColumnSchema>();
    ArrayList<Encoding> encodings = new ArrayList<>();
    encodings.add(Encoding.DICTIONARY);
    ColumnSchema id = new ColumnSchema();
    id.setColumnName("ID");
    id.setColumnar(true);
    id.setDataType(DataType.INT);
    id.setEncodingList(encodings);
    id.setColumnUniqueId(UUID.randomUUID().toString());
    id.setDimensionColumn(true);
    id.setColumnGroup(1);
    columnSchemas.add(id);
    ColumnSchema date = new ColumnSchema();
    date.setColumnName("date");
    date.setColumnar(true);
    date.setDataType(DataType.STRING);
    date.setEncodingList(encodings);
    date.setColumnUniqueId(UUID.randomUUID().toString());
    date.setDimensionColumn(true);
    date.setColumnGroup(2);
    date.setSortColumn(true);
    columnSchemas.add(date);
    ColumnSchema country = new ColumnSchema();
    country.setColumnName("country");
    country.setColumnar(true);
    country.setDataType(DataType.STRING);
    country.setEncodingList(encodings);
    country.setColumnUniqueId(UUID.randomUUID().toString());
    country.setDimensionColumn(true);
    country.setColumnGroup(3);
    country.setSortColumn(true);
    columnSchemas.add(country);
    ColumnSchema name = new ColumnSchema();
    name.setColumnName("name");
    name.setColumnar(true);
    name.setDataType(DataType.STRING);
    name.setEncodingList(encodings);
    name.setColumnUniqueId(UUID.randomUUID().toString());
    name.setDimensionColumn(true);
    name.setColumnGroup(4);
    name.setSortColumn(true);
    columnSchemas.add(name);
    ColumnSchema phonetype = new ColumnSchema();
    phonetype.setColumnName("phonetype");
    phonetype.setColumnar(true);
    phonetype.setDataType(DataType.STRING);
    phonetype.setEncodingList(encodings);
    phonetype.setColumnUniqueId(UUID.randomUUID().toString());
    phonetype.setDimensionColumn(true);
    phonetype.setColumnGroup(5);
    phonetype.setSortColumn(true);
    columnSchemas.add(phonetype);
    ColumnSchema serialname = new ColumnSchema();
    serialname.setColumnName("serialname");
    serialname.setColumnar(true);
    serialname.setDataType(DataType.STRING);
    serialname.setEncodingList(encodings);
    serialname.setColumnUniqueId(UUID.randomUUID().toString());
    serialname.setDimensionColumn(true);
    serialname.setColumnGroup(6);
    serialname.setSortColumn(true);
    columnSchemas.add(serialname);
    ColumnSchema salary = new ColumnSchema();
    salary.setColumnName("salary");
    salary.setColumnar(true);
    salary.setDataType(DataType.INT);
    salary.setEncodingList(new ArrayList<Encoding>());
    salary.setColumnUniqueId(UUID.randomUUID().toString());
    salary.setDimensionColumn(false);
    salary.setColumnGroup(7);
    columnSchemas.add(salary);
    tableSchema.setListOfColumns(columnSchemas);
    SchemaEvolution schemaEvol = new SchemaEvolution();
    schemaEvol.setSchemaEvolutionEntryList(new ArrayList<SchemaEvolutionEntry>());
    tableSchema.setSchemaEvalution(schemaEvol);
    tableSchema.setTableId(UUID.randomUUID().toString());
    tableInfo.setTableUniqueName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName() + "_" + absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
    tableInfo.setLastUpdatedTime(System.currentTimeMillis());
    tableInfo.setFactTable(tableSchema);
    tableInfo.setAggregateTableList(new ArrayList<TableSchema>());
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String schemaFilePath = carbonTablePath.getSchemaFilePath();
    String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath);
    tableInfo.setMetaDataFilepath(schemaMetadataPath);
    CarbonMetadata.getInstance().loadTableMetadata(tableInfo);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    org.apache.carbondata.format.TableInfo thriftTableInfo = schemaConverter.fromWrapperToExternalTableInfo(tableInfo, tableInfo.getDatabaseName(), tableInfo.getFactTable().getTableName());
    org.apache.carbondata.format.SchemaEvolutionEntry schemaEvolutionEntry = new org.apache.carbondata.format.SchemaEvolutionEntry(tableInfo.getLastUpdatedTime());
    thriftTableInfo.getFact_table().getSchema_evolution().getSchema_evolution_history().add(schemaEvolutionEntry);
    FileFactory.FileType fileType = FileFactory.getFileType(schemaMetadataPath);
    if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
        FileFactory.mkdirs(schemaMetadataPath, fileType);
    }
    ThriftWriter thriftWriter = new ThriftWriter(schemaFilePath, false);
    thriftWriter.open();
    thriftWriter.write(thriftTableInfo);
    thriftWriter.close();
    return CarbonMetadata.getInstance().getCarbonTable(tableInfo.getTableUniqueName());
}

Example 10 with TableSchema

use of org.apache.carbondata.core.metadata.schema.table.TableSchema in project carbondata by apache.

the class IncrementalColumnDictionaryGeneratorTest method writeDictionaryData.

@Test
public void writeDictionaryData() throws Exception {
    //Create required column schema
    ColumnSchema columnSchema = new ColumnSchema();
    columnSchema.setColumnName("empNameCol");
    columnSchema.setColumnUniqueId("empNameCol");
    CarbonDimension carbonDimension = new CarbonDimension(columnSchema, 0, 0, 0, 0, 0);
    // Create the generator and add the keys to dictionary
    IncrementalColumnDictionaryGenerator generator = new IncrementalColumnDictionaryGenerator(carbonDimension, 10);
    // Create a table schema for saving the dictionary
    TableSchema tableSchema = new TableSchema();
    tableSchema.setTableName("TestTable");
    tableSchema.setListOfColumns(Arrays.asList(columnSchema));
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    TableInfo tableInfo = new TableInfo();
    tableInfo.setFactTable(tableSchema);
    tableInfo.setTableUniqueName("TestTable");
    tableInfo.setDatabaseName("test");
    String storePath = System.getProperty("java.io.tmpdir") + "/tmp";
    File dictPath = new File(storePath + "/test/TestTable/Metadata/");
    System.out.print(dictPath.mkdirs());
    tableInfo.setStorePath(storePath);
    CarbonTable carbonTable = new CarbonTable();
    carbonTable.loadCarbonTable(tableInfo);
    // Add the table to metadata
    metadata.addCarbonTable(carbonTable);
    /// Write the dictionary and verify whether its written successfully
    generator.writeDictionaryData("TestTable");
    File dictionaryFile = new File(dictPath, "empNameCol.dict");
    System.out.println(dictionaryFile.getCanonicalPath());
    assertTrue(dictionaryFile.exists());
    dictionaryFile.delete();
    // cleanup created files
    metadata.removeTable(carbonTable.getTableUniqueName());
    cleanUpDirectory(new File(storePath));
}