use of co.cask.cdap.logging.write.LogLocation in project cdap by caskdata.
the class CDAPLogAppenderTest method testCDAPLogAppender.
@Test
public void testCDAPLogAppender() throws Exception {
int syncInterval = 1024 * 1024;
CDAPLogAppender cdapLogAppender = new CDAPLogAppender();
cdapLogAppender.setSyncIntervalBytes(syncInterval);
cdapLogAppender.setMaxFileLifetimeMs(TimeUnit.DAYS.toMillis(1));
cdapLogAppender.setMaxFileSizeInBytes(104857600);
cdapLogAppender.setDirPermissions("700");
cdapLogAppender.setFilePermissions("600");
cdapLogAppender.setFileRetentionDurationDays(1);
cdapLogAppender.setLogCleanupIntervalMins(10);
cdapLogAppender.setFileCleanupTransactionTimeout(30);
AppenderContext context = new LocalAppenderContext(injector.getInstance(DatasetFramework.class), injector.getInstance(TransactionSystemClient.class), injector.getInstance(LocationFactory.class), new NoOpMetricsCollectionService());
context.start();
cdapLogAppender.setContext(context);
cdapLogAppender.start();
FileMetaDataReader fileMetaDataReader = injector.getInstance(FileMetaDataReader.class);
LoggingEvent event = new LoggingEvent("co.cask.Test", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), Level.ERROR, "test message", null, null);
Map<String, String> properties = new HashMap<>();
properties.put(NamespaceLoggingContext.TAG_NAMESPACE_ID, "default");
properties.put(ApplicationLoggingContext.TAG_APPLICATION_ID, "testApp");
properties.put(FlowletLoggingContext.TAG_FLOW_ID, "testFlow");
properties.put(FlowletLoggingContext.TAG_FLOWLET_ID, "testFlowlet");
event.setMDCPropertyMap(properties);
cdapLogAppender.doAppend(event);
cdapLogAppender.stop();
context.stop();
try {
List<LogLocation> files = fileMetaDataReader.listFiles(cdapLogAppender.getLoggingPath(properties), 0, Long.MAX_VALUE);
Assert.assertEquals(1, files.size());
LogLocation logLocation = files.get(0);
Assert.assertEquals(LogLocation.VERSION_1, logLocation.getFrameworkVersion());
Assert.assertTrue(logLocation.getLocation().exists());
CloseableIterator<LogEvent> logEventCloseableIterator = logLocation.readLog(Filter.EMPTY_FILTER, 0, Long.MAX_VALUE, Integer.MAX_VALUE);
int logCount = 0;
while (logEventCloseableIterator.hasNext()) {
logCount++;
LogEvent logEvent = logEventCloseableIterator.next();
Assert.assertEquals(event.getMessage(), logEvent.getLoggingEvent().getMessage());
}
logEventCloseableIterator.close();
Assert.assertEquals(1, logCount);
// checking permission
String expectedPermissions = "rw-------";
for (LogLocation file : files) {
Location location = file.getLocation();
Assert.assertEquals(expectedPermissions, location.getPermissions());
}
} catch (Exception e) {
Assert.fail();
}
}
use of co.cask.cdap.logging.write.LogLocation in project cdap by caskdata.
the class FileMetaDataReader method getFilesInRange.
@VisibleForTesting
List<LogLocation> getFilesInRange(List<LogLocation> files, long startTimeInMs) {
// return if its empty
if (files.isEmpty()) {
return files;
}
// sort the list
Collections.sort(files, LOG_LOCATION_COMPARATOR);
// iterate the list from the end
// we continue when the start timestamp of the log file is higher than the startTimeInMs
// when we reach a file where start time is lower than the startTimeInMs we return the list from this index.
// as the files with same startTimeInMs is sorted by creation timestamp,
// we will return the file with most recent creation time - which is the expected behavior.
// if we reach the beginning of the list, we return the entire list.
List<LogLocation> filteredList = new ArrayList<>();
for (LogLocation logLocation : Lists.reverse(files)) {
long eventTimestamp = logLocation.getEventTimeMs();
filteredList.add(0, logLocation);
if (eventTimestamp < startTimeInMs) {
break;
}
}
return filteredList;
}
use of co.cask.cdap.logging.write.LogLocation in project cdap by caskdata.
the class FileLogReader method getLogNext.
@Override
public void getLogNext(final LoggingContext loggingContext, final ReadRange readRange, final int maxEvents, final Filter filter, final Callback callback) {
if (readRange == ReadRange.LATEST) {
getLogPrev(loggingContext, readRange, maxEvents, filter, callback);
return;
}
callback.init();
try {
Filter logFilter = new AndFilter(ImmutableList.of(LoggingContextHelper.createFilter(loggingContext), filter));
long fromTimeMs = readRange.getFromMillis() + 1;
LOG.trace("Using fromTimeMs={}, readRange={}", fromTimeMs, readRange);
List<LogLocation> sortedFilesInRange = fileMetadataReader.listFiles(LoggingContextHelper.getLogPathIdentifier(loggingContext), readRange.getFromMillis(), readRange.getToMillis());
if (sortedFilesInRange.isEmpty()) {
return;
}
for (LogLocation file : sortedFilesInRange) {
LOG.trace("Reading file {}", file);
file.readLog(logFilter, fromTimeMs, Long.MAX_VALUE, maxEvents - callback.getCount(), callback);
if (callback.getCount() >= maxEvents) {
break;
}
}
} catch (Throwable e) {
LOG.error("Got exception: ", e);
throw Throwables.propagate(e);
}
}
use of co.cask.cdap.logging.write.LogLocation in project cdap by caskdata.
the class DistributedLogFrameworkTest method testFramework.
@Test
public void testFramework() throws Exception {
DistributedLogFramework framework = injector.getInstance(DistributedLogFramework.class);
CConfiguration cConf = injector.getInstance(CConfiguration.class);
framework.startAndWait();
// Send some logs to Kafka.
LoggingContext context = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, "test");
// Make sure all events get flushed in the same batch
long eventTimeBase = System.currentTimeMillis() + cConf.getInt(Constants.Logging.PIPELINE_EVENT_DELAY_MS);
final int msgCount = 50;
for (int i = 0; i < msgCount; i++) {
// Publish logs in descending timestamp order
publishLog(cConf.get(Constants.Logging.KAFKA_TOPIC), context, ImmutableList.of(createLoggingEvent("co.cask.test." + i, Level.INFO, "Testing " + i, eventTimeBase - i)));
}
// Read the logs back. They should be sorted by timestamp order.
final FileMetaDataReader metaDataReader = injector.getInstance(FileMetaDataReader.class);
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
List<LogLocation> locations = metaDataReader.listFiles(new LogPathIdentifier(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, "test"), 0, Long.MAX_VALUE);
if (locations.size() != 1) {
return false;
}
LogLocation location = locations.get(0);
int i = 0;
try {
try (CloseableIterator<LogEvent> iter = location.readLog(Filter.EMPTY_FILTER, 0, Long.MAX_VALUE, msgCount)) {
while (iter.hasNext()) {
String expectedMsg = "Testing " + (msgCount - i - 1);
LogEvent event = iter.next();
if (!expectedMsg.equals(event.getLoggingEvent().getMessage())) {
return false;
}
i++;
}
return i == msgCount;
}
} catch (Exception e) {
// and the time when actual content are flushed to the file
return false;
}
}
}, 10, TimeUnit.SECONDS, msgCount, TimeUnit.MILLISECONDS);
framework.stopAndWait();
// Check the checkpoint is persisted correctly. Since all messages are processed,
// the checkpoint should be the same as the message count.
Checkpoint checkpoint = injector.getInstance(CheckpointManagerFactory.class).create(cConf.get(Constants.Logging.KAFKA_TOPIC), Bytes.toBytes(100)).getCheckpoint(0);
Assert.assertEquals(msgCount, checkpoint.getNextOffset());
}
use of co.cask.cdap.logging.write.LogLocation in project cdap by caskdata.
the class FileMetadataTest method testFileMetadataReadWriteAcrossFormats.
@Test
public void testFileMetadataReadWriteAcrossFormats() throws Exception {
DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
DatasetManager datasetManager = new DefaultDatasetManager(datasetFramework, NamespaceId.SYSTEM, co.cask.cdap.common.service.RetryStrategies.noRetry());
Transactional transactional = Transactions.createTransactionalWithRetry(Transactions.createTransactional(new MultiThreadDatasetCache(new SystemDatasetInstantiator(datasetFramework), injector.getInstance(TransactionSystemClient.class), NamespaceId.SYSTEM, ImmutableMap.<String, String>of(), null, null)), RetryStrategies.retryOnConflict(20, 100));
FileMetaDataManager fileMetaDataManager = injector.getInstance(FileMetaDataManager.class);
FileMetaDataWriter fileMetaDataWriter = new FileMetaDataWriter(datasetManager, transactional);
LogPathIdentifier logPathIdentifier = new LogPathIdentifier(NamespaceId.DEFAULT.getNamespace(), "testApp", "testFlow");
LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
Location location = locationFactory.create(TMP_FOLDER.newFolder().getPath()).append("/logs");
long currentTime = System.currentTimeMillis();
LoggingContext loggingContext = LoggingContextHelper.getLoggingContext(NamespaceId.DEFAULT.getNamespace(), "testApp", "testFlow");
// 10 files in old format
for (int i = 1; i <= 10; i++) {
// i is the event time
fileMetaDataManager.writeMetaData(loggingContext, currentTime + i, location.append("testFile" + Integer.toString(i)));
}
long eventTime = currentTime + 20;
long newCurrentTime = currentTime + 100;
// 10 files in new format
for (int i = 1; i <= 10; i++) {
fileMetaDataWriter.writeMetaData(logPathIdentifier, eventTime + i, newCurrentTime + i, location.append("testFileNew" + Integer.toString(i)));
}
// reader test
FileMetaDataReader fileMetadataReader = injector.getInstance(FileMetaDataReader.class);
// scan only in old files time range
List<LogLocation> locations = fileMetadataReader.listFiles(logPathIdentifier, currentTime + 2, currentTime + 6);
// should include files from currentTime (1..6)
Assert.assertEquals(6, locations.size());
for (LogLocation logLocation : locations) {
Assert.assertEquals(LogLocation.VERSION_0, logLocation.getFrameworkVersion());
}
// scan only in new files time range
locations = fileMetadataReader.listFiles(logPathIdentifier, eventTime + 2, eventTime + 6);
// should include files from currentTime (1..6)
Assert.assertEquals(6, locations.size());
for (LogLocation logLocation : locations) {
Assert.assertEquals(LogLocation.VERSION_1, logLocation.getFrameworkVersion());
}
// scan time range across formats
locations = fileMetadataReader.listFiles(logPathIdentifier, currentTime + 2, eventTime + 6);
// should include files from old range (1..10) and new range (1..6)
Assert.assertEquals(16, locations.size());
for (int i = 0; i < locations.size(); i++) {
if (i < 10) {
Assert.assertEquals(LogLocation.VERSION_0, locations.get(i).getFrameworkVersion());
Assert.assertEquals(location.append("testFile" + Integer.toString(i + 1)), locations.get(i).getLocation());
} else {
Assert.assertEquals(LogLocation.VERSION_1, locations.get(i).getFrameworkVersion());
Assert.assertEquals(location.append("testFileNew" + Integer.toString(i - 9)), locations.get(i).getLocation());
}
}
}
Aggregations