Make StatsLog drop less.
+ Create a thread-safe LogEventQueue to buffer log events.
+ The socket listner thread will read from socket and write to the buffer as quickly as possible
to minimize the data loss in socket.
+ All pushed data is fetched from the the buffer and processed in a dedicated thread. After an
event is fetched from the queue, we no longer block the socket listener thread.
+ Report event queue stats via statsdstats, including the min and max queue event history span in
the queue (to understand how slow statsd can be and how fast the events can be)
Bug: 119031518
Test: unit tests added in statsd_test
Change-Id: I6b65ed9a678935b2e24302ba4b36e69c157adde4
diff --git a/cmds/statsd/src/logd/LogEvent.cpp b/cmds/statsd/src/logd/LogEvent.cpp
index d9f5415..d5e66fd 100644
--- a/cmds/statsd/src/logd/LogEvent.cpp
+++ b/cmds/statsd/src/logd/LogEvent.cpp
@@ -118,6 +118,7 @@
LogEvent::LogEvent(int32_t tagId, int64_t wallClockTimestampNs, int64_t elapsedTimestampNs) {
mLogdTimestampNs = wallClockTimestampNs;
+ mElapsedTimestampNs = elapsedTimestampNs;
mTagId = tagId;
mLogUid = 0;
mContext = create_android_logger(1937006964); // the event tag shared by all stats logs
@@ -246,7 +247,8 @@
mValues.push_back(FieldValue(Field(mTagId, getSimpleField(4)), Value(trainInfo.status)));
}
-LogEvent::LogEvent(int32_t tagId, int64_t timestampNs) : LogEvent(tagId, timestampNs, 0) {}
+LogEvent::LogEvent(int32_t tagId, int64_t timestampNs) : LogEvent(tagId, timestampNs, timestampNs) {
+}
LogEvent::LogEvent(int32_t tagId, int64_t timestampNs, int32_t uid) {
mLogdTimestampNs = timestampNs;
diff --git a/cmds/statsd/src/logd/LogEventQueue.cpp b/cmds/statsd/src/logd/LogEventQueue.cpp
new file mode 100644
index 0000000..146464b
--- /dev/null
+++ b/cmds/statsd/src/logd/LogEventQueue.cpp
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define DEBUG false // STOPSHIP if true
+#include "Log.h"
+
+#include "LogEventQueue.h"
+
+namespace android {
+namespace os {
+namespace statsd {
+
+using std::unique_lock;
+using std::unique_ptr;
+
+unique_ptr<LogEvent> LogEventQueue::waitPop() {
+ std::unique_lock<std::mutex> lock(mMutex);
+
+ if (mQueue.empty()) {
+ mCondition.wait(lock, [this] { return !this->mQueue.empty(); });
+ }
+
+ unique_ptr<LogEvent> item = std::move(mQueue.front());
+ mQueue.pop();
+
+ return item;
+}
+
+bool LogEventQueue::push(unique_ptr<LogEvent> item, int64_t* oldestTimestampNs) {
+ bool success;
+ {
+ std::unique_lock<std::mutex> lock(mMutex);
+ if (mQueue.size() < mQueueLimit) {
+ mQueue.push(std::move(item));
+ success = true;
+ } else {
+ // safe operation as queue must not be empty.
+ *oldestTimestampNs = mQueue.front()->GetElapsedTimestampNs();
+ success = false;
+ }
+ }
+
+ mCondition.notify_one();
+ return success;
+}
+
+} // namespace statsd
+} // namespace os
+} // namespace android
diff --git a/cmds/statsd/src/logd/LogEventQueue.h b/cmds/statsd/src/logd/LogEventQueue.h
new file mode 100644
index 0000000..b4fd63f
--- /dev/null
+++ b/cmds/statsd/src/logd/LogEventQueue.h
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "LogEvent.h"
+
+#include <condition_variable>
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <thread>
+
+namespace android {
+namespace os {
+namespace statsd {
+
+/**
+ * A zero copy thread safe queue buffer for producing and consuming LogEvent.
+ */
+class LogEventQueue {
+public:
+ explicit LogEventQueue(size_t maxSize) : mQueueLimit(maxSize){};
+
+ /**
+ * Blocking read one event from the queue.
+ */
+ std::unique_ptr<LogEvent> waitPop();
+
+ /**
+ * Puts a LogEvent ptr to the end of the queue.
+ * Returns false on failure when the queue is full, and output the oldest event timestamp
+ * in the queue.
+ */
+ bool push(std::unique_ptr<LogEvent> event, int64_t* oldestTimestampNs);
+
+private:
+ const size_t mQueueLimit;
+ std::condition_variable mCondition;
+ std::mutex mMutex;
+ std::queue<std::unique_ptr<LogEvent>> mQueue;
+};
+
+} // namespace statsd
+} // namespace os
+} // namespace android
diff --git a/cmds/statsd/src/logd/LogListener.cpp b/cmds/statsd/src/logd/LogListener.cpp
deleted file mode 100644
index ddb26f9..0000000
--- a/cmds/statsd/src/logd/LogListener.cpp
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright (C) 2017 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "logd/LogListener.h"
-
-namespace android {
-namespace os {
-namespace statsd {
-
-LogListener::LogListener() {
-}
-
-LogListener::~LogListener() {
-}
-
-} // namespace statsd
-} // namespace os
-} // namespace android
diff --git a/cmds/statsd/src/logd/LogListener.h b/cmds/statsd/src/logd/LogListener.h
deleted file mode 100644
index d8b06e9..0000000
--- a/cmds/statsd/src/logd/LogListener.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (C) 2017 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include "logd/LogEvent.h"
-
-#include <utils/RefBase.h>
-
-namespace android {
-namespace os {
-namespace statsd {
-
-/**
- * Callback for LogReader
- */
-class LogListener : public virtual android::RefBase {
-public:
- LogListener();
- virtual ~LogListener();
-
- virtual void OnLogEvent(LogEvent* msg) = 0;
-};
-
-} // namespace statsd
-} // namespace os
-} // namespace android