libs/input/Resampler.cpp

/**
 * Copyright 2024 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#define LOG_TAG "LegacyResampler"

#include <algorithm>
#include <chrono>
#include <ostream>

#include <android-base/logging.h>
#include <android-base/properties.h>
#include <ftl/enum.h>

#include <input/Resampler.h>
#include <utils/Timers.h>

namespace android {
namespace {

const bool IS_DEBUGGABLE_BUILD =
#if defined(__ANDROID__)
        android::base::GetBoolProperty("ro.debuggable", false);
#else
        true;
#endif

bool debugResampling() {
    if (!IS_DEBUGGABLE_BUILD) {
        static const bool DEBUG_TRANSPORT_RESAMPLING =
                __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling",
                                          ANDROID_LOG_INFO);
        return DEBUG_TRANSPORT_RESAMPLING;
    }
    return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling", ANDROID_LOG_INFO);
}

using std::chrono::nanoseconds;

constexpr std::chrono::milliseconds RESAMPLE_LATENCY{5};

constexpr std::chrono::milliseconds RESAMPLE_MIN_DELTA{2};

constexpr std::chrono::milliseconds RESAMPLE_MAX_DELTA{20};

constexpr std::chrono::milliseconds RESAMPLE_MAX_PREDICTION{8};

bool canResampleTool(ToolType toolType) {
    return toolType == ToolType::FINGER || toolType == ToolType::MOUSE ||
            toolType == ToolType::STYLUS || toolType == ToolType::UNKNOWN;
}

inline float lerp(float a, float b, float alpha) {
    return a + alpha * (b - a);
}

PointerCoords calculateResampledCoords(const PointerCoords& a, const PointerCoords& b,
                                       float alpha) {
    // We use the value of alpha to initialize resampledCoords with the latest sample information.
    PointerCoords resampledCoords = (alpha < 1.0f) ? a : b;
    resampledCoords.isResampled = true;
    resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_X, lerp(a.getX(), b.getX(), alpha));
    resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, lerp(a.getY(), b.getY(), alpha));
    return resampledCoords;
}

bool equalXY(const PointerCoords& a, const PointerCoords& b) {
    return (a.getX() == b.getX()) && (a.getY() == b.getY());
}

void setMotionEventPointerCoords(MotionEvent& motionEvent, size_t sampleIndex, size_t pointerIndex,
                                 const PointerCoords& pointerCoords) {
    // Ideally, we should not cast away const. In this particular case, it's safe to cast away const
    // and dereference getHistoricalRawPointerCoords returned pointer because motionEvent is a
    // nonconst reference to a MotionEvent object, so mutating the object should not be undefined
    // behavior; moreover, the invoked method guarantees to return a valid pointer. Otherwise, it
    // fatally logs. Alternatively, we could've created a new MotionEvent from scratch, but this
    // approach is simpler and more efficient.
    PointerCoords& motionEventCoords = const_cast<PointerCoords&>(
            *(motionEvent.getHistoricalRawPointerCoords(pointerIndex, sampleIndex)));
    motionEventCoords.setAxisValue(AMOTION_EVENT_AXIS_X, pointerCoords.getX());
    motionEventCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, pointerCoords.getY());
    motionEventCoords.isResampled = pointerCoords.isResampled;
}

std::ostream& operator<<(std::ostream& os, const PointerCoords& pointerCoords) {
    os << "(" << pointerCoords.getX() << ", " << pointerCoords.getY() << ")";
    return os;
}

} // namespace

void LegacyResampler::updateLatestSamples(const MotionEvent& motionEvent) {
    const size_t numSamples = motionEvent.getHistorySize() + 1;
    const size_t latestIndex = numSamples - 1;
    const size_t secondToLatestIndex = (latestIndex > 0) ? (latestIndex - 1) : 0;
    for (size_t sampleIndex = secondToLatestIndex; sampleIndex < numSamples; ++sampleIndex) {
        std::vector<Pointer> pointers;
        const size_t numPointers = motionEvent.getPointerCount();
        for (size_t pointerIndex = 0; pointerIndex < numPointers; ++pointerIndex) {
            pointers.push_back(Pointer{*(motionEvent.getPointerProperties(pointerIndex)),
                                       *(motionEvent.getHistoricalRawPointerCoords(pointerIndex,
                                                                                   sampleIndex))});
        }
        mLatestSamples.pushBack(
                Sample{nanoseconds{motionEvent.getHistoricalEventTime(sampleIndex)}, pointers});
    }
}

LegacyResampler::Sample LegacyResampler::messageToSample(const InputMessage& message) {
    std::vector<Pointer> pointers;
    for (uint32_t i = 0; i < message.body.motion.pointerCount; ++i) {
        pointers.push_back(Pointer{message.body.motion.pointers[i].properties,
                                   message.body.motion.pointers[i].coords});
    }
    return Sample{nanoseconds{message.body.motion.eventTime}, pointers};
}

bool LegacyResampler::pointerPropertiesResampleable(const Sample& target, const Sample& auxiliary) {
    if (target.pointers.size() > auxiliary.pointers.size()) {
        LOG_IF(INFO, debugResampling())
                << "Not resampled. Auxiliary sample has fewer pointers than target sample.";
        return false;
    }
    for (size_t i = 0; i < target.pointers.size(); ++i) {
        if (target.pointers[i].properties.id != auxiliary.pointers[i].properties.id) {
            LOG_IF(INFO, debugResampling()) << "Not resampled. Pointer ID mismatch.";
            return false;
        }
        if (target.pointers[i].properties.toolType != auxiliary.pointers[i].properties.toolType) {
            LOG_IF(INFO, debugResampling()) << "Not resampled. Pointer ToolType mismatch.";
            return false;
        }
        if (!canResampleTool(target.pointers[i].properties.toolType)) {
            LOG_IF(INFO, debugResampling())
                    << "Not resampled. Cannot resample "
                    << ftl::enum_string(target.pointers[i].properties.toolType) << " ToolType.";
            return false;
        }
    }
    return true;
}

bool LegacyResampler::canInterpolate(const InputMessage& message) const {
    LOG_IF(FATAL, mLatestSamples.empty())
            << "Not resampled. mLatestSamples must not be empty to interpolate.";

    const Sample& pastSample = *(mLatestSamples.end() - 1);
    const Sample& futureSample = messageToSample(message);

    if (!pointerPropertiesResampleable(pastSample, futureSample)) {
        return false;
    }

    const nanoseconds delta = futureSample.eventTime - pastSample.eventTime;
    if (delta < RESAMPLE_MIN_DELTA) {
        LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns.";
        return false;
    }
    return true;
}

std::optional<LegacyResampler::Sample> LegacyResampler::attemptInterpolation(
        nanoseconds resampleTime, const InputMessage& futureSample) const {
    if (!canInterpolate(futureSample)) {
        return std::nullopt;
    }
    LOG_IF(FATAL, mLatestSamples.empty())
            << "Not resampled. mLatestSamples must not be empty to interpolate.";

    const Sample& pastSample = *(mLatestSamples.end() - 1);

    const nanoseconds delta =
            nanoseconds{futureSample.body.motion.eventTime} - pastSample.eventTime;
    const float alpha =
            std::chrono::duration<float, std::milli>(resampleTime - pastSample.eventTime) / delta;

    std::vector<Pointer> resampledPointers;
    for (size_t i = 0; i < pastSample.pointers.size(); ++i) {
        const PointerCoords& resampledCoords =
                calculateResampledCoords(pastSample.pointers[i].coords,
                                         futureSample.body.motion.pointers[i].coords, alpha);
        resampledPointers.push_back(Pointer{pastSample.pointers[i].properties, resampledCoords});
    }
    return Sample{resampleTime, resampledPointers};
}

bool LegacyResampler::canExtrapolate() const {
    if (mLatestSamples.size() < 2) {
        LOG_IF(INFO, debugResampling()) << "Not resampled. Not enough data.";
        return false;
    }

    const Sample& pastSample = *(mLatestSamples.end() - 2);
    const Sample& presentSample = *(mLatestSamples.end() - 1);

    if (!pointerPropertiesResampleable(presentSample, pastSample)) {
        return false;
    }

    const nanoseconds delta = presentSample.eventTime - pastSample.eventTime;
    if (delta < RESAMPLE_MIN_DELTA) {
        LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns.";
        return false;
    } else if (delta > RESAMPLE_MAX_DELTA) {
        LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too large: " << delta << "ns.";
        return false;
    }
    return true;
}

std::optional<LegacyResampler::Sample> LegacyResampler::attemptExtrapolation(
        nanoseconds resampleTime) const {
    if (!canExtrapolate()) {
        return std::nullopt;
    }
    LOG_IF(FATAL, mLatestSamples.size() < 2)
            << "Not resampled. mLatestSamples must have at least two samples to extrapolate.";

    const Sample& pastSample = *(mLatestSamples.end() - 2);
    const Sample& presentSample = *(mLatestSamples.end() - 1);

    const nanoseconds delta = presentSample.eventTime - pastSample.eventTime;
    // The farthest future time to which we can extrapolate. If the given resampleTime exceeds this,
    // we use this value as the resample time target.
    const nanoseconds farthestPrediction =
            presentSample.eventTime + std::min<nanoseconds>(delta / 2, RESAMPLE_MAX_PREDICTION);
    const nanoseconds newResampleTime =
            (resampleTime > farthestPrediction) ? (farthestPrediction) : (resampleTime);
    LOG_IF(INFO, debugResampling() && newResampleTime == farthestPrediction)
            << "Resample time is too far in the future. Adjusting prediction from "
            << (resampleTime - presentSample.eventTime) << " to "
            << (farthestPrediction - presentSample.eventTime) << "ns.";
    const float alpha =
            std::chrono::duration<float, std::milli>(newResampleTime - pastSample.eventTime) /
            delta;

    std::vector<Pointer> resampledPointers;
    for (size_t i = 0; i < presentSample.pointers.size(); ++i) {
        const PointerCoords& resampledCoords =
                calculateResampledCoords(pastSample.pointers[i].coords,
                                         presentSample.pointers[i].coords, alpha);
        resampledPointers.push_back(Pointer{presentSample.pointers[i].properties, resampledCoords});
    }
    return Sample{newResampleTime, resampledPointers};
}

inline void LegacyResampler::addSampleToMotionEvent(const Sample& sample,
                                                    MotionEvent& motionEvent) {
    motionEvent.addSample(sample.eventTime.count(), sample.asPointerCoords().data(),
                          motionEvent.getId());
}

nanoseconds LegacyResampler::getResampleLatency() const {
    return RESAMPLE_LATENCY;
}

void LegacyResampler::overwriteMotionEventSamples(MotionEvent& motionEvent) const {
    const size_t numSamples = motionEvent.getHistorySize() + 1;
    for (size_t sampleIndex = 0; sampleIndex < numSamples; ++sampleIndex) {
        overwriteStillPointers(motionEvent, sampleIndex);
    }
}

void LegacyResampler::overwriteStillPointers(MotionEvent& motionEvent, size_t sampleIndex) const {
    for (size_t pointerIndex = 0; pointerIndex < motionEvent.getPointerCount(); ++pointerIndex) {
        const PointerCoords& pointerCoords =
                *(motionEvent.getHistoricalRawPointerCoords(pointerIndex, sampleIndex));
        if (equalXY(mLastRealSample->pointers[pointerIndex].coords, pointerCoords)) {
            LOG_IF(INFO, debugResampling())
                    << "Pointer ID: " << motionEvent.getPointerId(pointerIndex)
                    << " did not move. Overwriting its coordinates from " << pointerCoords << " to "
                    << mLastRealSample->pointers[pointerIndex].coords;
            setMotionEventPointerCoords(motionEvent, sampleIndex, pointerIndex,
                                        mPreviousPrediction->pointers[pointerIndex].coords);
        }
    }
}

void LegacyResampler::resampleMotionEvent(nanoseconds frameTime, MotionEvent& motionEvent,
                                          const InputMessage* futureSample) {
    const nanoseconds resampleTime = frameTime - RESAMPLE_LATENCY;

    if (resampleTime.count() == motionEvent.getEventTime()) {
        LOG_IF(INFO, debugResampling()) << "Not resampled. Resample time equals motion event time.";
        return;
    }

    updateLatestSamples(motionEvent);

    const std::optional<Sample> sample = (futureSample != nullptr)
            ? (attemptInterpolation(resampleTime, *futureSample))
            : (attemptExtrapolation(resampleTime));
    if (sample.has_value()) {
        addSampleToMotionEvent(*sample, motionEvent);
        if (mPreviousPrediction.has_value()) {
            overwriteMotionEventSamples(motionEvent);
        }
        // mPreviousPrediction is only updated whenever extrapolation occurs because extrapolation
        // is about predicting upcoming scenarios.
        if (futureSample == nullptr) {
            mPreviousPrediction = sample;
        }
    }
    mLastRealSample = *(mLatestSamples.end() - 1);
}

} // namespace android