Add support for deinterlaced videos playback

This is a follow up to #10254 to improve the playback of cut scenes in Layton's Mystery Journey.
It uses ffmpeg's yadif filter for deinterlacing.

CMakeLists.txt

@@ -453,6 +453,7 @@ endif()
 # List of all FFmpeg components required
 set(FFmpeg_COMPONENTS
     avcodec
+    avfilter
     avutil
     swscale)
 

externals/ffmpeg/CMakeLists.txt

@@ -131,7 +131,6 @@ if (NOT WIN32)
         COMMAND
             /bin/bash ${FFmpeg_PREFIX}/configure
                 --disable-avdevice
-                --disable-avfilter
                 --disable-avformat
                 --disable-doc
                 --disable-everything
@@ -143,6 +142,7 @@ if (NOT WIN32)
                 --enable-decoder=h264
                 --enable-decoder=vp8
                 --enable-decoder=vp9
+                --enable-filter=yadif
                 --cc="${FFmpeg_CC}"
                 --cxx="${FFmpeg_CXX}"
                 ${FFmpeg_HWACCEL_FLAGS}
@@ -199,7 +199,7 @@ if (NOT WIN32)
     endif()
 else(WIN32)
     # Use yuzu FFmpeg binaries
-    set(FFmpeg_EXT_NAME "ffmpeg-4.4")
+    set(FFmpeg_EXT_NAME "ffmpeg-5.1.3")
     set(FFmpeg_PATH "${CMAKE_BINARY_DIR}/externals/${FFmpeg_EXT_NAME}")
     download_bundled_external("ffmpeg/" ${FFmpeg_EXT_NAME} "")
     set(FFmpeg_FOUND YES)
@@ -210,6 +210,7 @@ else(WIN32)
     set(FFmpeg_LIBRARIES
     ${FFmpeg_LIBRARY_DIR}/swscale.lib
     ${FFmpeg_LIBRARY_DIR}/avcodec.lib
+    ${FFmpeg_LIBRARY_DIR}/avfilter.lib
     ${FFmpeg_LIBRARY_DIR}/avutil.lib
     CACHE PATH "Paths to FFmpeg libraries" FORCE)
     # exported variables

src/input_common/drivers/joycon.cpp

@@ -195,8 +195,8 @@ void Joycons::RegisterNewDevice(SDL_hid_device_info* device_info) {
                 OnMotionUpdate(port, type, id, value);
             }},
             .on_ring_data = {[this](f32 ring_data) { OnRingConUpdate(ring_data); }},
-            .on_amiibo_data = {[this, port](const std::vector<u8>& amiibo_data) {
-                OnAmiiboUpdate(port, amiibo_data);
+            .on_amiibo_data = {[this, port, type](const std::vector<u8>& amiibo_data) {
+                OnAmiiboUpdate(port, type, amiibo_data);
             }},
             .on_camera_data = {[this, port](const std::vector<u8>& camera_data,
                                             Joycon::IrsResolution format) {
@@ -398,8 +398,9 @@ void Joycons::OnRingConUpdate(f32 ring_data) {
     SetAxis(identifier, 100, ring_data);
 }
 
-void Joycons::OnAmiiboUpdate(std::size_t port, const std::vector<u8>& amiibo_data) {
-    const auto identifier = GetIdentifier(port, Joycon::ControllerType::Right);
+void Joycons::OnAmiiboUpdate(std::size_t port, Joycon::ControllerType type,
+                             const std::vector<u8>& amiibo_data) {
+    const auto identifier = GetIdentifier(port, type);
     const auto nfc_state = amiibo_data.empty() ? Common::Input::NfcState::AmiiboRemoved
                                                : Common::Input::NfcState::NewAmiibo;
     SetNfc(identifier, {nfc_state, amiibo_data});

src/input_common/drivers/joycon.h

@@ -81,7 +81,8 @@ private:
     void OnMotionUpdate(std::size_t port, Joycon::ControllerType type, int id,
                         const Joycon::MotionData& value);
     void OnRingConUpdate(f32 ring_data);
-    void OnAmiiboUpdate(std::size_t port, const std::vector<u8>& amiibo_data);
+    void OnAmiiboUpdate(std::size_t port, Joycon::ControllerType type,
+                        const std::vector<u8>& amiibo_data);
     void OnCameraUpdate(std::size_t port, const std::vector<u8>& camera_data,
                         Joycon::IrsResolution format);
 

src/input_common/helpers/joycon_protocol/common_protocol.cpp

@@ -265,7 +265,7 @@ DriverResult JoyconCommonProtocol::SendMCUData(ReportMode report_mode, MCUSubCom
 
 DriverResult JoyconCommonProtocol::WaitSetMCUMode(ReportMode report_mode, MCUMode mode) {
     MCUCommandResponse output{};
-    constexpr std::size_t MaxTries{8};
+    constexpr std::size_t MaxTries{16};
     std::size_t tries{};
 
     do {

src/input_common/helpers/joycon_protocol/joycon_types.h

@@ -577,8 +577,8 @@ static_assert(sizeof(NFCPollingCommandData) == 0x05, "NFCPollingCommandData is a
 
 struct NFCRequestState {
     NFCReadCommand command_argument;
-    u8 packet_id;
     INSERT_PADDING_BYTES(0x1);
+    u8 packet_id;
     MCUPacketFlag packet_flag;
     u8 data_length;
     union {

src/input_common/helpers/joycon_protocol/nfc.cpp

@@ -64,6 +64,20 @@ DriverResult NfcProtocol::StartNFCPollingMode() {
     if (result == DriverResult::Success) {
         result = WaitUntilNfcIsReady();
     }
+    if (result == DriverResult::Success) {
+        MCUCommandResponse output{};
+        result = SendStopPollingRequest(output);
+    }
+    if (result == DriverResult::Success) {
+        result = WaitUntilNfcIsReady();
+    }
+    if (result == DriverResult::Success) {
+        MCUCommandResponse output{};
+        result = SendStartPollingRequest(output);
+    }
+    if (result == DriverResult::Success) {
+        result = WaitUntilNfcIsPolling();
+    }
     if (result == DriverResult::Success) {
         is_enabled = true;
     }
@@ -77,24 +91,21 @@ DriverResult NfcProtocol::ScanAmiibo(std::vector<u8>& data) {
     }
     update_counter = 0;
 
-    LOG_DEBUG(Input, "Start NFC pooling Mode");
+    LOG_DEBUG(Input, "Scan for amiibos");
     ScopedSetBlocking sb(this);
     DriverResult result{DriverResult::Success};
     TagFoundData tag_data{};
 
     if (result == DriverResult::Success) {
-        result = StartPolling(tag_data);
-    }
-    if (result == DriverResult::Success) {
-        result = ReadTag(tag_data);
-    }
-    if (result == DriverResult::Success) {
-        result = WaitUntilNfcIsReady();
-    }
-    if (result == DriverResult::Success) {
-        result = StartPolling(tag_data, 7);
+        result = IsTagInRange(tag_data);
     }
+
     if (result == DriverResult::Success) {
+        std::string uuid_string;
+        for (auto& content : tag_data.uuid) {
+            uuid_string += fmt::format(" {:02x}", content);
+        }
+        LOG_INFO(Input, "Tag detected, type={}, uuid={}", tag_data.type, uuid_string);
         result = GetAmiiboData(data);
     }
 
@@ -102,12 +113,17 @@ DriverResult NfcProtocol::ScanAmiibo(std::vector<u8>& data) {
 }
 
 bool NfcProtocol::HasAmiibo() {
+    if (update_counter++ < AMIIBO_UPDATE_DELAY) {
+        return true;
+    }
+    update_counter = 0;
+
     ScopedSetBlocking sb(this);
     DriverResult result{DriverResult::Success};
     TagFoundData tag_data{};
 
     if (result == DriverResult::Success) {
-        result = StartPolling(tag_data);
+        result = IsTagInRange(tag_data, 7);
     }
 
     return result == DriverResult::Success;
@@ -119,7 +135,7 @@ DriverResult NfcProtocol::WaitUntilNfcIsReady() {
     std::size_t tries = 0;
 
     do {
-        auto result = SendStartWaitingRecieveRequest(output);
+        auto result = SendNextPackageRequest(output, {});
 
         if (result != DriverResult::Success) {
             return result;
@@ -134,13 +150,14 @@ DriverResult NfcProtocol::WaitUntilNfcIsReady() {
     return DriverResult::Success;
 }
 
-DriverResult NfcProtocol::StartPolling(TagFoundData& data, std::size_t timeout_limit) {
-    LOG_DEBUG(Input, "Start Polling for tag");
+DriverResult NfcProtocol::WaitUntilNfcIsPolling() {
+    constexpr std::size_t timeout_limit = 10;
     MCUCommandResponse output{};
     std::size_t tries = 0;
 
     do {
-        const auto result = SendStartPollingRequest(output);
+        auto result = SendNextPackageRequest(output, {});
+
         if (result != DriverResult::Success) {
             return result;
         }
@@ -149,7 +166,26 @@ DriverResult NfcProtocol::StartPolling(TagFoundData& data, std::size_t timeout_l
         }
     } while (output.mcu_report != MCUReport::NFCState ||
              (output.mcu_data[1] << 8) + output.mcu_data[0] != 0x0500 ||
-             output.mcu_data[6] != 0x09);
+             output.mcu_data[5] != 0x31 || output.mcu_data[6] != 0x01);
+
+    return DriverResult::Success;
+}
+
+DriverResult NfcProtocol::IsTagInRange(TagFoundData& data, std::size_t timeout_limit) {
+    MCUCommandResponse output{};
+    std::size_t tries = 0;
+
+    do {
+        const auto result = SendNextPackageRequest(output, {});
+        if (result != DriverResult::Success) {
+            return result;
+        }
+        if (tries++ > timeout_limit) {
+            return DriverResult::Timeout;
+        }
+    } while (output.mcu_report != MCUReport::NFCState ||
+             (output.mcu_data[1] << 8) + output.mcu_data[0] != 0x0500 ||
+             (output.mcu_data[6] != 0x09 && output.mcu_data[6] != 0x04));
 
     data.type = output.mcu_data[12];
     data.uuid.resize(output.mcu_data[14]);
@@ -158,85 +194,22 @@ DriverResult NfcProtocol::StartPolling(TagFoundData& data, std::size_t timeout_l
     return DriverResult::Success;
 }
 
-DriverResult NfcProtocol::ReadTag(const TagFoundData& data) {
-    constexpr std::size_t timeout_limit = 10;
-    MCUCommandResponse output{};
-    std::size_t tries = 0;
-
-    std::string uuid_string;
-    for (auto& content : data.uuid) {
-        uuid_string += fmt::format(" {:02x}", content);
-    }
-
-    LOG_INFO(Input, "Tag detected, type={}, uuid={}", data.type, uuid_string);
-
-    tries = 0;
-    NFCPages ntag_pages = NFCPages::Block0;
-    // Read Tag data
-    while (true) {
-        auto result = SendReadAmiiboRequest(output, ntag_pages);
-        const auto nfc_status = static_cast<NFCStatus>(output.mcu_data[6]);
-
-        if (result != DriverResult::Success) {
-            return result;
-        }
-
-        if ((output.mcu_report == MCUReport::NFCReadData ||
-             output.mcu_report == MCUReport::NFCState) &&
-            nfc_status == NFCStatus::TagLost) {
-            return DriverResult::ErrorReadingData;
-        }
-
-        if (output.mcu_report == MCUReport::NFCReadData && output.mcu_data[1] == 0x07 &&
-            output.mcu_data[2] == 0x01) {
-            if (data.type != 2) {
-                continue;
-            }
-            switch (output.mcu_data[24]) {
-            case 0:
-                ntag_pages = NFCPages::Block135;
-                break;
-            case 3:
-                ntag_pages = NFCPages::Block45;
-                break;
-            case 4:
-                ntag_pages = NFCPages::Block231;
-                break;
-            default:
-                return DriverResult::ErrorReadingData;
-            }
-            continue;
-        }
-
-        if (output.mcu_report == MCUReport::NFCState && nfc_status == NFCStatus::LastPackage) {
-            // finished
-            SendStopPollingRequest(output);
-            return DriverResult::Success;
-        }
-
-        // Ignore other state reports
-        if (output.mcu_report == MCUReport::NFCState) {
-            continue;
-        }
-
-        if (tries++ > timeout_limit) {
-            return DriverResult::Timeout;
-        }
-    }
-
-    return DriverResult::Success;
-}
-
 DriverResult NfcProtocol::GetAmiiboData(std::vector<u8>& ntag_data) {
-    constexpr std::size_t timeout_limit = 10;
+    constexpr std::size_t timeout_limit = 60;
     MCUCommandResponse output{};
     std::size_t tries = 0;
 
-    NFCPages ntag_pages = NFCPages::Block135;
+    u8 package_index = 0;
     std::size_t ntag_buffer_pos = 0;
+    auto result = SendReadAmiiboRequest(output, NFCPages::Block135);
+
+    if (result != DriverResult::Success) {
+        return result;
+    }
+
     // Read Tag data
-    while (true) {
-        auto result = SendReadAmiiboRequest(output, ntag_pages);
+    while (tries++ < timeout_limit) {
+        result = SendNextPackageRequest(output, package_index);
         const auto nfc_status = static_cast<NFCStatus>(output.mcu_data[6]);
 
         if (result != DriverResult::Success) {
@@ -259,6 +232,7 @@ DriverResult NfcProtocol::GetAmiiboData(std::vector<u8>& ntag_data) {
                 memcpy(ntag_data.data() + ntag_buffer_pos, output.mcu_data.data() + 6,
                        payload_size);
             }
+            package_index++;
             continue;
         }
 
@@ -266,18 +240,9 @@ DriverResult NfcProtocol::GetAmiiboData(std::vector<u8>& ntag_data) {
             LOG_INFO(Input, "Finished reading amiibo");
             return DriverResult::Success;
         }
-
-        // Ignore other state reports
-        if (output.mcu_report == MCUReport::NFCState) {
-            continue;
-        }
-
-        if (tries++ > timeout_limit) {
-            return DriverResult::Timeout;
-        }
     }
 
-    return DriverResult::Success;
+    return DriverResult::Timeout;
 }
 
 DriverResult NfcProtocol::SendStartPollingRequest(MCUCommandResponse& output) {
@@ -321,10 +286,10 @@ DriverResult NfcProtocol::SendStopPollingRequest(MCUCommandResponse& output) {
                        output);
 }
 
-DriverResult NfcProtocol::SendStartWaitingRecieveRequest(MCUCommandResponse& output) {
+DriverResult NfcProtocol::SendNextPackageRequest(MCUCommandResponse& output, u8 packet_id) {
     NFCRequestState request{
         .command_argument = NFCReadCommand::StartWaitingRecieve,
-        .packet_id = 0x0,
+        .packet_id = packet_id,
         .packet_flag = MCUPacketFlag::LastCommandPacket,
         .data_length = 0,
         .raw_data = {},

src/input_common/helpers/joycon_protocol/nfc.h

@@ -42,9 +42,9 @@ private:
 
     DriverResult WaitUntilNfcIsReady();
 
-    DriverResult StartPolling(TagFoundData& data, std::size_t timeout_limit = 1);
+    DriverResult WaitUntilNfcIsPolling();
 
-    DriverResult ReadTag(const TagFoundData& data);
+    DriverResult IsTagInRange(TagFoundData& data, std::size_t timeout_limit = 1);
 
     DriverResult GetAmiiboData(std::vector<u8>& data);
 
@@ -52,7 +52,7 @@ private:
 
     DriverResult SendStopPollingRequest(MCUCommandResponse& output);
 
-    DriverResult SendStartWaitingRecieveRequest(MCUCommandResponse& output);
+    DriverResult SendNextPackageRequest(MCUCommandResponse& output, u8 packet_id);
 
     DriverResult SendReadAmiiboRequest(MCUCommandResponse& output, NFCPages ntag_pages);
 

src/video_core/host1x/codecs/codec.cpp

@@ -5,6 +5,7 @@
 #include <fstream>
 #include <vector>
 #include "common/assert.h"
+#include "common/scope_exit.h"
 #include "common/settings.h"
 #include "video_core/host1x/codecs/codec.h"
 #include "video_core/host1x/codecs/h264.h"
@@ -14,6 +15,8 @@
 #include "video_core/memory_manager.h"
 
 extern "C" {
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
 #include <libavutil/opt.h>
 #ifdef LIBVA_FOUND
 // for querying VAAPI driver information
@@ -85,6 +88,10 @@ Codec::~Codec() {
     // Free libav memory
     avcodec_free_context(&av_codec_ctx);
     av_buffer_unref(&av_gpu_decoder);
+
+    if (filters_initialized) {
+        avfilter_graph_free(&av_filter_graph);
+    }
 }
 
 bool Codec::CreateGpuAvDevice() {
@@ -167,6 +174,62 @@ void Codec::InitializeGpuDecoder() {
     av_codec_ctx->get_format = GetGpuFormat;
 }
 
+void Codec::InitializeAvFilters(AVFrame* frame) {
+    const AVFilter* buffer_src = avfilter_get_by_name("buffer");
+    const AVFilter* buffer_sink = avfilter_get_by_name("buffersink");
+    AVFilterInOut* inputs = avfilter_inout_alloc();
+    AVFilterInOut* outputs = avfilter_inout_alloc();
+    SCOPE_EXIT({
+        avfilter_inout_free(&inputs);
+        avfilter_inout_free(&outputs);
+    });
+
+    // Don't know how to get the accurate time_base but it doesn't matter for yadif filter
+    // so just use 1/1 to make buffer filter happy
+    std::string args = fmt::format("video_size={}x{}:pix_fmt={}:time_base=1/1", frame->width,
+                                   frame->height, frame->format);
+
+    av_filter_graph = avfilter_graph_alloc();
+    int ret = avfilter_graph_create_filter(&av_filter_src_ctx, buffer_src, "in", args.c_str(),
+                                           nullptr, av_filter_graph);
+    if (ret < 0) {
+        LOG_ERROR(Service_NVDRV, "avfilter_graph_create_filter source error: {}", ret);
+        return;
+    }
+
+    ret = avfilter_graph_create_filter(&av_filter_sink_ctx, buffer_sink, "out", nullptr, nullptr,
+                                       av_filter_graph);
+    if (ret < 0) {
+        LOG_ERROR(Service_NVDRV, "avfilter_graph_create_filter sink error: {}", ret);
+        return;
+    }
+
+    inputs->name = av_strdup("out");
+    inputs->filter_ctx = av_filter_sink_ctx;
+    inputs->pad_idx = 0;
+    inputs->next = nullptr;
+
+    outputs->name = av_strdup("in");
+    outputs->filter_ctx = av_filter_src_ctx;
+    outputs->pad_idx = 0;
+    outputs->next = nullptr;
+
+    const char* description = "yadif=1:-1:0";
+    ret = avfilter_graph_parse_ptr(av_filter_graph, description, &inputs, &outputs, nullptr);
+    if (ret < 0) {
+        LOG_ERROR(Service_NVDRV, "avfilter_graph_parse_ptr error: {}", ret);
+        return;
+    }
+
+    ret = avfilter_graph_config(av_filter_graph, nullptr);
+    if (ret < 0) {
+        LOG_ERROR(Service_NVDRV, "avfilter_graph_config error: {}", ret);
+        return;
+    }
+
+    filters_initialized = true;
+}
+
 void Codec::Initialize() {
     const AVCodecID codec = [&] {
         switch (current_codec) {
@@ -271,8 +334,34 @@ void Codec::Decode() {
         UNIMPLEMENTED_MSG("Unexpected video format: {}", final_frame->format);
         return;
     }
-    av_frames.push(std::move(final_frame));
-    if (av_frames.size() > 10) {
+    if (!final_frame->interlaced_frame) {
+        av_frames.push(std::move(final_frame));
+    } else {
+        if (!filters_initialized) {
+            InitializeAvFilters(final_frame.get());
+        }
+        if (const int ret = av_buffersrc_add_frame_flags(av_filter_src_ctx, final_frame.get(),
+                                                         AV_BUFFERSRC_FLAG_KEEP_REF);
+            ret) {
+            LOG_DEBUG(Service_NVDRV, "av_buffersrc_add_frame_flags error {}", ret);
+            return;
+        }
+        while (true) {
+            auto filter_frame = AVFramePtr{av_frame_alloc(), AVFrameDeleter};
+
+            int ret = av_buffersink_get_frame(av_filter_sink_ctx, filter_frame.get());
+
+            if (ret == AVERROR(EAGAIN) || ret == AVERROR(AVERROR_EOF))
+                break;
+            if (ret < 0) {
+                LOG_DEBUG(Service_NVDRV, "av_buffersink_get_frame error {}", ret);
+                return;
+            }
+
+            av_frames.push(std::move(filter_frame));
+        }
+    }
+    while (av_frames.size() > 10) {
         LOG_TRACE(Service_NVDRV, "av_frames.push overflow dropped frame");
         av_frames.pop();
     }

src/video_core/host1x/codecs/codec.h

@@ -15,6 +15,7 @@ extern "C" {
 #pragma GCC diagnostic ignored "-Wconversion"
 #endif
 #include <libavcodec/avcodec.h>
+#include <libavfilter/avfilter.h>
 #if defined(__GNUC__) || defined(__clang__)
 #pragma GCC diagnostic pop
 #endif
@@ -61,17 +62,24 @@ public:
 private:
     void InitializeAvCodecContext();
 
+    void InitializeAvFilters(AVFrame* frame);
+
     void InitializeGpuDecoder();
 
     bool CreateGpuAvDevice();
 
     bool initialized{};
+    bool filters_initialized{};
     Host1x::NvdecCommon::VideoCodec current_codec{Host1x::NvdecCommon::VideoCodec::None};
 
     const AVCodec* av_codec{nullptr};
     AVCodecContext* av_codec_ctx{nullptr};
     AVBufferRef* av_gpu_decoder{nullptr};
 
+    AVFilterContext* av_filter_src_ctx{nullptr};
+    AVFilterContext* av_filter_sink_ctx{nullptr};
+    AVFilterGraph* av_filter_graph{nullptr};
+
     Host1x::Host1x& host1x;
     const Host1x::NvdecCommon::NvdecRegisters& state;
     std::unique_ptr<Decoder::H264> h264_decoder;