基于FFmpeg的Unity视频播放器（一）

基于FFmpeg的Unity视频播放器

using FFmpeg.AutoGen;
 using FFmpeg.AutoGen.Example;
 using System;
 using System.Collections.Generic;
 using System.Drawing;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Threading;
 using UnityEngine;namespace UnityFFmpeg
 {/// 
/// FFPlayer/// 
public sealed unsafe class FFPlayer : IDisposable{private string _url;int error;AVHWDeviceType deviceType;private Size _frameSize;private AVPixelFormat _pixelFormat;private AVSampleFormat _sample_fmt;private readonly AVCodecContext* _pVideoContext;private readonly AVCodecContext* _pAudioContext;private readonly AVFormatContext* _pFormatContext;private readonly int _videoStreamIndex;private readonly int _audioStreamIndex;private readonly AVFrame* _audioFrame;private readonly AVFrame* _videoFrame;private AVFrame* _g2cFrame;private AVFrame _tempFrame;private readonly AVPacket* _packet;SwrContext* _audioSwrContext;int outChannelCount;AVSampleFormat outFormat;AVPixelFormat destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_RGB24;private Action _onVideoData;private Action _onAudioData;private Action _onVideoSize;Thread thread = null;public FFPlayer(string url, Action onVideoSize, Action onVideoData, Action onAudioData){_url = url;_onVideoSize += onVideoSize;_onVideoData += onVideoData;_onAudioData += onAudioData;Init();_pFormatContext = ffmpeg.avformat_alloc_context();var pFormatContext = _pFormatContext;ffmpeg.avformat_open_input(&pFormatContext, url, null, null).ThrowExceptionIfError();ffmpeg.avformat_find_stream_info(_pFormatContext, null).ThrowExceptionIfError();AVCodec* videoCodec = null;_videoStreamIndex = ffmpeg.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &videoCodec, 0).ThrowExceptionIfError();AVCodec* audioCodec = null;_audioStreamIndex = ffmpeg.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_AUDIO, -1, -1, &audioCodec, 0).ThrowExceptionIfError();Debug.LogWarning("_videoStreamIndex:" + _videoStreamIndex);Debug.LogWarning("_audioStreamIndex:" + _audioStreamIndex);_pVideoContext = ffmpeg.avcodec_alloc_context3(videoCodec);Debug.LogWarning("deviceType:" + deviceType);if (deviceType != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE){ffmpeg.av_hwdevice_ctx_create(&_pVideoContext->hw_device_ctx, deviceType, null, null, 0).ThrowExceptionIfError();}AVCodecParameters* vavcp = _pFormatContext->streams[_videoStreamIndex]->codecpar;ffmpeg.avcodec_parameters_to_context(_pVideoContext, vavcp).ThrowExceptionIfError();_pAudioContext = ffmpeg.avcodec_alloc_context3(audioCodec);AVCodecParameters* aavcp = _pFormatContext->streams[_audioStreamIndex]->codecpar;ffmpeg.avcodec_parameters_to_context(_pAudioContext, aavcp).ThrowExceptionIfError();ffmpeg.avcodec_open2(_pVideoContext, videoCodec, null).ThrowExceptionIfError();ffmpeg.avcodec_open2(_pAudioContext, audioCodec, null).ThrowExceptionIfError();_frameSize = new Size(_pVideoContext->width, _pVideoContext->height);if (_onVideoSize != null){_onVideoSize(_frameSize.Width, _frameSize.Height);}_pixelFormat = _pVideoContext->pix_fmt;_sample_fmt = _pVideoContext->sample_fmt;//frame->16bit 44100 PCM 统一音频采样格式与采样率//创建swrcontext上下文件_audioSwrContext = ffmpeg.swr_alloc();//音频格式  输入的采样设置参数AVSampleFormat inFormat = _pAudioContext->sample_fmt;// 出入的采样格式outFormat = AVSampleFormat.AV_SAMPLE_FMT_S16;// 输入采样率int inSampleRate = _pAudioContext->sample_rate;// 输出采样率int outSampleRate = 44100;// 输入声道布局ulong in_ch_layout = _pAudioContext->channel_layout;//输出声道布局int out_ch_layout = ffmpeg.AV_CH_LAYOUT_STEREO;//给Swrcontext 分配空间，设置公共参数ffmpeg.swr_alloc_set_opts(_audioSwrContext, out_ch_layout, outFormat, outSampleRate,(long)in_ch_layout, inFormat, inSampleRate, 0, null);// 初始化ffmpeg.swr_init(_audioSwrContext);// 获取声道数量outChannelCount = ffmpeg.av_get_channel_layout_nb_channels((ulong)out_ch_layout);_packet = ffmpeg.av_packet_alloc();_audioFrame = ffmpeg.av_frame_alloc();_videoFrame = ffmpeg.av_frame_alloc();_g2cFrame = ffmpeg.av_frame_alloc();thread = new Thread(new ThreadStart(DecodeTest));thread.IsBackground = true;thread.Start();}private void Init(){ffmpeg.RootPath = Application.streamingAssetsPath + "/FFmpeg/x86_64";Debug.LogWarning($"FFmpeg version info: {ffmpeg.av_version_info()}");SetupLogging();ConfigureHWDecoder();}bool isVideo;private void DecodeTest(){while (true){ffmpeg.av_frame_unref(_audioFrame);ffmpeg.av_frame_unref(_videoFrame);do{//开始读取源文件，进行解码error = ffmpeg.av_read_frame(_pFormatContext, _packet);if (error == ffmpeg.AVERROR_EOF){Debug.LogWarning("over");break;}if (_packet->stream_index == _audioStreamIndex){error = ffmpeg.avcodec_send_packet(_pAudioContext, _packet);//解码error = ffmpeg.avcodec_receive_frame(_pAudioContext, _audioFrame);isVideo = false;}else if (_packet->stream_index == _videoStreamIndex){error = ffmpeg.avcodec_send_packet(_pVideoContext, _packet);//解码error = ffmpeg.avcodec_receive_frame(_pVideoContext, _videoFrame);isVideo = true;}}while (error == ffmpeg.AVERROR(ffmpeg.EAGAIN));if (!isVideo){byte* out_buffer = (byte*)Marshal.AllocHGlobal(2 * 44100);//将每一帧数据转换成pcmffmpeg.swr_convert(_audioSwrContext, &out_buffer, 2 * 44100, (byte**)&_audioFrame->data, _audioFrame->nb_samples);//获取实际的缓存大小int out_buffer_size = ffmpeg.av_samples_get_buffer_size(null, outChannelCount, _audioFrame->nb_samples, outFormat, 1);if (out_buffer_size > 0){byte[] data = new byte[out_buffer_size];Marshal.Copy((IntPtr)out_buffer, data, 0, out_buffer_size);if (_onAudioData != null){_onAudioData(data);}}else{Debug.LogWarning("out_buffer_size:" + out_buffer_size);}Marshal.FreeCoTaskMem((IntPtr)out_buffer);}if (isVideo){if (_pVideoContext->hw_device_ctx != null){ffmpeg.av_hwframe_transfer_data(_g2cFrame, _videoFrame, 0).ThrowExceptionIfError();_tempFrame = *_g2cFrame;}else{_tempFrame = *_videoFrame;}var sourcePixelFormat = GetHWPixelFormat(deviceType);using (var vfc = new VideoFrameConverter(_frameSize, sourcePixelFormat, _frameSize, destinationPixelFormat)){AVFrame convertedFrame = vfc.Convert(_tempFrame);IntPtr imgPtr = (IntPtr)convertedFrame.data[0];int dataLen = _frameSize.Width * _frameSize.Height * 3;byte[] data = new byte[dataLen];Marshal.Copy((IntPtr)convertedFrame.data[0], data, 0, data.Length);if (_onVideoData != null){_onVideoData(data);}Marshal.FreeCoTaskMem(imgPtr);}}Thread.Sleep(12);}}public void Play(){}public void Pause(){}public void Stop(){}private unsafe void SetupLogging(){ffmpeg.av_log_set_level(ffmpeg.AV_LOG_VERBOSE);// do not convert to local functionav_log_set_callback_callback logCallback = (p0, level, format, vl) =>{if (level > ffmpeg.av_log_get_level()){return;}var lineSize = 1024;var lineBuffer = stackalloc byte[lineSize];var printPrefix = 1;ffmpeg.av_log_format_line(p0, level, format, vl, lineBuffer, lineSize, &printPrefix);var line = Marshal.PtrToStringAnsi((IntPtr)lineBuffer);Debug.LogWarning(line);};ffmpeg.av_log_set_callback(logCallback);}private void ConfigureHWDecoder(){deviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE;var availableHWDecoders = new Dictionary();var type = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE;var number = 0;while ((type = ffmpeg.av_hwdevice_iterate_types(type)) != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE){Debug.Log($"{++number}. {type}");availableHWDecoders.Add(number, type);}if (availableHWDecoders.Count == 0){Debug.Log("Your system have no hardware decoders.");deviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE;return;}int decoderNumber = availableHWDecoders.SingleOrDefault(t => t.Value == AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2).Key;if (decoderNumber == 0){decoderNumber = availableHWDecoders.First().Key;}Debug.LogWarning($"Selected [{decoderNumber}]");int.TryParse(Console.ReadLine(), out var inputDecoderNumber);availableHWDecoders.TryGetValue(inputDecoderNumber == 0 ? decoderNumber : inputDecoderNumber, out deviceType);}private AVPixelFormat GetHWPixelFormat(AVHWDeviceType hWDevice){switch (hWDevice){case AVHWDeviceType.AV_HWDEVICE_TYPE_NONE:return AVPixelFormat.AV_PIX_FMT_NONE;case AVHWDeviceType.AV_HWDEVICE_TYPE_VDPAU:return AVPixelFormat.AV_PIX_FMT_VDPAU;case AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA:return AVPixelFormat.AV_PIX_FMT_CUDA;case AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI:return AVPixelFormat.AV_PIX_FMT_VAAPI;case AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2:return AVPixelFormat.AV_PIX_FMT_NV12;case AVHWDeviceType.AV_HWDEVICE_TYPE_QSV:return AVPixelFormat.AV_PIX_FMT_QSV;case AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX:return AVPixelFormat.AV_PIX_FMT_VIDEOTOOLBOX;case AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA:return AVPixelFormat.AV_PIX_FMT_NV12;case AVHWDeviceType.AV_HWDEVICE_TYPE_DRM:return AVPixelFormat.AV_PIX_FMT_DRM_PRIME;case AVHWDeviceType.AV_HWDEVICE_TYPE_OPENCL:return AVPixelFormat.AV_PIX_FMT_OPENCL;case AVHWDeviceType.AV_HWDEVICE_TYPE_MEDIACODEC:return AVPixelFormat.AV_PIX_FMT_MEDIACODEC;default:return AVPixelFormat.AV_PIX_FMT_NONE;}}public void Dispose(){if (thread != null){if (thread.IsAlive){thread.Abort();}}ffmpeg.av_frame_unref(_audioFrame);ffmpeg.av_free(_audioFrame);ffmpeg.av_frame_unref(_videoFrame);ffmpeg.av_free(_videoFrame);ffmpeg.av_frame_unref(_g2cFrame);ffmpeg.av_free(_g2cFrame);ffmpeg.av_packet_unref(_packet);ffmpeg.av_free(_packet);ffmpeg.avcodec_close(_pVideoContext);ffmpeg.avcodec_close(_pAudioContext);var pFormatContext = _pFormatContext;ffmpeg.avformat_close_input(&pFormatContext);}}
 }
 

using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.UI;
 using UnityFFmpeg;public class TestPlayer : MonoBehaviour
 {Texture2D texture2D;Queue imgQueue = new Queue();Queue audioQueue = new Queue();public RawImage rawImage;FFPlayer ffPlayer;// Start is called before the first frame updatevoid Start(){//var url = "rtmp://58.200.131.2:1935/livetv/hunantv";var url = Application.streamingAssetsPath + "/test.mp4";ffPlayer = new FFPlayer(url, OnVideoSize, OnVideoData, OAudioData);}byte[] data;// Update is called once per framevoid Update(){if (imgQueue.Count > 0){data = imgQueue.Dequeue();if (texture2D != null){texture2D.LoadRawTextureData(data);texture2D.Apply();}}}private void OnVideoSize(int width, int height){texture2D = new Texture2D(width, height, TextureFormat.RGB24, false);texture2D.Apply();rawImage.texture = texture2D;}private void OnVideoData(byte[] data){imgQueue.Enqueue(data);}private void OAudioData(byte[] data){audioQueue.Enqueue(data);}byte[] bsBuffer;private void OnAudioFilterRead(float[] data, int channels){if (audioQueue.Count > 0){bsBuffer = audioQueue.Dequeue();float[] _buffer = ByteArrayToFloatArray(bsBuffer, bsBuffer.Length);for (int i = 0; i < data.Length; i++){data[i] = _buffer[i];}}}public static float[] ByteArrayToFloatArray(byte[] byteArray, int length){float[] resultFloatArray = new float[length / 2];if (resultFloatArray == null || resultFloatArray.Length != (length / 2)){resultFloatArray = new float[length / 2];}int arrIdx = 0;for (int i = 0; i < length; i += 2){resultFloatArray[arrIdx++] = BytesToFloat(byteArray[i], byteArray[i + 1]);}return resultFloatArray;}static float BytesToFloat(byte firstByte, byte secondByte){return (float)((short)((int)secondByte << 8 | (int)firstByte)) / 32768f;}private void OnApplicationQuit(){ffPlayer.Dispose();}
 }
 

播放4K视频

最后是工程地址 https://gitee.com/awnuxcvbn/UnityFFmpeg