595 lines
15 KiB
C#
Executable File
595 lines
15 KiB
C#
Executable File
using System.Collections;
|
|
using System.Collections.Generic;
|
|
using UnityEngine;
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// Copyright 2015-2021 RenderHeads Ltd. All rights reserved.
|
|
//-----------------------------------------------------------------------------
|
|
|
|
namespace RenderHeads.Media.AVProVideo
|
|
{
|
|
/// <summary>
|
|
/// Utility class to resample MediaPlayer video frames to allow for smoother playback
|
|
/// Keeps a buffer of frames with timestamps and presents them using its own clock
|
|
/// </summary>
|
|
public class Resampler
|
|
{
|
|
private class TimestampedRenderTexture
|
|
{
|
|
public RenderTexture texture = null;
|
|
public long timestamp = 0;
|
|
public bool used = false;
|
|
}
|
|
|
|
public enum ResampleMode
|
|
{
|
|
POINT, LINEAR
|
|
}
|
|
|
|
private List<TimestampedRenderTexture[]> _buffer = new List<TimestampedRenderTexture[]>();
|
|
private MediaPlayer _mediaPlayer;
|
|
private RenderTexture[] _outputTexture = null;
|
|
|
|
private int _start = 0;
|
|
private int _end = 0;
|
|
private int _bufferSize = 0;
|
|
|
|
private long _baseTimestamp = 0;
|
|
private float _elapsedTimeSinceBase = 0f;
|
|
|
|
private Material _blendMat;
|
|
|
|
private ResampleMode _resampleMode;
|
|
private string _name = "";
|
|
|
|
private long _lastTimeStamp = -1;
|
|
|
|
private int _droppedFrames = 0;
|
|
|
|
private long _lastDisplayedTimestamp = 0;
|
|
private int _frameDisplayedTimer = 0;
|
|
private long _currentDisplayedTimestamp = 0;
|
|
|
|
public int DroppedFrames
|
|
{
|
|
get { return _droppedFrames; }
|
|
}
|
|
|
|
public int FrameDisplayedTimer
|
|
{
|
|
get { return _frameDisplayedTimer; }
|
|
}
|
|
|
|
public long BaseTimestamp
|
|
{
|
|
get { return _baseTimestamp; }
|
|
set { _baseTimestamp = value; }
|
|
}
|
|
|
|
public float ElapsedTimeSinceBase
|
|
{
|
|
get { return _elapsedTimeSinceBase; }
|
|
set { _elapsedTimeSinceBase = value; }
|
|
}
|
|
|
|
public float LastT
|
|
{
|
|
get; private set;
|
|
}
|
|
|
|
public long TextureTimeStamp
|
|
{
|
|
get; private set;
|
|
}
|
|
|
|
private const string ShaderPropT = "_t";
|
|
private const string ShaderPropAftertex = "_AfterTex";
|
|
private int _propAfterTex;
|
|
private int _propT;
|
|
private float _videoFrameRate;
|
|
|
|
public void OnVideoEvent(MediaPlayer mp, MediaPlayerEvent.EventType et, ErrorCode errorCode)
|
|
{
|
|
switch (et)
|
|
{
|
|
case MediaPlayerEvent.EventType.MetaDataReady:
|
|
_videoFrameRate = mp.Info.GetVideoFrameRate();
|
|
_elapsedTimeSinceBase = 0f;
|
|
if (_videoFrameRate > 0f)
|
|
{
|
|
_elapsedTimeSinceBase = _bufferSize / _videoFrameRate;
|
|
}
|
|
break;
|
|
case MediaPlayerEvent.EventType.Closing:
|
|
Reset();
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
public Resampler(MediaPlayer player, string name, int bufferSize = 2, ResampleMode resampleMode = ResampleMode.LINEAR)
|
|
{
|
|
_bufferSize = Mathf.Max(2, bufferSize);
|
|
|
|
player.Events.AddListener(OnVideoEvent);
|
|
|
|
_mediaPlayer = player;
|
|
|
|
Shader blendShader = Shader.Find("AVProVideo/Internal/BlendFrames");
|
|
if (blendShader != null)
|
|
{
|
|
_blendMat = new Material(blendShader);
|
|
_propT = Shader.PropertyToID(ShaderPropT);
|
|
_propAfterTex = Shader.PropertyToID(ShaderPropAftertex);
|
|
}
|
|
else
|
|
{
|
|
Debug.LogError("[AVProVideo] Failed to find BlendFrames shader");
|
|
}
|
|
|
|
_resampleMode = resampleMode;
|
|
_name = name;
|
|
|
|
Debug.Log("[AVProVideo] Resampler " + _name + " started");
|
|
}
|
|
|
|
public Texture[] OutputTexture
|
|
{
|
|
get { return _outputTexture; }
|
|
}
|
|
|
|
public void Reset()
|
|
{
|
|
_lastTimeStamp = -1;
|
|
_baseTimestamp = 0;
|
|
InvalidateBuffer();
|
|
}
|
|
|
|
public void Release()
|
|
{
|
|
ReleaseRenderTextures();
|
|
if (_blendMat != null)
|
|
{
|
|
if (Application.isPlaying)
|
|
{
|
|
Material.Destroy(_blendMat);
|
|
}
|
|
else
|
|
{
|
|
Material.DestroyImmediate(_blendMat);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void ReleaseRenderTextures()
|
|
{
|
|
for (int i = 0; i < _buffer.Count; ++i)
|
|
{
|
|
for (int j = 0; j < _buffer[i].Length; ++j)
|
|
{
|
|
if (_buffer[i][j].texture != null)
|
|
{
|
|
RenderTexture.ReleaseTemporary(_buffer[i][j].texture);
|
|
_buffer[i][j].texture = null;
|
|
}
|
|
}
|
|
|
|
if (_outputTexture != null && _outputTexture[i] != null)
|
|
{
|
|
RenderTexture.ReleaseTemporary(_outputTexture[i]);
|
|
}
|
|
}
|
|
|
|
_outputTexture = null;
|
|
}
|
|
|
|
private void ConstructRenderTextures()
|
|
{
|
|
ReleaseRenderTextures();
|
|
_buffer.Clear();
|
|
|
|
_outputTexture = new RenderTexture[_mediaPlayer.TextureProducer.GetTextureCount()];
|
|
|
|
for (int i = 0; i < _mediaPlayer.TextureProducer.GetTextureCount(); ++i)
|
|
{
|
|
Texture tex = _mediaPlayer.TextureProducer.GetTexture(i);
|
|
_buffer.Add(new TimestampedRenderTexture[_bufferSize]);
|
|
for (int j = 0; j < _bufferSize; ++j)
|
|
{
|
|
_buffer[i][j] = new TimestampedRenderTexture();
|
|
}
|
|
|
|
for (int j = 0; j < _buffer[i].Length; ++j)
|
|
{
|
|
_buffer[i][j].texture = RenderTexture.GetTemporary(tex.width, tex.height, 0);
|
|
_buffer[i][j].timestamp = 0;
|
|
_buffer[i][j].used = false;
|
|
}
|
|
|
|
_outputTexture[i] = RenderTexture.GetTemporary(tex.width, tex.height, 0);
|
|
_outputTexture[i].filterMode = tex.filterMode;
|
|
_outputTexture[i].wrapMode = tex.wrapMode;
|
|
_outputTexture[i].anisoLevel = tex.anisoLevel;
|
|
// TODO: set up the mips level too?
|
|
}
|
|
}
|
|
|
|
private bool CheckRenderTexturesValid()
|
|
{
|
|
for (int i = 0; i < _mediaPlayer.TextureProducer.GetTextureCount(); ++i)
|
|
{
|
|
Texture tex = _mediaPlayer.TextureProducer.GetTexture(i);
|
|
for (int j = 0; j < _buffer.Count; ++j)
|
|
{
|
|
if (_buffer[i][j].texture == null || _buffer[i][j].texture.width != tex.width || _buffer[i][j].texture.height != tex.height)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
if (_outputTexture == null || _outputTexture[i] == null || _outputTexture[i].width != tex.width || _outputTexture[i].height != tex.height)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
//finds closest frame that occurs before given index
|
|
private int FindBeforeFrameIndex(int frameIdx)
|
|
{
|
|
if (frameIdx >= _buffer.Count)
|
|
{
|
|
return -1;
|
|
}
|
|
|
|
int foundFrame = -1;
|
|
float smallestDif = float.MaxValue;
|
|
int closest = -1;
|
|
float smallestElapsed = float.MaxValue;
|
|
|
|
for (int i = 0; i < _buffer[frameIdx].Length; ++i)
|
|
{
|
|
if (_buffer[frameIdx][i].used)
|
|
{
|
|
float elapsed = (_buffer[frameIdx][i].timestamp - _baseTimestamp) / 10000000f;
|
|
|
|
//keep track of closest after frame, just in case no before frame was found
|
|
if (elapsed < smallestElapsed)
|
|
{
|
|
closest = i;
|
|
smallestElapsed = elapsed;
|
|
}
|
|
|
|
float dif = _elapsedTimeSinceBase - elapsed;
|
|
|
|
if (dif >= 0 && dif < smallestDif)
|
|
{
|
|
smallestDif = dif;
|
|
foundFrame = i;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (foundFrame < 0)
|
|
{
|
|
if (closest < 0)
|
|
{
|
|
return -1;
|
|
}
|
|
|
|
return closest;
|
|
}
|
|
|
|
return foundFrame;
|
|
}
|
|
|
|
private int FindClosestFrame(int frameIdx)
|
|
{
|
|
if (frameIdx >= _buffer.Count)
|
|
{
|
|
return -1;
|
|
}
|
|
|
|
int foundPos = -1;
|
|
float smallestDif = float.MaxValue;
|
|
|
|
for (int i = 0; i < _buffer[frameIdx].Length; ++i)
|
|
{
|
|
if (_buffer[frameIdx][i].used)
|
|
{
|
|
float elapsed = (_buffer[frameIdx][i].timestamp - _baseTimestamp) / 10000000f;
|
|
float dif = Mathf.Abs(_elapsedTimeSinceBase - elapsed);
|
|
if (dif < smallestDif)
|
|
{
|
|
foundPos = i;
|
|
smallestDif = dif;
|
|
}
|
|
}
|
|
}
|
|
|
|
return foundPos;
|
|
}
|
|
|
|
//point update selects closest frame and uses that as output
|
|
private void PointUpdate()
|
|
{
|
|
for (int i = 0; i < _buffer.Count; ++i)
|
|
{
|
|
int frameIndex = FindClosestFrame(i);
|
|
if (frameIndex < 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
_outputTexture[i].DiscardContents();
|
|
Graphics.Blit(_buffer[i][frameIndex].texture, _outputTexture[i]);
|
|
TextureTimeStamp = _currentDisplayedTimestamp = _buffer[i][frameIndex].timestamp;
|
|
}
|
|
|
|
}
|
|
|
|
//Updates currently displayed frame
|
|
private void SampleFrame(int frameIdx, int bufferIdx)
|
|
{
|
|
_outputTexture[bufferIdx].DiscardContents();
|
|
Graphics.Blit(_buffer[bufferIdx][frameIdx].texture, _outputTexture[bufferIdx]);
|
|
TextureTimeStamp = _currentDisplayedTimestamp = _buffer[bufferIdx][frameIdx].timestamp;
|
|
}
|
|
|
|
//Same as sample frame, but does a lerp of the two given frames and outputs that image instead
|
|
private void SampleFrames(int bufferIdx, int frameIdx1, int frameIdx2, float t)
|
|
{
|
|
_blendMat.SetFloat(_propT, t);
|
|
_blendMat.SetTexture(_propAfterTex, _buffer[bufferIdx][frameIdx2].texture);
|
|
_outputTexture[bufferIdx].DiscardContents();
|
|
Graphics.Blit(_buffer[bufferIdx][frameIdx1].texture, _outputTexture[bufferIdx], _blendMat);
|
|
TextureTimeStamp = (long)Mathf.Lerp(_buffer[bufferIdx][frameIdx1].timestamp, _buffer[bufferIdx][frameIdx2].timestamp, t);
|
|
_currentDisplayedTimestamp = _buffer[bufferIdx][frameIdx1].timestamp;
|
|
}
|
|
|
|
private void LinearUpdate()
|
|
{
|
|
for (int i = 0; i < _buffer.Count; ++i)
|
|
{
|
|
//find closest frame
|
|
int frameIndex = FindBeforeFrameIndex(i);
|
|
|
|
//no valid frame, this should never ever happen actually...
|
|
if (frameIndex < 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
//resample or just use last frame and set current elapsed time to that frame
|
|
float frameElapsed = (_buffer[i][frameIndex].timestamp - _baseTimestamp) / 10000000f;
|
|
if (frameElapsed > _elapsedTimeSinceBase)
|
|
{
|
|
SampleFrame(frameIndex, i);
|
|
LastT = -1f;
|
|
}
|
|
else
|
|
{
|
|
int next = (frameIndex + 1) % _buffer[i].Length;
|
|
float nextElapsed = (_buffer[i][next].timestamp - _baseTimestamp) / 10000000f;
|
|
|
|
//no larger frame, move elapsed time back a bit since we cant predict the future
|
|
if (nextElapsed < frameElapsed)
|
|
{
|
|
SampleFrame(frameIndex, i);
|
|
LastT = 2f;
|
|
}
|
|
//have a before and after frame, interpolate
|
|
else
|
|
{
|
|
|
|
float range = nextElapsed - frameElapsed;
|
|
float t = (_elapsedTimeSinceBase - frameElapsed) / range;
|
|
SampleFrames(i, frameIndex, next, t);
|
|
LastT = t;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private void InvalidateBuffer()
|
|
{
|
|
_elapsedTimeSinceBase = (_bufferSize / 2) / _videoFrameRate;
|
|
|
|
for (int i = 0; i < _buffer.Count; ++i)
|
|
{
|
|
for (int j = 0; j < _buffer[i].Length; ++j)
|
|
{
|
|
_buffer[i][j].used = false;
|
|
}
|
|
}
|
|
|
|
_start = _end = 0;
|
|
}
|
|
|
|
private float GuessFrameRate()
|
|
{
|
|
int fpsCount = 0;
|
|
long fps = 0;
|
|
|
|
for (int k = 0; k < _buffer[0].Length; k++)
|
|
{
|
|
if (_buffer[0][k].used)
|
|
{
|
|
// Find the pair with the smallest difference
|
|
long smallestDiff = long.MaxValue;
|
|
for (int j = k + 1; j < _buffer[0].Length; j++)
|
|
{
|
|
if (_buffer[0][j].used)
|
|
{
|
|
long diff = System.Math.Abs(_buffer[0][k].timestamp - _buffer[0][j].timestamp);
|
|
if (diff < smallestDiff)
|
|
{
|
|
smallestDiff = diff;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (smallestDiff != long.MaxValue)
|
|
{
|
|
fps += smallestDiff;
|
|
fpsCount++;
|
|
}
|
|
}
|
|
}
|
|
if (fpsCount > 1)
|
|
{
|
|
fps /= fpsCount;
|
|
}
|
|
return 10000000f / (float)fps;
|
|
}
|
|
|
|
public void Update()
|
|
{
|
|
if (_mediaPlayer.TextureProducer == null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
//recreate textures if invalid
|
|
if (_mediaPlayer.TextureProducer == null || _mediaPlayer.TextureProducer.GetTexture() == null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (!CheckRenderTexturesValid())
|
|
{
|
|
ConstructRenderTextures();
|
|
}
|
|
|
|
long currentTimestamp = _mediaPlayer.TextureProducer.GetTextureTimeStamp();
|
|
|
|
//if frame has been updated, do a calculation to estimate dropped frames
|
|
if (currentTimestamp != _lastTimeStamp)
|
|
{
|
|
float dif = Mathf.Abs(currentTimestamp - _lastTimeStamp);
|
|
float frameLength = (10000000f / _videoFrameRate);
|
|
if (dif > frameLength * 1.1f && dif < frameLength * 3.1f)
|
|
{
|
|
_droppedFrames += (int)((dif - frameLength) / frameLength + 0.5);
|
|
}
|
|
_lastTimeStamp = currentTimestamp;
|
|
}
|
|
|
|
//Adding texture to buffer logic
|
|
long timestamp = _mediaPlayer.TextureProducer.GetTextureTimeStamp();
|
|
bool insertNewFrame = !_mediaPlayer.Control.IsSeeking();
|
|
//if buffer is not empty, we need to check if we need to reject the new frame
|
|
if (_start != _end || _buffer[0][_end].used)
|
|
{
|
|
int lastFrame = (_end + _buffer[0].Length - 1) % _buffer[0].Length;
|
|
//frame is not new and thus we do not need to store it
|
|
if (timestamp == _buffer[0][lastFrame].timestamp)
|
|
{
|
|
insertNewFrame = false;
|
|
}
|
|
}
|
|
|
|
bool bufferWasNotFull = (_start != _end) || (!_buffer[0][_end].used);
|
|
|
|
if (insertNewFrame)
|
|
{
|
|
//buffer empty, reset base timestamp to current
|
|
if (_start == _end && !_buffer[0][_end].used)
|
|
{
|
|
_baseTimestamp = timestamp;
|
|
}
|
|
|
|
//update buffer counters, if buffer is full, we get rid of the earliest frame by incrementing the start counter
|
|
if (_end == _start && _buffer[0][_end].used)
|
|
{
|
|
_start = (_start + 1) % _buffer[0].Length;
|
|
}
|
|
|
|
for (int i = 0; i < _mediaPlayer.TextureProducer.GetTextureCount(); ++i)
|
|
{
|
|
Texture currentTexture = _mediaPlayer.TextureProducer.GetTexture(i);
|
|
|
|
//store frame info
|
|
_buffer[i][_end].texture.DiscardContents();
|
|
Graphics.Blit(currentTexture, _buffer[i][_end].texture);
|
|
_buffer[i][_end].timestamp = timestamp;
|
|
_buffer[i][_end].used = true;
|
|
}
|
|
|
|
_end = (_end + 1) % _buffer[0].Length;
|
|
}
|
|
|
|
bool bufferNotFull = (_start != _end) || (!_buffer[0][_end].used);
|
|
|
|
if (bufferNotFull)
|
|
{
|
|
for (int i = 0; i < _buffer.Count; ++i)
|
|
{
|
|
_outputTexture[i].DiscardContents();
|
|
Graphics.Blit(_buffer[i][_start].texture, _outputTexture[i]);
|
|
_currentDisplayedTimestamp = _buffer[i][_start].timestamp;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// If we don't have a valid frame rate and the buffer is now full, guess the frame rate by looking at the buffered timestamps
|
|
if (bufferWasNotFull && _videoFrameRate <= 0f)
|
|
{
|
|
_videoFrameRate = GuessFrameRate();
|
|
_elapsedTimeSinceBase = (_bufferSize / 2) / _videoFrameRate;
|
|
}
|
|
}
|
|
|
|
if (_mediaPlayer.Control.IsPaused())
|
|
{
|
|
InvalidateBuffer();
|
|
}
|
|
|
|
//we always wait until buffer is full before display things, just assign first frame in buffer to output so that the user can see something
|
|
if (bufferNotFull)
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (_mediaPlayer.Control.IsPlaying() && !_mediaPlayer.Control.IsFinished())
|
|
{
|
|
//correct elapsed time if too far out
|
|
long ts = _buffer[0][(_start + _bufferSize / 2) % _bufferSize].timestamp - _baseTimestamp;
|
|
double dif = Mathf.Abs(((float)((double)_elapsedTimeSinceBase * 10000000) - ts));
|
|
double threshold = (_buffer[0].Length / 2) / _videoFrameRate * 10000000;
|
|
|
|
if (dif > threshold)
|
|
{
|
|
_elapsedTimeSinceBase = ts / 10000000f;
|
|
}
|
|
|
|
if (_resampleMode == ResampleMode.POINT)
|
|
{
|
|
PointUpdate();
|
|
}
|
|
else if (_resampleMode == ResampleMode.LINEAR)
|
|
{
|
|
LinearUpdate();
|
|
}
|
|
|
|
_elapsedTimeSinceBase += Time.unscaledDeltaTime;
|
|
}
|
|
}
|
|
|
|
public void UpdateTimestamp()
|
|
{
|
|
if (_lastDisplayedTimestamp != _currentDisplayedTimestamp)
|
|
{
|
|
_lastDisplayedTimestamp = _currentDisplayedTimestamp;
|
|
_frameDisplayedTimer = 0;
|
|
}
|
|
_frameDisplayedTimer++;
|
|
}
|
|
}
|
|
}
|
|
|