#if UNITY_EDITOR || UNITY_STANDALONE_OSX || UNITY_STANDALONE_WIN || UNITY_WSA_10_0 || UNITY_IOS || UNITY_ANDROID #define UNITY_PLATFORM_SUPPORTS_LINEAR #endif using System; using System.Collections.Generic; using UnityEngine; //----------------------------------------------------------------------------- // Copyright 2015-2021 RenderHeads Ltd. All rights reserved. //----------------------------------------------------------------------------- namespace RenderHeads.Media.AVProVideo { /// /// Base class for all platform specific MediaPlayers /// public abstract partial class BaseMediaPlayer : IMediaPlayer, IMediaControl, IMediaInfo, IMediaCache, ITextureProducer, IMediaSubtitles, IVideoTracks, IAudioTracks, ITextTracks, IBufferedDisplay, System.IDisposable { public BaseMediaPlayer() { InitTracks(); } public abstract string GetVersion(); public abstract string GetExpectedVersion(); /// public abstract bool OpenMedia(string path, long offset, string customHttpHeaders, MediaHints mediaHints, int forceFileFormat = 0, bool startWithHighestBitrate = false); #if NETFX_CORE /// public virtual bool OpenMedia(Windows.Storage.Streams.IRandomAccessStream ras, string path, long offset, string customHttpHeaders) { return false; } #endif /// public virtual bool OpenMediaFromBuffer(byte[] buffer) { return false; } /// public virtual bool StartOpenMediaFromBuffer(ulong length) { return false; } /// public virtual bool AddChunkToMediaBuffer(byte[] chunk, ulong offset, ulong length) { return false; } /// public virtual bool EndOpenMediaFromBuffer() { return false; } /// public virtual void CloseMedia() { #if UNITY_EDITOR _displayRateLastRealTime = 0f; #endif _displayRateTimer = 0f; _displayRateLastFrameCount = 0; _displayRate = 0f; _stallDetectionTimer = 0f; _stallDetectionFrame = 0; _lastError = ErrorCode.None; _textTracks.Clear(); _audioTracks.Clear(); _videoTracks.Clear(); _currentTextCue = null; _mediaHints = new MediaHints(); } /// public abstract void SetLooping(bool looping); /// public abstract bool IsLooping(); /// public abstract bool HasMetaData(); /// public abstract bool CanPlay(); /// public abstract void Play(); /// public abstract void Pause(); /// public abstract void Stop(); /// public virtual void Rewind() { SeekFast(0.0); } /// public abstract void Seek(double time); /// public abstract void SeekFast(double time); /// public virtual void SeekWithTolerance(double time, double timeDeltaBefore, double timeDeltaAfter) { Seek(time); } /// public abstract double GetCurrentTime(); /// public virtual DateTime GetProgramDateTime() { return DateTime.MinValue; } /// public abstract float GetPlaybackRate(); /// public abstract void SetPlaybackRate(float rate); // Basic Properties /// public abstract double GetDuration(); /// public abstract int GetVideoWidth(); /// public abstract int GetVideoHeight(); /// public abstract float GetVideoFrameRate(); /// public virtual float GetVideoDisplayRate() { return _displayRate; } /// public abstract bool HasAudio(); /// public abstract bool HasVideo(); /// public bool IsVideoStereo() { return GetTextureStereoPacking() != StereoPacking.None; } // Basic State /// public abstract bool IsSeeking(); /// public abstract bool IsPlaying(); /// public abstract bool IsPaused(); /// public abstract bool IsFinished(); /// public abstract bool IsBuffering(); /// public virtual bool WaitForNextFrame(Camera dummyCamera, int previousFrameCount) { return false; } // Textures /// public virtual int GetTextureCount() { return 1; } /// public abstract Texture GetTexture(int index = 0); /// public abstract int GetTextureFrameCount(); /// public virtual bool SupportsTextureFrameCount() { return true; } /// public virtual long GetTextureTimeStamp() { return long.MinValue; } /// public abstract bool RequiresVerticalFlip(); /// public virtual float GetTexturePixelAspectRatio() { return 1f; } /// public virtual Matrix4x4 GetYpCbCrTransform() { return Matrix4x4.identity; } /// public virtual float[] GetAffineTransform() { return new float[] { 1.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f }; } /// public virtual float[] GetTextureTransform() { return GetAffineTransform(); } /// public virtual Matrix4x4 GetTextureMatrix() { float[] transform = GetAffineTransform(); if (transform == null || transform.Length != 6) return Matrix4x4.identity; Vector4 v0 = new Vector4(transform[0], transform[1], 0, 0); Vector4 v1 = new Vector4(transform[2], transform[3], 0, 0); Vector4 v2 = new Vector4( 0, 0, 1, 0); Vector4 v3 = new Vector4(transform[4], transform[5], 0, 1); Matrix4x4 xfrm = new Matrix4x4(v0, v1, v2, v3); return xfrm; } public StereoPacking GetTextureStereoPacking() { StereoPacking result = InternalGetTextureStereoPacking(); if (result == StereoPacking.Unknown) { // If stereo is unknown, fall back to media hints or no packing result = _mediaHints.stereoPacking; } return result; } internal abstract StereoPacking InternalGetTextureStereoPacking(); public virtual TransparencyMode GetTextureTransparency() { return _mediaHints.transparency; } public AlphaPacking GetTextureAlphaPacking() { if (GetTextureTransparency() == TransparencyMode.Transparent) { return _mediaHints.alphaPacking; } return AlphaPacking.None; } // Audio General /// public abstract void MuteAudio(bool bMuted); /// public abstract bool IsMuted(); /// public abstract void SetVolume(float volume); /// public virtual void SetBalance(float balance) { } /// public abstract float GetVolume(); /// public virtual float GetBalance() { return 0f; } // Audio Grabbing /// public virtual int GetAudioChannelCount() { return -1; } /// public virtual AudioChannelMaskFlags GetAudioChannelMask() { return 0; } /// public virtual int GrabAudio(float[] audioData, int audioDataFloatCount, int channelCount) { return 0; } /// public virtual int GetAudioBufferedSampleCount() { return 0; } /// public virtual void AudioConfigurationChanged(bool deviceChanged) { } // 360 Audio /// public virtual void SetAudioHeadRotation(Quaternion q) { } /// public virtual void ResetAudioHeadRotation() { } /// public virtual void SetAudioChannelMode(Audio360ChannelMode channelMode) { } /// public virtual void SetAudioFocusEnabled(bool enabled) { } /// public virtual void SetAudioFocusProperties(float offFocusLevel, float widthDegrees) { } /// public virtual void SetAudioFocusRotation(Quaternion q) { } /// public virtual void ResetAudioFocus() { } // Streaming /// public virtual long GetEstimatedTotalBandwidthUsed() { return -1; } /// public virtual void SetPlayWithoutBuffering(bool playWithoutBuffering) { } // Caching /// public virtual bool IsMediaCachingSupported() { return false; } /// public virtual void AddMediaToCache(string url, string headers, MediaCachingOptions options) { } /// public virtual void CancelDownloadOfMediaToCache(string url) { } /// public virtual void PauseDownloadOfMediaToCache(string url) { } /// public virtual void ResumeDownloadOfMediaToCache(string url) { } /// public virtual void RemoveMediaFromCache(string url) { } /// public virtual CachedMediaStatus GetCachedMediaStatus(string url, ref float progress) { return CachedMediaStatus.NotCached; } // /// // public virtual bool IsMediaCached() { return false; } // External playback /// public virtual bool IsExternalPlaybackSupported() { return false; } /// public virtual bool IsExternalPlaybackActive() { return false; } /// public virtual void SetAllowsExternalPlayback(bool enable) { } /// public virtual void SetExternalPlaybackVideoGravity(ExternalPlaybackVideoGravity gravity) { } // Authentication //public virtual void SetKeyServerURL(string url) { } /// public virtual void SetKeyServerAuthToken(string token) { } /// public virtual void SetOverrideDecryptionKey(byte[] key) { } // General /// public abstract void Update(); /// public /*abstract*/virtual void BeginRender() { } /// public abstract void Render(); /// public abstract void Dispose(); // Internal method public virtual bool GetDecoderPerformance(ref int activeDecodeThreadCount, ref int decodedFrameCount, ref int droppedFrameCount) { return false; } #if false public void Update() { Native.Update(_instance); if (UpdateTracks()) { } if (UpdateTextCue()) { } } #endif public virtual void EndUpdate() { } public virtual IntPtr GetNativePlayerHandle() { return IntPtr.Zero; } public ErrorCode GetLastError() { ErrorCode errorCode = _lastError; _lastError = ErrorCode.None; return errorCode; } /// public virtual long GetLastExtendedErrorCode() { return 0; } public string GetPlayerDescription() { return _playerDescription; } /// public virtual bool PlayerSupportsLinearColorSpace() { #if UNITY_PLATFORM_SUPPORTS_LINEAR return true; #else return false; #endif } protected string _playerDescription = string.Empty; protected ErrorCode _lastError = ErrorCode.None; protected FilterMode _defaultTextureFilterMode = FilterMode.Bilinear; protected TextureWrapMode _defaultTextureWrapMode = TextureWrapMode.Clamp; protected int _defaultTextureAnisoLevel = 1; protected MediaHints _mediaHints; protected TimeRanges _seekableTimes = new TimeRanges(); protected TimeRanges _bufferedTimes = new TimeRanges(); public TimeRanges GetSeekableTimes() { return _seekableTimes; } public TimeRanges GetBufferedTimes() { return _bufferedTimes; } public void GetTextureProperties(out FilterMode filterMode, out TextureWrapMode wrapMode, out int anisoLevel) { filterMode = _defaultTextureFilterMode; wrapMode = _defaultTextureWrapMode; anisoLevel = _defaultTextureAnisoLevel; } public void SetTextureProperties(FilterMode filterMode = FilterMode.Bilinear, TextureWrapMode wrapMode = TextureWrapMode.Clamp, int anisoLevel = 0) { _defaultTextureFilterMode = filterMode; _defaultTextureWrapMode = wrapMode; _defaultTextureAnisoLevel = anisoLevel; for (int i = 0; i < GetTextureCount(); ++i) { ApplyTextureProperties(GetTexture(i)); } } protected virtual void ApplyTextureProperties(Texture texture) { if (texture != null) { texture.filterMode = _defaultTextureFilterMode; texture.wrapMode = _defaultTextureWrapMode; texture.anisoLevel = _defaultTextureAnisoLevel; } } #region Video Display Rate #if UNITY_EDITOR private float _displayRateLastRealTime = 0f; #endif private float _displayRateTimer; private int _displayRateLastFrameCount; private float _displayRate = 1f; protected void UpdateDisplayFrameRate() { const float IntervalSeconds = 0.5f; if (_displayRateTimer >= IntervalSeconds) { int frameCount = GetTextureFrameCount(); int frameDelta = (frameCount - _displayRateLastFrameCount); _displayRate = (float)frameDelta / _displayRateTimer; _displayRateTimer -= IntervalSeconds; if (_displayRateTimer >= IntervalSeconds) _displayRateTimer -= IntervalSeconds; if (_displayRateTimer >= IntervalSeconds) _displayRateTimer = 0f; _displayRateLastFrameCount = frameCount; } float deltaTime = Time.deltaTime; #if UNITY_EDITOR if (!Application.isPlaying) { // When not playing Time.deltaTime isn't valid so we have to derive it deltaTime = (Time.realtimeSinceStartup - _displayRateLastRealTime); _displayRateLastRealTime = Time.realtimeSinceStartup; } #endif _displayRateTimer += deltaTime; } #endregion // Video Display Rate #region Stall Detection protected bool IsExpectingNewVideoFrame() { if (HasVideo()) { // If we're playing then we expect a new frame if (!IsFinished() && (!IsPaused() && IsPlaying() && GetPlaybackRate() != 0.0f)) { // Check that the video is not a single frame and therefore there is no other frame to display bool isSingleFrame = (GetTextureFrameCount() > 0 && GetDurationFrames() == 1); if (!isSingleFrame) { // NOTE: if a new frame isn't available then we could either be seeking or stalled return true; } } } return false; } /// public virtual bool IsPlaybackStalled() { const float StallDetectionDuration = 0.5f; // Manually detect stalled video if the platform doesn't have native support to detect it if (SupportsTextureFrameCount() && IsExpectingNewVideoFrame()) { // Detect a new video frame int frameCount = GetTextureFrameCount(); if (frameCount != _stallDetectionFrame) { _stallDetectionTimer = 0f; _stallDetectionFrame = frameCount; } else { // Update the detection timer, but never more than once a Unity frame if (_stallDetectionGuard != Time.frameCount) { _stallDetectionTimer += Time.deltaTime; } } _stallDetectionGuard = Time.frameCount; float thresholdDuration = StallDetectionDuration; // Scale by the playback rate, but should be at least StallDetectionDuration thresholdDuration = Mathf.Max(thresholdDuration / Mathf.Abs(GetPlaybackRate()), StallDetectionDuration); // If a valid FPS is available then make sure the thresholdDuration // is at least double that. This is mainly for very low FPS // content (eg 1 or 2 FPS) float fps = GetVideoFrameRate(); if (fps > 0f && !float.IsNaN(fps)) { thresholdDuration = Mathf.Max(thresholdDuration, 2f / fps); } return (_stallDetectionTimer > thresholdDuration); } else { _stallDetectionTimer = 0f; } return false; } private float _stallDetectionTimer; private int _stallDetectionFrame; private int _stallDetectionGuard; #endregion // Stall Detection protected List _subtitles; protected Subtitle _currentSubtitle; /// public bool LoadSubtitlesSRT(string data) { if (string.IsNullOrEmpty(data)) { // Disable subtitles _subtitles = null; _currentSubtitle = null; } else { _subtitles = SubtitleUtils.ParseSubtitlesSRT(data); _currentSubtitle = null; } return (_subtitles != null); } /// public virtual void UpdateSubtitles() { if (_subtitles != null) { double time = GetCurrentTime(); // TODO: implement a more efficient subtitle index searcher int searchIndex = 0; if (_currentSubtitle != null) { if (!_currentSubtitle.IsTime(time)) { if (time > _currentSubtitle.timeEnd) { searchIndex = _currentSubtitle.index + 1; } _currentSubtitle = null; } } if (_currentSubtitle == null) { for (int i = searchIndex; i < _subtitles.Count; i++) { if (_subtitles[i].IsTime(time)) { _currentSubtitle = _subtitles[i]; break; } } } } } /// public virtual int GetSubtitleIndex() { int result = -1; if (_currentSubtitle != null) { result = _currentSubtitle.index; } return result; } /// public virtual string GetSubtitleText() { string result = string.Empty; if (_currentSubtitle != null) { result = _currentSubtitle.text; } else if (_currentTextCue != null) { result = _currentTextCue.Text; } return result; } public virtual void OnEnable() { } /// public int GetCurrentTimeFrames(float overrideFrameRate = 0f) { int result = 0; float frameRate = (overrideFrameRate > 0f) ? overrideFrameRate : GetVideoFrameRate(); if (frameRate > 0f) { result = Helper.ConvertTimeSecondsToFrame(GetCurrentTime(), frameRate); result = Mathf.Min(result, GetMaxFrameNumber(overrideFrameRate)); } return result; } /// public int GetDurationFrames(float overrideFrameRate = 0f) { int result = 0; float frameRate = (overrideFrameRate > 0f) ? overrideFrameRate : GetVideoFrameRate(); if (frameRate > 0f) { result = Helper.ConvertTimeSecondsToFrame(GetDuration(), frameRate); } return result; } /// public int GetMaxFrameNumber(float overrideFrameRate = 0f) { int result = GetDurationFrames(overrideFrameRate); result = Mathf.Max(0, result - 1); return result; } /// public void SeekToFrameRelative(int frameOffset, float overrideFrameRate = 0f) { float frameRate = (overrideFrameRate > 0f)?overrideFrameRate:GetVideoFrameRate(); if (frameRate > 0f) { int frame = Helper.ConvertTimeSecondsToFrame(GetCurrentTime(), frameRate); frame += frameOffset; frame = Mathf.Clamp(frame, 0, GetMaxFrameNumber(frameRate)); double time = Helper.ConvertFrameToTimeSeconds(frame, frameRate); Seek(time); } } /// public void SeekToFrame(int frame, float overrideFrameRate = 0f) { float frameRate = (overrideFrameRate > 0f)?overrideFrameRate:GetVideoFrameRate(); if (frameRate > 0f) { frame = Mathf.Clamp(frame, 0, GetMaxFrameNumber(frameRate)); double time = Helper.ConvertFrameToTimeSeconds(frame, frameRate); Seek(time); } } #region IBufferedDisplay Implementation private int _unityFrameCountBufferedDisplayGuard = -1; /// public long UpdateBufferedDisplay() { // Guard to make sure we're only updating the buffered frame once per Unity frame if (Time.frameCount == _unityFrameCountBufferedDisplayGuard) return GetTextureTimeStamp(); _unityFrameCountBufferedDisplayGuard = Time.frameCount; return InternalUpdateBufferedDisplay(); } internal virtual long InternalUpdateBufferedDisplay() { return 0; } /// public virtual BufferedFramesState GetBufferedFramesState() { return new BufferedFramesState(); } /// public virtual void SetSlaves(IBufferedDisplay[] slaves) { } /// public virtual void SetBufferedDisplayMode(BufferedFrameSelectionMode mode, IBufferedDisplay master = null) { } /// public virtual void SetBufferedDisplayOptions(bool pauseOnPrerollComplete) { } #endregion // IBufferedDisplay Implementation protected PlaybackQualityStats _playbackQualityStats = new PlaybackQualityStats(); public PlaybackQualityStats GetPlaybackQualityStats() { return _playbackQualityStats; } } }