OpenShot Library | libopenshot  0.2.5
FFmpegReader.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard
12  * (http://www.openshotstudios.com). This file is part of
13  * OpenShot Library (http://www.openshot.org), an open-source project
14  * dedicated to delivering high quality video editing and animation solutions
15  * to the world.
16  *
17  * This file is originally based on the Libavformat API example, and then modified
18  * by the libopenshot project.
19  *
20  * OpenShot Library (libopenshot) is free software: you can redistribute it
21  * and/or modify it under the terms of the GNU Lesser General Public License
22  * as published by the Free Software Foundation, either version 3 of the
23  * License, or (at your option) any later version.
24  *
25  * OpenShot Library (libopenshot) is distributed in the hope that it will be
26  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28  * GNU Lesser General Public License for more details.
29  *
30  * You should have received a copy of the GNU Lesser General Public License
31  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
32  */
33 
34 #include "FFmpegReader.h"
35 #include "Exceptions.h"
36 
37 #include <thread> // for std::this_thread::sleep_for
38 #include <chrono> // for std::chrono::milliseconds
39 
40 #define ENABLE_VAAPI 0
41 
42 #if HAVE_HW_ACCEL
43 #pragma message "You are compiling with experimental hardware decode"
44 #else
45 #pragma message "You are compiling only with software decode"
46 #endif
47 
48 #if HAVE_HW_ACCEL
49 #define MAX_SUPPORTED_WIDTH 1950
50 #define MAX_SUPPORTED_HEIGHT 1100
51 
52 #if ENABLE_VAAPI
53 #include "libavutil/hwcontext_vaapi.h"
54 
55 typedef struct VAAPIDecodeContext {
56  VAProfile va_profile;
57  VAEntrypoint va_entrypoint;
58  VAConfigID va_config;
59  VAContextID va_context;
60 
61 #if FF_API_STRUCT_VAAPI_CONTEXT
62  // FF_DISABLE_DEPRECATION_WARNINGS
63  int have_old_context;
64  struct vaapi_context *old_context;
65  AVBufferRef *device_ref;
66  // FF_ENABLE_DEPRECATION_WARNINGS
67 #endif
68 
69  AVHWDeviceContext *device;
70  AVVAAPIDeviceContext *hwctx;
71 
72  AVHWFramesContext *frames;
73  AVVAAPIFramesContext *hwfc;
74 
75  enum AVPixelFormat surface_format;
76  int surface_count;
77  } VAAPIDecodeContext;
78 #endif // ENABLE_VAAPI
79 #endif // HAVE_HW_ACCEL
80 
81 
82 using namespace openshot;
83 
84 int hw_de_on = 0;
85 #if HAVE_HW_ACCEL
86  AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
87  AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
88 #endif
89 
90 FFmpegReader::FFmpegReader(const std::string& path, bool inspect_reader)
91  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
92  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
93  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
94  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
95  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
96  packet(NULL), max_concurrent_frames(OPEN_MP_NUM_PROCESSORS) {
97 
98  // Initialize FFMpeg, and register all formats and codecs
101 
102  // Init cache
103  working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels);
104  missing_frames.SetMaxBytesFromInfo(max_concurrent_frames * 2, info.width, info.height, info.sample_rate, info.channels);
106 
107  // Open and Close the reader, to populate its attributes (such as height, width, etc...)
108  if (inspect_reader) {
109  Open();
110  Close();
111  }
112 }
113 
115  if (is_open)
116  // Auto close reader if not already done
117  Close();
118 }
119 
120 // This struct holds the associated video frame and starting sample # for an audio packet.
121 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) {
122  // Is frame even close to this one?
123  if (abs(location.frame - frame) >= 2)
124  // This is too far away to be considered
125  return false;
126 
127  // Note that samples_per_frame can vary slightly frame to frame when the
128  // audio sampling rate is not an integer multiple of the video fps.
129  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
130  if (abs(diff) <= amount)
131  // close
132  return true;
133 
134  // not close
135  return false;
136 }
137 
138 #if HAVE_HW_ACCEL
139 
140 // Get hardware pix format
141 static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
142 {
143  const enum AVPixelFormat *p;
144 
145  for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
146  switch (*p) {
147 #if defined(__linux__)
148  // Linux pix formats
149  case AV_PIX_FMT_VAAPI:
150  hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI;
151  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI;
152  return *p;
153  break;
154  case AV_PIX_FMT_VDPAU:
155  hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU;
156  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU;
157  return *p;
158  break;
159 #endif
160 #if defined(_WIN32)
161  // Windows pix formats
162  case AV_PIX_FMT_DXVA2_VLD:
163  hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD;
164  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2;
165  return *p;
166  break;
167  case AV_PIX_FMT_D3D11:
168  hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11;
169  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA;
170  return *p;
171  break;
172 #endif
173 #if defined(__APPLE__)
174  // Apple pix formats
175  case AV_PIX_FMT_VIDEOTOOLBOX:
176  hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX;
177  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
178  return *p;
179  break;
180 #endif
181  // Cross-platform pix formats
182  case AV_PIX_FMT_CUDA:
183  hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA;
184  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA;
185  return *p;
186  break;
187  case AV_PIX_FMT_QSV:
188  hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV;
189  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV;
190  return *p;
191  break;
192  default:
193  // This is only here to silence unused-enum warnings
194  break;
195  }
196  }
197  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)");
198  return AV_PIX_FMT_NONE;
199 }
200 
201 int FFmpegReader::IsHardwareDecodeSupported(int codecid)
202 {
203  int ret;
204  switch (codecid) {
205  case AV_CODEC_ID_H264:
206  case AV_CODEC_ID_MPEG2VIDEO:
207  case AV_CODEC_ID_VC1:
208  case AV_CODEC_ID_WMV1:
209  case AV_CODEC_ID_WMV2:
210  case AV_CODEC_ID_WMV3:
211  ret = 1;
212  break;
213  default :
214  ret = 0;
215  break;
216  }
217  return ret;
218 }
219 #endif // HAVE_HW_ACCEL
220 
222  // Open reader if not already open
223  if (!is_open) {
224  // Initialize format context
225  pFormatCtx = NULL;
226  {
228  }
229 
230  // Open video file
231  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
232  throw InvalidFile("File could not be opened.", path);
233 
234  // Retrieve stream information
235  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
236  throw NoStreamsFound("No streams found in file.", path);
237 
238  videoStream = -1;
239  audioStream = -1;
240  // Loop through each stream, and identify the video and audio stream index
241  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
242  // Is this a video stream?
243  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
244  videoStream = i;
245  }
246  // Is this an audio stream?
247  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
248  audioStream = i;
249  }
250  }
251  if (videoStream == -1 && audioStream == -1)
252  throw NoStreamsFound("No video or audio streams found in this file.", path);
253 
254  // Is there a video stream?
255  if (videoStream != -1) {
256  // Set the stream index
257  info.video_stream_index = videoStream;
258 
259  // Set the codec and codec context pointers
260  pStream = pFormatCtx->streams[videoStream];
261 
262  // Find the codec ID from stream
263  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
264 
265  // Get codec and codec context from stream
266  AVCodec *pCodec = avcodec_find_decoder(codecId);
267  AVDictionary *opts = NULL;
268  int retry_decode_open = 2;
269  // If hw accel is selected but hardware cannot handle repeat with software decoding
270  do {
271  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
272 #if HAVE_HW_ACCEL
273  if (hw_de_on && (retry_decode_open==2)) {
274  // Up to here no decision is made if hardware or software decode
275  hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
276  }
277 #endif
278  retry_decode_open = 0;
279 
280  // Set number of threads equal to number of processors (not to exceed 16)
281  pCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
282 
283  if (pCodec == NULL) {
284  throw InvalidCodec("A valid video codec could not be found for this file.", path);
285  }
286 
287  // Init options
288  av_dict_set(&opts, "strict", "experimental", 0);
289 #if HAVE_HW_ACCEL
290  if (hw_de_on && hw_de_supported) {
291  // Open Hardware Acceleration
292  int i_decoder_hw = 0;
293  char adapter[256];
294  char *adapter_ptr = NULL;
295  int adapter_num;
297  fprintf(stderr, "Hardware decoding device number: %d\n", adapter_num);
298 
299  // Set hardware pix format (callback)
300  pCodecCtx->get_format = get_hw_dec_format;
301 
302  if (adapter_num < 3 && adapter_num >=0) {
303 #if defined(__linux__)
304  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
305  adapter_ptr = adapter;
307  switch (i_decoder_hw) {
308  case 1:
309  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
310  break;
311  case 2:
312  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
313  break;
314  case 6:
315  hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU;
316  break;
317  case 7:
318  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
319  break;
320  default:
321  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
322  break;
323  }
324 
325 #elif defined(_WIN32)
326  adapter_ptr = NULL;
328  switch (i_decoder_hw) {
329  case 2:
330  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
331  break;
332  case 3:
333  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
334  break;
335  case 4:
336  hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA;
337  break;
338  case 7:
339  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
340  break;
341  default:
342  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
343  break;
344  }
345 #elif defined(__APPLE__)
346  adapter_ptr = NULL;
348  switch (i_decoder_hw) {
349  case 5:
350  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
351  break;
352  case 7:
353  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
354  break;
355  default:
356  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
357  break;
358  }
359 #endif
360 
361  } else {
362  adapter_ptr = NULL; // Just to be sure
363  }
364 
365  // Check if it is there and writable
366 #if defined(__linux__)
367  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
368 #elif defined(_WIN32)
369  if( adapter_ptr != NULL ) {
370 #elif defined(__APPLE__)
371  if( adapter_ptr != NULL ) {
372 #endif
373  ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device");
374  }
375  else {
376  adapter_ptr = NULL; // use default
377  ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default");
378  }
379 
380  hw_device_ctx = NULL;
381  // Here the first hardware initialisations are made
382  if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
383  if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) {
384  throw InvalidCodec("Hardware device reference create failed.", path);
385  }
386 
387  /*
388  av_buffer_unref(&ist->hw_frames_ctx);
389  ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
390  if (!ist->hw_frames_ctx) {
391  av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
392  return AVERROR(ENOMEM);
393  }
394 
395  frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
396 
397  frames_ctx->format = AV_PIX_FMT_CUDA;
398  frames_ctx->sw_format = avctx->sw_pix_fmt;
399  frames_ctx->width = avctx->width;
400  frames_ctx->height = avctx->height;
401 
402  av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
403  av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
404 
405 
406  ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx);
407  ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
408  if (ret < 0) {
409  av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
410  return ret;
411  }
412  */
413  }
414  else {
415  throw InvalidCodec("Hardware device create failed.", path);
416  }
417  }
418 #endif // HAVE_HW_ACCEL
419 
420  // Disable per-frame threading for album arts
421  // Using FF_THREAD_FRAME adds one frame decoding delay per thread,
422  // but there's only one frame in this case.
423  if (HasAlbumArt())
424  {
425  pCodecCtx->thread_type &= ~FF_THREAD_FRAME;
426  }
427 
428  // Open video codec
429  if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
430  throw InvalidCodec("A video codec was found, but could not be opened.", path);
431 
432 #if HAVE_HW_ACCEL
433  if (hw_de_on && hw_de_supported) {
434  AVHWFramesConstraints *constraints = NULL;
435  void *hwconfig = NULL;
436  hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx);
437 
438 // TODO: needs va_config!
439 #if ENABLE_VAAPI
440  ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
441  constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
442 #endif // ENABLE_VAAPI
443  if (constraints) {
444  if (pCodecCtx->coded_width < constraints->min_width ||
445  pCodecCtx->coded_height < constraints->min_height ||
446  pCodecCtx->coded_width > constraints->max_width ||
447  pCodecCtx->coded_height > constraints->max_height) {
448  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n");
449  hw_de_supported = 0;
450  retry_decode_open = 1;
451  AV_FREE_CONTEXT(pCodecCtx);
452  if (hw_device_ctx) {
453  av_buffer_unref(&hw_device_ctx);
454  hw_device_ctx = NULL;
455  }
456  }
457  else {
458  // All is just peachy
459  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
460  retry_decode_open = 0;
461  }
462  av_hwframe_constraints_free(&constraints);
463  if (hwconfig) {
464  av_freep(&hwconfig);
465  }
466  }
467  else {
468  int max_h, max_w;
469  //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" )));
471  //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" )));
473  ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n");
474  //cerr << "Constraints could not be found using default limit\n";
475  if (pCodecCtx->coded_width < 0 ||
476  pCodecCtx->coded_height < 0 ||
477  pCodecCtx->coded_width > max_w ||
478  pCodecCtx->coded_height > max_h ) {
479  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
480  hw_de_supported = 0;
481  retry_decode_open = 1;
482  AV_FREE_CONTEXT(pCodecCtx);
483  if (hw_device_ctx) {
484  av_buffer_unref(&hw_device_ctx);
485  hw_device_ctx = NULL;
486  }
487  }
488  else {
489  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
490  retry_decode_open = 0;
491  }
492  }
493  } // if hw_de_on && hw_de_supported
494  else {
495  ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n");
496  }
497 #else
498  retry_decode_open = 0;
499 #endif // HAVE_HW_ACCEL
500  } while (retry_decode_open); // retry_decode_open
501  // Free options
502  av_dict_free(&opts);
503 
504  // Update the File Info struct with video details (if a video stream is found)
505  UpdateVideoInfo();
506  }
507 
508  // Is there an audio stream?
509  if (audioStream != -1) {
510  // Set the stream index
511  info.audio_stream_index = audioStream;
512 
513  // Get a pointer to the codec context for the audio stream
514  aStream = pFormatCtx->streams[audioStream];
515 
516  // Find the codec ID from stream
517  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
518 
519  // Get codec and codec context from stream
520  AVCodec *aCodec = avcodec_find_decoder(codecId);
521  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
522 
523  // Set number of threads equal to number of processors (not to exceed 16)
524  aCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
525 
526  if (aCodec == NULL) {
527  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
528  }
529 
530  // Init options
531  AVDictionary *opts = NULL;
532  av_dict_set(&opts, "strict", "experimental", 0);
533 
534  // Open audio codec
535  if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
536  throw InvalidCodec("An audio codec was found, but could not be opened.", path);
537 
538  // Free options
539  av_dict_free(&opts);
540 
541  // Update the File Info struct with audio details (if an audio stream is found)
542  UpdateAudioInfo();
543  }
544 
545  // Add format metadata (if any)
546  AVDictionaryEntry *tag = NULL;
547  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
548  QString str_key = tag->key;
549  QString str_value = tag->value;
550  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
551  }
552 
553  // Init previous audio location to zero
554  previous_packet_location.frame = -1;
555  previous_packet_location.sample_start = 0;
556 
557  // Adjust cache size based on size of frame and audio
558  working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels);
559  missing_frames.SetMaxBytesFromInfo(max_concurrent_frames * 2, info.width, info.height, info.sample_rate, info.channels);
561 
562  // Mark as "open"
563  is_open = true;
564  }
565 }
566 
568  // Close all objects, if reader is 'open'
569  if (is_open) {
570  // Mark as "closed"
571  is_open = false;
572 
573  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close");
574 
575  if (packet) {
576  // Remove previous packet before getting next one
577  RemoveAVPacket(packet);
578  packet = NULL;
579  }
580 
581  // Close the codec
582  if (info.has_video) {
583  avcodec_flush_buffers(pCodecCtx);
584  AV_FREE_CONTEXT(pCodecCtx);
585 #if HAVE_HW_ACCEL
586  if (hw_de_on) {
587  if (hw_device_ctx) {
588  av_buffer_unref(&hw_device_ctx);
589  hw_device_ctx = NULL;
590  }
591  }
592 #endif // HAVE_HW_ACCEL
593  }
594  if (info.has_audio) {
595  avcodec_flush_buffers(aCodecCtx);
596  AV_FREE_CONTEXT(aCodecCtx);
597  }
598 
599  // Clear final cache
600  final_cache.Clear();
601  working_cache.Clear();
602  missing_frames.Clear();
603 
604  // Clear processed lists
605  {
606  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
607  processed_video_frames.clear();
608  processed_audio_frames.clear();
609  processing_video_frames.clear();
610  processing_audio_frames.clear();
611  missing_audio_frames.clear();
612  missing_video_frames.clear();
613  missing_audio_frames_source.clear();
614  missing_video_frames_source.clear();
615  checked_frames.clear();
616  }
617 
618  // Close the video file
619  avformat_close_input(&pFormatCtx);
620  av_freep(&pFormatCtx);
621 
622  // Reset some variables
623  last_frame = 0;
624  largest_frame_processed = 0;
625  seek_audio_frame_found = 0;
626  seek_video_frame_found = 0;
627  current_video_frame = 0;
628  has_missing_frames = false;
629 
630  last_video_frame.reset();
631  }
632 }
633 
634 bool FFmpegReader::HasAlbumArt() {
635  // Check if the video stream we use is an attached picture
636  // This won't return true if the file has a cover image as a secondary stream
637  // like an MKV file with an attached image file
638  return pFormatCtx && videoStream >= 0 && pFormatCtx->streams[videoStream]
639  && (pFormatCtx->streams[videoStream]->disposition & AV_DISPOSITION_ATTACHED_PIC);
640 }
641 
642 void FFmpegReader::UpdateAudioInfo() {
643  // Set values of FileInfo struct
644  info.has_audio = true;
645  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
646  info.acodec = aCodecCtx->codec->name;
647  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
648  if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
649  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
650  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
651  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
652  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
653 
654  // Set audio timebase
655  info.audio_timebase.num = aStream->time_base.num;
656  info.audio_timebase.den = aStream->time_base.den;
657 
658  // Get timebase of audio stream (if valid) and greater than the current duration
659  if (aStream->duration > 0.0f && aStream->duration > info.duration)
660  info.duration = aStream->duration * info.audio_timebase.ToDouble();
661 
662  // Check for an invalid video length
663  if (info.has_video && info.video_length <= 0) {
664  // Calculate the video length from the audio duration
666  }
667 
668  // Set video timebase (if no video stream was found)
669  if (!info.has_video) {
670  // Set a few important default video settings (so audio can be divided into frames)
671  info.fps.num = 24;
672  info.fps.den = 1;
673  info.video_timebase.num = 1;
674  info.video_timebase.den = 24;
676  info.width = 720;
677  info.height = 480;
678  }
679 
680  // Fix invalid video lengths for certain types of files (MP3 for example)
681  if (info.has_video && ((info.duration * info.fps.ToDouble()) - info.video_length > 60)) {
683  }
684 
685  // Add audio metadata (if any found)
686  AVDictionaryEntry *tag = NULL;
687  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
688  QString str_key = tag->key;
689  QString str_value = tag->value;
690  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
691  }
692 }
693 
694 void FFmpegReader::UpdateVideoInfo() {
695  if (check_fps)
696  // Already initialized all the video metadata, no reason to do it again
697  return;
698 
699  // Set values of FileInfo struct
700  info.has_video = true;
701  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
702  info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
703  info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
704  info.vcodec = pCodecCtx->codec->name;
705  info.video_bit_rate = (pFormatCtx->bit_rate / 8);
706 
707  // Frame rate from the container and codec
708  AVRational framerate = av_guess_frame_rate(pFormatCtx, pStream, NULL);
709  info.fps.num = framerate.num;
710  info.fps.den = framerate.den;
711 
712  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo", "info.fps.num", info.fps.num, "info.fps.den", info.fps.den);
713 
714  // TODO: remove excessive debug info in the next releases
715  // The debug info below is just for comparison and troubleshooting on users side during the transition period
716  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo (pStream->avg_frame_rate)", "num", pStream->avg_frame_rate.num, "den", pStream->avg_frame_rate.den);
717 
718  if (pStream->sample_aspect_ratio.num != 0) {
719  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
720  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
721  } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) {
722  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
723  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
724  } else {
725  info.pixel_ratio.num = 1;
726  info.pixel_ratio.den = 1;
727  }
728  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
729 
730  // Calculate the DAR (display aspect ratio)
732 
733  // Reduce size fraction
734  size.Reduce();
735 
736  // Set the ratio based on the reduced fraction
737  info.display_ratio.num = size.num;
738  info.display_ratio.den = size.den;
739 
740  // Get scan type and order from codec context/params
741  if (!check_interlace) {
742  check_interlace = true;
743  AVFieldOrder field_order = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->field_order;
744  switch(field_order) {
745  case AV_FIELD_PROGRESSIVE:
746  info.interlaced_frame = false;
747  break;
748  case AV_FIELD_TT:
749  case AV_FIELD_TB:
750  info.interlaced_frame = true;
751  info.top_field_first = true;
752  break;
753  case AV_FIELD_BT:
754  case AV_FIELD_BB:
755  info.interlaced_frame = true;
756  info.top_field_first = false;
757  break;
758  case AV_FIELD_UNKNOWN:
759  // Check again later?
760  check_interlace = false;
761  break;
762  }
763  // check_interlace will prevent these checks being repeated,
764  // unless it was cleared because we got an AV_FIELD_UNKNOWN response.
765  }
766 
767  // Set the video timebase
768  info.video_timebase.num = pStream->time_base.num;
769  info.video_timebase.den = pStream->time_base.den;
770 
771  // Set the duration in seconds, and video length (# of frames)
772  info.duration = pStream->duration * info.video_timebase.ToDouble();
773 
774  // Check for valid duration (if found)
775  if (info.duration <= 0.0f && pFormatCtx->duration >= 0)
776  // Use the format's duration
777  info.duration = pFormatCtx->duration / AV_TIME_BASE;
778 
779  // Calculate duration from filesize and bitrate (if any)
780  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0)
781  // Estimate from bitrate, total bytes, and framerate
783 
784  // No duration found in stream of file
785  if (info.duration <= 0.0f) {
786  // No duration is found in the video stream
787  info.duration = -1;
788  info.video_length = -1;
789  is_duration_known = false;
790  } else {
791  // Yes, a duration was found
792  is_duration_known = true;
793 
794  // Calculate number of frames
796  }
797 
798  // Override an invalid framerate
799  if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
800  // Calculate FPS, duration, video bit rate, and video length manually
801  // by scanning through all the video stream packets
802  CheckFPS();
803  }
804 
805  // Add video metadata (if any)
806  AVDictionaryEntry *tag = NULL;
807  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
808  QString str_key = tag->key;
809  QString str_value = tag->value;
810  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
811  }
812 }
813 
815  return this->is_duration_known;
816 }
817 
818 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame) {
819  // Check for open reader (or throw exception)
820  if (!is_open)
821  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
822 
823  // Adjust for a requested frame that is too small or too large
824  if (requested_frame < 1)
825  requested_frame = 1;
826  if (requested_frame > info.video_length && is_duration_known)
827  requested_frame = info.video_length;
828  if (info.has_video && info.video_length == 0)
829  // Invalid duration of video file
830  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
831 
832  // Debug output
833  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame);
834 
835  // Check the cache for this frame
836  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
837  if (frame) {
838  // Debug output
839  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame);
840 
841  // Return the cached frame
842  return frame;
843  } else {
844  // Check the cache a 2nd time (due to a potential previous lock)
845  frame = final_cache.GetFrame(requested_frame);
846  if (frame) {
847  // Debug output
848  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame);
849 
850  // Return the cached frame
851  } else {
852  // Frame is not in cache
853  // Reset seek count
854  seek_count = 0;
855 
856  // Check for first frame (always need to get frame 1 before other frames, to correctly calculate offsets)
857  if (last_frame == 0 && requested_frame != 1)
858  // Get first frame
859  ReadStream(1);
860 
861  // Are we within X frames of the requested frame?
862  int64_t diff = requested_frame - last_frame;
863  if (diff >= 1 && diff <= 20) {
864  // Continue walking the stream
865  frame = ReadStream(requested_frame);
866  } else {
867  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
868  if (enable_seek)
869  // Only seek if enabled
870  Seek(requested_frame);
871 
872  else if (!enable_seek && diff < 0) {
873  // Start over, since we can't seek, and the requested frame is smaller than our position
874  Close();
875  Open();
876  }
877 
878  // Then continue walking the stream
879  frame = ReadStream(requested_frame);
880  }
881  }
882  return frame;
883  }
884 }
885 
886 // Read the stream until we find the requested Frame
887 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame) {
888  // Allocate video frame
889  bool end_of_stream = false;
890  bool check_seek = false;
891  bool frame_finished = false;
892  int packet_error = -1;
893 
894  // Minimum number of packets to process (for performance reasons)
895  int packets_processed = 0;
896  int minimum_packets = 1;
897  int max_packets = 4096;
898 
899  // Debug output
900  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "max_concurrent_frames", max_concurrent_frames);
901 
902  // Loop through the stream until the correct frame is found
903  while (true) {
904  // Get the next packet into a local variable called packet
905  packet_error = GetNextPacket();
906 
907  int processing_video_frames_size = 0;
908  int processing_audio_frames_size = 0;
909  {
910  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
911  processing_video_frames_size = processing_video_frames.size();
912  processing_audio_frames_size = processing_audio_frames.size();
913  }
914 
915  // Wait if too many frames are being processed
916  while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
917  std::this_thread::sleep_for(std::chrono::milliseconds(3));
918  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
919  processing_video_frames_size = processing_video_frames.size();
920  processing_audio_frames_size = processing_audio_frames.size();
921  }
922 
923  // Get the next packet (if any)
924  if (packet_error < 0) {
925  // Break loop when no more packets found
926  end_of_stream = true;
927  break;
928  }
929 
930  // Debug output
931  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "minimum_packets", minimum_packets, "packets_processed", packets_processed, "is_seeking", is_seeking);
932 
933  // Video packet
934  if (info.has_video && packet->stream_index == videoStream) {
935  // Reset this counter, since we have a video packet
936  num_packets_since_video_frame = 0;
937 
938  // Check the status of a seek (if any)
939  if (is_seeking) {
940  check_seek = CheckSeek(true);
941  } else {
942  check_seek = false;
943  }
944 
945  if (check_seek) {
946  // Jump to the next iteration of this loop
947  continue;
948  }
949 
950  // Packet may become NULL on Close inside Seek if CheckSeek returns false
951  if (!packet) {
952  // Jump to the next iteration of this loop
953  continue;
954  }
955 
956  // Get the AVFrame from the current packet
957  frame_finished = GetAVFrame();
958 
959  // Check if the AVFrame is finished and set it
960  if (frame_finished) {
961  // Update PTS / Frame Offset (if any)
962  UpdatePTSOffset(true);
963 
964  // Process Video Packet
965  ProcessVideoPacket(requested_frame);
966  }
967 
968  }
969  // Audio packet
970  else if (info.has_audio && packet->stream_index == audioStream) {
971  // Increment this (to track # of packets since the last video packet)
972  num_packets_since_video_frame++;
973 
974  // Check the status of a seek (if any)
975  if (is_seeking) {
976  check_seek = CheckSeek(false);
977  } else {
978  check_seek = false;
979  }
980 
981  if (check_seek) {
982  // Jump to the next iteration of this loop
983  continue;
984  }
985 
986  // Packet may become NULL on Close inside Seek if CheckSeek returns false
987  if (!packet) {
988  // Jump to the next iteration of this loop
989  continue;
990  }
991 
992  // Update PTS / Frame Offset (if any)
993  UpdatePTSOffset(false);
994 
995  // Determine related video frame and starting sample # from audio PTS
996  AudioLocation location = GetAudioPTSLocation(packet->pts);
997 
998  // Process Audio Packet
999  ProcessAudioPacket(requested_frame, location.frame, location.sample_start);
1000  }
1001 
1002  // Check if working frames are 'finished'
1003  if (!is_seeking) {
1004  // Check for final frames
1005  CheckWorkingFrames(false, requested_frame);
1006  }
1007 
1008  // Check if requested 'final' frame is available
1009  bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
1010 
1011  // Increment frames processed
1012  packets_processed++;
1013 
1014  // Break once the frame is found
1015  if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
1016  break;
1017 
1018  } // end while
1019 
1020  // Debug output
1021  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)", "packets_processed", packets_processed, "end_of_stream", end_of_stream, "largest_frame_processed", largest_frame_processed, "Working Cache Count", working_cache.Count());
1022 
1023  // End of stream?
1024  if (end_of_stream)
1025  // Mark the any other working frames as 'finished'
1026  CheckWorkingFrames(end_of_stream, requested_frame);
1027 
1028  // Return requested frame (if found)
1029  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1030  if (frame)
1031  // Return prepared frame
1032  return frame;
1033  else {
1034 
1035  // Check if largest frame is still cached
1036  frame = final_cache.GetFrame(largest_frame_processed);
1037  if (frame) {
1038  // return the largest processed frame (assuming it was the last in the video file)
1039  return frame;
1040  } else {
1041  // The largest processed frame is no longer in cache, return a blank frame
1042  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1043  f->AddColor(info.width, info.height, "#000");
1044  return f;
1045  }
1046  }
1047 
1048 }
1049 
1050 // Get the next packet (if any)
1051 int FFmpegReader::GetNextPacket() {
1052  int found_packet = 0;
1053  AVPacket *next_packet;
1054  next_packet = new AVPacket();
1055  found_packet = av_read_frame(pFormatCtx, next_packet);
1056 
1057  if (packet) {
1058  // Remove previous packet before getting next one
1059  RemoveAVPacket(packet);
1060  packet = NULL;
1061  }
1062  if (found_packet >= 0) {
1063  // Update current packet pointer
1064  packet = next_packet;
1065  } else {
1066  delete next_packet;
1067  }
1068  // Return if packet was found (or error number)
1069  return found_packet;
1070 }
1071 
1072 // Get an AVFrame (if any)
1073 bool FFmpegReader::GetAVFrame() {
1074  int frameFinished = -1;
1075  int ret = 0;
1076 
1077  // Decode video frame
1078  AVFrame *next_frame = AV_ALLOCATE_FRAME();
1079 
1080 #if IS_FFMPEG_3_2
1081  frameFinished = 0;
1082  ret = avcodec_send_packet(pCodecCtx, packet);
1083 
1084  #if HAVE_HW_ACCEL
1085  // Get the format from the variables set in get_hw_dec_format
1086  hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
1087  hw_de_av_device_type = hw_de_av_device_type_global;
1088  #endif // HAVE_HW_ACCEL
1089  if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
1090  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Packet not sent)");
1091  }
1092  else {
1093  AVFrame *next_frame2;
1094  #if HAVE_HW_ACCEL
1095  if (hw_de_on && hw_de_supported) {
1096  next_frame2 = AV_ALLOCATE_FRAME();
1097  }
1098  else
1099  #endif // HAVE_HW_ACCEL
1100  {
1101  next_frame2 = next_frame;
1102  }
1103  pFrame = AV_ALLOCATE_FRAME();
1104  while (ret >= 0) {
1105  ret = avcodec_receive_frame(pCodecCtx, next_frame2);
1106  if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
1107  break;
1108  }
1109  if (ret != 0) {
1110  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (invalid return frame received)");
1111  }
1112  #if HAVE_HW_ACCEL
1113  if (hw_de_on && hw_de_supported) {
1114  int err;
1115  if (next_frame2->format == hw_de_av_pix_fmt) {
1116  next_frame->format = AV_PIX_FMT_YUV420P;
1117  if ((err = av_hwframe_transfer_data(next_frame,next_frame2,0)) < 0) {
1118  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to transfer data to output frame)");
1119  }
1120  if ((err = av_frame_copy_props(next_frame,next_frame2)) < 0) {
1121  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to copy props to output frame)");
1122  }
1123  }
1124  }
1125  else
1126  #endif // HAVE_HW_ACCEL
1127  { // No hardware acceleration used -> no copy from GPU memory needed
1128  next_frame = next_frame2;
1129  }
1130 
1131  // TODO also handle possible further frames
1132  // Use only the first frame like avcodec_decode_video2
1133  if (frameFinished == 0 ) {
1134  frameFinished = 1;
1135  av_image_alloc(pFrame->data, pFrame->linesize, info.width, info.height, (AVPixelFormat)(pStream->codecpar->format), 1);
1136  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize,
1137  (AVPixelFormat)(pStream->codecpar->format), info.width, info.height);
1138  }
1139  }
1140  #if HAVE_HW_ACCEL
1141  if (hw_de_on && hw_de_supported) {
1142  AV_FREE_FRAME(&next_frame2);
1143  }
1144  #endif // HAVE_HW_ACCEL
1145  }
1146 #else
1147  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
1148 
1149  // always allocate pFrame (because we do that in the ffmpeg >= 3.2 as well); it will always be freed later
1150  pFrame = AV_ALLOCATE_FRAME();
1151 
1152  // is frame finished
1153  if (frameFinished) {
1154  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
1155  // must make a copy of the image data before this method is called again.
1156  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1157  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
1158  info.height);
1159  }
1160 #endif // IS_FFMPEG_3_2
1161 
1162  // deallocate the frame
1163  AV_FREE_FRAME(&next_frame);
1164 
1165  // Did we get a video frame?
1166  return frameFinished;
1167 }
1168 
1169 // Check the current seek position and determine if we need to seek again
1170 bool FFmpegReader::CheckSeek(bool is_video) {
1171  // Are we seeking for a specific frame?
1172  if (is_seeking) {
1173  // Determine if both an audio and video packet have been decoded since the seek happened.
1174  // If not, allow the ReadStream method to keep looping
1175  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
1176  return false;
1177 
1178  // Check for both streams
1179  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
1180  return false;
1181 
1182  // Determine max seeked frame
1183  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
1184  if (seek_video_frame_found > max_seeked_frame)
1185  max_seeked_frame = seek_video_frame_found;
1186 
1187  // determine if we are "before" the requested frame
1188  if (max_seeked_frame >= seeking_frame) {
1189  // SEEKED TOO FAR
1190  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
1191 
1192  // Seek again... to the nearest Keyframe
1193  Seek(seeking_frame - (10 * seek_count * seek_count));
1194  } else {
1195  // SEEK WORKED
1196  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
1197 
1198  // Seek worked, and we are "before" the requested frame
1199  is_seeking = false;
1200  seeking_frame = 0;
1201  seeking_pts = -1;
1202  }
1203  }
1204 
1205  // return the pts to seek to (if any)
1206  return is_seeking;
1207 }
1208 
1209 // Process a video packet
1210 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
1211  // Calculate current frame #
1212  int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
1213 
1214  // Track 1st video packet after a successful seek
1215  if (!seek_video_frame_found && is_seeking)
1216  seek_video_frame_found = current_frame;
1217 
1218  // Are we close enough to decode the frame? and is this frame # valid?
1219  if ((current_frame < (requested_frame - 20)) or (current_frame == -1)) {
1220  // Remove frame and packet
1221  RemoveAVFrame(pFrame);
1222 
1223  // Debug output
1224  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Skipped)", "requested_frame", requested_frame, "current_frame", current_frame);
1225 
1226  // Skip to next frame without decoding or caching
1227  return;
1228  }
1229 
1230  // Debug output
1231  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame);
1232 
1233  // Init some things local (for OpenMP)
1234  PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1235  int height = info.height;
1236  int width = info.width;
1237  int64_t video_length = info.video_length;
1238  AVFrame *my_frame = pFrame;
1239  pFrame = NULL;
1240 
1241  // Add video frame to list of processing video frames
1242  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1243  processing_video_frames[current_frame] = current_frame;
1244 
1245  // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
1246  AVFrame *pFrameRGB = nullptr;
1247  uint8_t *buffer = nullptr;
1248 
1249  // Allocate an AVFrame structure
1250  pFrameRGB = AV_ALLOCATE_FRAME();
1251  if (pFrameRGB == nullptr)
1252  throw OutOfMemory("Failed to allocate frame buffer", path);
1253 
1254  // Determine the max size of this source image (based on the timeline's size, the scaling mode,
1255  // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
1256  // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
1257  // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
1258  // the future.
1259  int max_width = info.width;
1260  int max_height = info.height;
1261 
1262  Clip *parent = (Clip *) ParentClip();
1263  if (parent) {
1264  if (parent->ParentTimeline()) {
1265  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1266  max_width = parent->ParentTimeline()->preview_width;
1267  max_height = parent->ParentTimeline()->preview_height;
1268  }
1269  if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
1270  // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
1271  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1272  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1273  max_width = std::max(float(max_width), max_width * max_scale_x);
1274  max_height = std::max(float(max_height), max_height * max_scale_y);
1275 
1276  } else if (parent->scale == SCALE_CROP) {
1277  // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
1278  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1279  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1280  QSize width_size(max_width * max_scale_x,
1281  round(max_width / (float(info.width) / float(info.height))));
1282  QSize height_size(round(max_height / (float(info.height) / float(info.width))),
1283  max_height * max_scale_y);
1284  // respect aspect ratio
1285  if (width_size.width() >= max_width && width_size.height() >= max_height) {
1286  max_width = std::max(max_width, width_size.width());
1287  max_height = std::max(max_height, width_size.height());
1288  } else {
1289  max_width = std::max(max_width, height_size.width());
1290  max_height = std::max(max_height, height_size.height());
1291  }
1292 
1293  } else {
1294  // No scaling, use original image size (slower)
1295  max_width = info.width;
1296  max_height = info.height;
1297  }
1298  }
1299 
1300  // Determine if image needs to be scaled (for performance reasons)
1301  int original_height = height;
1302  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
1303  // Override width and height (but maintain aspect ratio)
1304  float ratio = float(width) / float(height);
1305  int possible_width = round(max_height * ratio);
1306  int possible_height = round(max_width / ratio);
1307 
1308  if (possible_width <= max_width) {
1309  // use calculated width, and max_height
1310  width = possible_width;
1311  height = max_height;
1312  } else {
1313  // use max_width, and calculated height
1314  width = max_width;
1315  height = possible_height;
1316  }
1317  }
1318 
1319  // Determine required buffer size and allocate buffer
1320  const int bytes_per_pixel = 4;
1321  int buffer_size = width * height * bytes_per_pixel;
1322  buffer = new unsigned char[buffer_size]();
1323 
1324  // Copy picture data from one AVFrame (or AVPicture) to another one.
1325  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
1326 
1327  int scale_mode = SWS_FAST_BILINEAR;
1328  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
1329  scale_mode = SWS_BICUBIC;
1330  }
1331  SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width,
1332  height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
1333 
1334  // Resize / Convert to RGB
1335  sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
1336  original_height, pFrameRGB->data, pFrameRGB->linesize);
1337 
1338  // Create or get the existing frame object
1339  std::shared_ptr<Frame> f = CreateFrame(current_frame);
1340 
1341  // Add Image data to frame
1342  if (!ffmpeg_has_alpha(AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx))) {
1343  // Add image with no alpha channel, Speed optimization
1344  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer);
1345  } else {
1346  // Add image with alpha channel (this will be converted to premultipled when needed, but is slower)
1347  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer);
1348  }
1349 
1350  // Update working cache
1351  working_cache.Add(f);
1352 
1353  // Keep track of last last_video_frame
1354  last_video_frame = f;
1355 
1356  // Free the RGB image
1357  AV_FREE_FRAME(&pFrameRGB);
1358 
1359  // Remove frame and packet
1360  RemoveAVFrame(my_frame);
1361  sws_freeContext(img_convert_ctx);
1362 
1363  // Remove video frame from list of processing video frames
1364  {
1365  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1366  processing_video_frames.erase(current_frame);
1367  processed_video_frames[current_frame] = current_frame;
1368  }
1369 
1370  // Debug output
1371  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number);
1372 }
1373 
1374 // Process an audio packet
1375 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample) {
1376  // Track 1st audio packet after a successful seek
1377  if (!seek_audio_frame_found && is_seeking)
1378  seek_audio_frame_found = target_frame;
1379 
1380  // Are we close enough to decode the frame's audio?
1381  if (target_frame < (requested_frame - 20)) {
1382  // Debug output
1383  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample);
1384 
1385  // Skip to next frame without decoding or caching
1386  return;
1387  }
1388 
1389  // Debug output
1390  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample);
1391 
1392  // Init an AVFrame to hold the decoded audio samples
1393  int frame_finished = 0;
1394  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
1395  AV_RESET_FRAME(audio_frame);
1396 
1397  int packet_samples = 0;
1398  int data_size = 0;
1399 
1400 #if IS_FFMPEG_3_2
1401  int ret = 0;
1402  frame_finished = 1;
1403  while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) {
1404  frame_finished = 0;
1405  ret = avcodec_send_packet(aCodecCtx, packet);
1406  if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1407  avcodec_send_packet(aCodecCtx, NULL);
1408  break;
1409  }
1410  if (ret >= 0)
1411  packet->size = 0;
1412  ret = avcodec_receive_frame(aCodecCtx, audio_frame);
1413  if (ret >= 0)
1414  frame_finished = 1;
1415  if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1416  avcodec_flush_buffers(aCodecCtx);
1417  ret = 0;
1418  }
1419  if (ret >= 0) {
1420  ret = frame_finished;
1421  }
1422  }
1423  if (!packet->data && !frame_finished)
1424  {
1425  ret = -1;
1426  }
1427 #else
1428  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1429 #endif
1430 
1431  if (frame_finished) {
1432 
1433  // determine how many samples were decoded
1434  int plane_size = -1;
1435  data_size = av_samples_get_buffer_size(&plane_size,
1436  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels,
1437  audio_frame->nb_samples,
1438  (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
1439 
1440  // Calculate total number of samples
1441  packet_samples = audio_frame->nb_samples * AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
1442  }
1443 
1444  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
1445  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
1446 
1447  // DEBUG (FOR AUDIO ISSUES) - Get the audio packet start time (in seconds)
1448  int64_t adjusted_pts = packet->pts + audio_pts_offset;
1449  double audio_seconds = double(adjusted_pts) * info.audio_timebase.ToDouble();
1450  double sample_seconds = double(pts_total) / info.sample_rate;
1451 
1452  // Debug output
1453  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info A)", "pts_counter", pts_counter, "PTS", adjusted_pts, "Offset", audio_pts_offset, "PTS Diff", adjusted_pts - prev_pts, "Samples", pts_remaining_samples, "Sample PTS ratio", float(adjusted_pts - prev_pts) / pts_remaining_samples);
1454  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info B)", "Sample Diff", pts_remaining_samples - prev_samples - prev_pts, "Total", pts_total, "PTS Seconds", audio_seconds, "Sample Seconds", sample_seconds, "Seconds Diff", audio_seconds - sample_seconds, "raw samples", packet_samples);
1455 
1456  // DEBUG (FOR AUDIO ISSUES)
1457  prev_pts = adjusted_pts;
1458  pts_total += pts_remaining_samples;
1459  pts_counter++;
1460  prev_samples = pts_remaining_samples;
1461 
1462  // Add audio frame to list of processing audio frames
1463  {
1464  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1465  processing_audio_frames.insert(std::pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1466  }
1467 
1468  while (pts_remaining_samples) {
1469  // Get Samples per frame (for this frame number)
1470  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
1471 
1472  // Calculate # of samples to add to this frame
1473  int samples = samples_per_frame - previous_packet_location.sample_start;
1474  if (samples > pts_remaining_samples)
1475  samples = pts_remaining_samples;
1476 
1477  // Decrement remaining samples
1478  pts_remaining_samples -= samples;
1479 
1480  if (pts_remaining_samples > 0) {
1481  // next frame
1482  previous_packet_location.frame++;
1483  previous_packet_location.sample_start = 0;
1484 
1485  // Add audio frame to list of processing audio frames
1486  {
1487  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1488  processing_audio_frames.insert(std::pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1489  }
1490 
1491  } else {
1492  // Increment sample start
1493  previous_packet_location.sample_start += samples;
1494  }
1495  }
1496 
1497 
1498  // Allocate audio buffer
1499  int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE];
1500 
1501  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16);
1502 
1503  // Create output frame
1504  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1505  AV_RESET_FRAME(audio_converted);
1506  audio_converted->nb_samples = audio_frame->nb_samples;
1507  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1508 
1509  SWRCONTEXT *avr = NULL;
1510  int nb_samples = 0;
1511 
1512  // setup resample context
1513  avr = SWR_ALLOC();
1514  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1515  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1516  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
1517  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1518  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1519  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1520  av_opt_set_int(avr, "in_channels", info.channels, 0);
1521  av_opt_set_int(avr, "out_channels", info.channels, 0);
1522  SWR_INIT(avr);
1523 
1524  // Convert audio samples
1525  nb_samples = SWR_CONVERT(avr, // audio resample context
1526  audio_converted->data, // output data pointers
1527  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1528  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1529  audio_frame->data, // input data pointers
1530  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1531  audio_frame->nb_samples); // number of input samples to convert
1532 
1533  // Copy audio samples over original samples
1534  memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels);
1535 
1536  // Deallocate resample buffer
1537  SWR_CLOSE(avr);
1538  SWR_FREE(&avr);
1539  avr = NULL;
1540 
1541  // Free AVFrames
1542  av_free(audio_converted->data[0]);
1543  AV_FREE_FRAME(&audio_converted);
1544 
1545  int64_t starting_frame_number = -1;
1546  bool partial_frame = true;
1547  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) {
1548  // Array of floats (to hold samples for each channel)
1549  starting_frame_number = target_frame;
1550  int channel_buffer_size = packet_samples / info.channels;
1551  float *channel_buffer = new float[channel_buffer_size];
1552 
1553  // Init buffer array
1554  for (int z = 0; z < channel_buffer_size; z++)
1555  channel_buffer[z] = 0.0f;
1556 
1557  // Loop through all samples and add them to our Frame based on channel.
1558  // Toggle through each channel number, since channel data is stored like (left right left right)
1559  int channel = 0;
1560  int position = 0;
1561  for (int sample = 0; sample < packet_samples; sample++) {
1562  // Only add samples for current channel
1563  if (channel_filter == channel) {
1564  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1565  channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1566 
1567  // Increment audio position
1568  position++;
1569  }
1570 
1571  // increment channel (if needed)
1572  if ((channel + 1) < info.channels)
1573  // move to next channel
1574  channel++;
1575  else
1576  // reset channel
1577  channel = 0;
1578  }
1579 
1580  // Loop through samples, and add them to the correct frames
1581  int start = starting_sample;
1582  int remaining_samples = channel_buffer_size;
1583  float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1584  while (remaining_samples > 0) {
1585  // Get Samples per frame (for this frame number)
1586  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1587 
1588  // Calculate # of samples to add to this frame
1589  int samples = samples_per_frame - start;
1590  if (samples > remaining_samples)
1591  samples = remaining_samples;
1592 
1593  // Create or get the existing frame object
1594  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1595 
1596  // Determine if this frame was "partially" filled in
1597  if (samples_per_frame == start + samples)
1598  partial_frame = false;
1599  else
1600  partial_frame = true;
1601 
1602  // Add samples for current channel to the frame.
1603  f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 1.0f);
1604 
1605  // Debug output
1606  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame);
1607 
1608  // Add or update cache
1609  working_cache.Add(f);
1610 
1611  // Decrement remaining samples
1612  remaining_samples -= samples;
1613 
1614  // Increment buffer (to next set of samples)
1615  if (remaining_samples > 0)
1616  iterate_channel_buffer += samples;
1617 
1618  // Increment frame number
1619  starting_frame_number++;
1620 
1621  // Reset starting sample #
1622  start = 0;
1623  }
1624 
1625  // clear channel buffer
1626  delete[] channel_buffer;
1627  channel_buffer = NULL;
1628  iterate_channel_buffer = NULL;
1629  }
1630 
1631  // Clean up some arrays
1632  delete[] audio_buf;
1633  audio_buf = NULL;
1634 
1635  // Remove audio frame from list of processing audio frames
1636  {
1637  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1638  // Update all frames as completed
1639  for (int64_t f = target_frame; f < starting_frame_number; f++) {
1640  // Remove the frame # from the processing list. NOTE: If more than one thread is
1641  // processing this frame, the frame # will be in this list multiple times. We are only
1642  // removing a single instance of it here.
1643  processing_audio_frames.erase(processing_audio_frames.find(f));
1644 
1645  // Check and see if this frame is also being processed by another thread
1646  if (processing_audio_frames.count(f) == 0)
1647  // No other thread is processing it. Mark the audio as processed (final)
1648  processed_audio_frames[f] = f;
1649  }
1650 
1651  if (target_frame == starting_frame_number) {
1652  // This typically never happens, but just in case, remove the currently processing number
1653  processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1654  }
1655  }
1656 
1657  // Free audio frame
1658  AV_FREE_FRAME(&audio_frame);
1659 
1660  // Debug output
1661  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1);
1662 
1663 }
1664 
1665 
1666 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1667 void FFmpegReader::Seek(int64_t requested_frame) {
1668  // Adjust for a requested frame that is too small or too large
1669  if (requested_frame < 1)
1670  requested_frame = 1;
1671  if (requested_frame > info.video_length)
1672  requested_frame = info.video_length;
1673 
1674  int processing_video_frames_size = 0;
1675  int processing_audio_frames_size = 0;
1676  {
1677  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1678  processing_video_frames_size = processing_video_frames.size();
1679  processing_audio_frames_size = processing_audio_frames.size();
1680  }
1681 
1682  // Debug output
1683  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek", "requested_frame", requested_frame, "seek_count", seek_count, "last_frame", last_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "video_pts_offset", video_pts_offset);
1684 
1685  // Wait for any processing frames to complete
1686  while (processing_video_frames_size + processing_audio_frames_size > 0) {
1687  std::this_thread::sleep_for(std::chrono::milliseconds(3));
1688  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1689  processing_video_frames_size = processing_video_frames.size();
1690  processing_audio_frames_size = processing_audio_frames.size();
1691  }
1692 
1693  // Clear working cache (since we are seeking to another location in the file)
1694  working_cache.Clear();
1695  missing_frames.Clear();
1696 
1697  // Clear processed lists
1698  {
1699  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1700  processing_audio_frames.clear();
1701  processing_video_frames.clear();
1702  processed_video_frames.clear();
1703  processed_audio_frames.clear();
1704  missing_audio_frames.clear();
1705  missing_video_frames.clear();
1706  missing_audio_frames_source.clear();
1707  missing_video_frames_source.clear();
1708  checked_frames.clear();
1709  }
1710 
1711  // Reset the last frame variable
1712  last_frame = 0;
1713  current_video_frame = 0;
1714  largest_frame_processed = 0;
1715  num_checks_since_final = 0;
1716  num_packets_since_video_frame = 0;
1717  has_missing_frames = false;
1718  bool has_audio_override = info.has_audio;
1719  bool has_video_override = info.has_video;
1720 
1721  // Increment seek count
1722  seek_count++;
1723 
1724  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1725  int buffer_amount = std::max(max_concurrent_frames, 8);
1726  if (requested_frame - buffer_amount < 20) {
1727  // Close and re-open file (basically seeking to frame 1)
1728  Close();
1729  Open();
1730 
1731  // Update overrides (since closing and re-opening might update these)
1732  info.has_audio = has_audio_override;
1733  info.has_video = has_video_override;
1734 
1735  // Not actually seeking, so clear these flags
1736  is_seeking = false;
1737  if (seek_count == 1) {
1738  // Don't redefine this on multiple seek attempts for a specific frame
1739  seeking_frame = 1;
1740  seeking_pts = ConvertFrameToVideoPTS(1);
1741  }
1742  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1743  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1744 
1745  } else {
1746  // Seek to nearest key-frame (aka, i-frame)
1747  bool seek_worked = false;
1748  int64_t seek_target = 0;
1749 
1750  // Seek video stream (if any), except album arts
1751  if (!seek_worked && info.has_video && !HasAlbumArt()) {
1752  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1753  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1754  fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME);
1755  } else {
1756  // VIDEO SEEK
1757  is_video_seek = true;
1758  seek_worked = true;
1759  }
1760  }
1761 
1762  // Seek audio stream (if not already seeked... and if an audio stream is found)
1763  if (!seek_worked && info.has_audio) {
1764  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1765  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1766  fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME);
1767  } else {
1768  // AUDIO SEEK
1769  is_video_seek = false;
1770  seek_worked = true;
1771  }
1772  }
1773 
1774  // Was the seek successful?
1775  if (seek_worked) {
1776  // Flush audio buffer
1777  if (info.has_audio)
1778  avcodec_flush_buffers(aCodecCtx);
1779 
1780  // Flush video buffer
1781  if (info.has_video)
1782  avcodec_flush_buffers(pCodecCtx);
1783 
1784  // Reset previous audio location to zero
1785  previous_packet_location.frame = -1;
1786  previous_packet_location.sample_start = 0;
1787 
1788  // init seek flags
1789  is_seeking = true;
1790  if (seek_count == 1) {
1791  // Don't redefine this on multiple seek attempts for a specific frame
1792  seeking_pts = seek_target;
1793  seeking_frame = requested_frame;
1794  }
1795  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1796  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1797 
1798  } else {
1799  // seek failed
1800  is_seeking = false;
1801  seeking_pts = 0;
1802  seeking_frame = 0;
1803 
1804  // dislable seeking for this reader (since it failed)
1805  // TODO: Find a safer way to do this... not sure how common it is for a seek to fail.
1806  enable_seek = false;
1807 
1808  // Close and re-open file (basically seeking to frame 1)
1809  Close();
1810  Open();
1811 
1812  // Update overrides (since closing and re-opening might update these)
1813  info.has_audio = has_audio_override;
1814  info.has_video = has_video_override;
1815  }
1816  }
1817 }
1818 
1819 // Get the PTS for the current video packet
1820 int64_t FFmpegReader::GetVideoPTS() {
1821  int64_t current_pts = 0;
1822  if (packet->dts != AV_NOPTS_VALUE)
1823  current_pts = packet->dts;
1824 
1825  // Return adjusted PTS
1826  return current_pts;
1827 }
1828 
1829 // Update PTS Offset (if any)
1830 void FFmpegReader::UpdatePTSOffset(bool is_video) {
1831  // Determine the offset between the PTS and Frame number (only for 1st frame)
1832  if (is_video) {
1833  // VIDEO PACKET
1834  if (video_pts_offset == 99999) // Has the offset been set yet?
1835  {
1836  // Find the difference between PTS and frame number
1837  video_pts_offset = 0 - GetVideoPTS();
1838 
1839  // Find the difference between PTS and frame number
1840  // Also, determine if PTS is invalid (too far away from zero)
1841  // We compare the PTS to the timebase value equal to 1 second (which means the PTS
1842  // must be within the -1 second to +1 second of zero, otherwise we ignore it)
1843  // TODO: Please see https://github.com/OpenShot/libopenshot/pull/565#issuecomment-690985272
1844  // for ideas to improve this logic.
1845  int64_t max_offset = info.video_timebase.Reciprocal().ToFloat();
1846  if (video_pts_offset < -max_offset || video_pts_offset > max_offset) {
1847  // Ignore PTS, it seems invalid
1848  video_pts_offset = 0;
1849  }
1850 
1851  // debug output
1852  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Video)", "video_pts_offset", video_pts_offset, "is_video", is_video);
1853  }
1854  } else {
1855  // AUDIO PACKET
1856  if (audio_pts_offset == 99999) // Has the offset been set yet?
1857  {
1858  // Find the difference between PTS and frame number
1859  // Also, determine if PTS is invalid (too far away from zero)
1860  // We compare the PTS to the timebase value equal to 1 second (which means the PTS
1861  // must be within the -1 second to +1 second of zero, otherwise we ignore it)
1862  // TODO: Please see https://github.com/OpenShot/libopenshot/pull/565#issuecomment-690985272
1863  // for ideas to improve this logic.
1864  audio_pts_offset = 0 - packet->pts;
1865  int64_t max_offset = info.audio_timebase.Reciprocal().ToFloat();
1866  if (audio_pts_offset < -max_offset || audio_pts_offset > max_offset) {
1867  // Ignore PTS, it seems invalid
1868  audio_pts_offset = 0;
1869  }
1870 
1871  // debug output
1872  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Audio)", "audio_pts_offset", audio_pts_offset, "is_video", is_video);
1873  }
1874  }
1875 }
1876 
1877 // Convert PTS into Frame Number
1878 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) {
1879  // Apply PTS offset
1880  pts = pts + video_pts_offset;
1881  int64_t previous_video_frame = current_video_frame;
1882 
1883  // Get the video packet start time (in seconds)
1884  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1885 
1886  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1887  int64_t frame = round(video_seconds * info.fps.ToDouble()) + 1;
1888 
1889  // Keep track of the expected video frame #
1890  if (current_video_frame == 0)
1891  current_video_frame = frame;
1892  else {
1893 
1894  // Sometimes frames are duplicated due to identical (or similar) timestamps
1895  if (frame == previous_video_frame) {
1896  // return -1 frame number
1897  frame = -1;
1898  } else {
1899  // Increment expected frame
1900  current_video_frame++;
1901  }
1902 
1903  if (current_video_frame < frame)
1904  // has missing frames
1905  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)", "calculated frame", frame, "previous_video_frame", previous_video_frame, "current_video_frame", current_video_frame);
1906 
1907  // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine
1908  // if we are missing a video frame.
1909  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1910  while (current_video_frame < frame) {
1911  if (!missing_video_frames.count(current_video_frame)) {
1912  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame);
1913  missing_video_frames.insert(std::pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1914  missing_video_frames_source.insert(std::pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1915  }
1916 
1917  // Mark this reader as containing missing frames
1918  has_missing_frames = true;
1919 
1920  // Increment current frame
1921  current_video_frame++;
1922  }
1923  }
1924 
1925  // Return frame #
1926  return frame;
1927 }
1928 
1929 // Convert Frame Number into Video PTS
1930 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) {
1931  // Get timestamp of this frame (in seconds)
1932  double seconds = double(frame_number) / info.fps.ToDouble();
1933 
1934  // Calculate the # of video packets in this timestamp
1935  int64_t video_pts = round(seconds / info.video_timebase.ToDouble());
1936 
1937  // Apply PTS offset (opposite)
1938  return video_pts - video_pts_offset;
1939 }
1940 
1941 // Convert Frame Number into Video PTS
1942 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) {
1943  // Get timestamp of this frame (in seconds)
1944  double seconds = double(frame_number) / info.fps.ToDouble();
1945 
1946  // Calculate the # of audio packets in this timestamp
1947  int64_t audio_pts = round(seconds / info.audio_timebase.ToDouble());
1948 
1949  // Apply PTS offset (opposite)
1950  return audio_pts - audio_pts_offset;
1951 }
1952 
1953 // Calculate Starting video frame and sample # for an audio PTS
1954 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) {
1955  // Apply PTS offset
1956  pts = pts + audio_pts_offset;
1957 
1958  // Get the audio packet start time (in seconds)
1959  double audio_seconds = double(pts) * info.audio_timebase.ToDouble();
1960 
1961  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1962  double frame = (audio_seconds * info.fps.ToDouble()) + 1;
1963 
1964  // Frame # as a whole number (no more decimals)
1965  int64_t whole_frame = int64_t(frame);
1966 
1967  // Remove the whole number, and only get the decimal of the frame
1968  double sample_start_percentage = frame - double(whole_frame);
1969 
1970  // Get Samples per frame
1971  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
1972 
1973  // Calculate the sample # to start on
1974  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
1975 
1976  // Protect against broken (i.e. negative) timestamps
1977  if (whole_frame < 1)
1978  whole_frame = 1;
1979  if (sample_start < 0)
1980  sample_start = 0;
1981 
1982  // Prepare final audio packet location
1983  AudioLocation location = {whole_frame, sample_start};
1984 
1985  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
1986  if (previous_packet_location.frame != -1) {
1987  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) {
1988  int64_t orig_frame = location.frame;
1989  int orig_start = location.sample_start;
1990 
1991  // Update sample start, to prevent gaps in audio
1992  location.sample_start = previous_packet_location.sample_start;
1993  location.frame = previous_packet_location.frame;
1994 
1995  // Debug output
1996  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
1997 
1998  } else {
1999  // Debug output
2000  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2001 
2002  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2003  for (int64_t audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
2004  if (!missing_audio_frames.count(audio_frame)) {
2005  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame);
2006  missing_audio_frames.insert(std::pair<int64_t, int64_t>(audio_frame, previous_packet_location.frame - 1));
2007  }
2008  }
2009  }
2010  }
2011 
2012  // Set previous location
2013  previous_packet_location = location;
2014 
2015  // Return the associated video frame and starting sample #
2016  return location;
2017 }
2018 
2019 // Create a new Frame (or return an existing one) and add it to the working queue.
2020 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame) {
2021  // Check working cache
2022  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
2023 
2024  if (!output) {
2025  // Lock
2026  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2027 
2028  // (re-)Check working cache
2029  output = working_cache.GetFrame(requested_frame);
2030  if(output) return output;
2031 
2032  // Create a new frame on the working cache
2033  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
2034  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
2035  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
2036  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
2037 
2038  working_cache.Add(output);
2039 
2040  // Set the largest processed frame (if this is larger)
2041  if (requested_frame > largest_frame_processed)
2042  largest_frame_processed = requested_frame;
2043  }
2044  // Return frame
2045  return output;
2046 }
2047 
2048 // Determine if frame is partial due to seek
2049 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
2050 
2051  // Sometimes a seek gets partial frames, and we need to remove them
2052  bool seek_trash = false;
2053  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
2054  if (seek_video_frame_found > max_seeked_frame) {
2055  max_seeked_frame = seek_video_frame_found;
2056  }
2057  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
2058  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) {
2059  seek_trash = true;
2060  }
2061 
2062  return seek_trash;
2063 }
2064 
2065 // Check if a frame is missing and attempt to replace its frame image (and
2066 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame) {
2067  // Lock
2068  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2069 
2070  // Increment check count for this frame (or init to 1)
2071  ++checked_frames[requested_frame];
2072 
2073  // Debug output
2074  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame", "requested_frame", requested_frame, "has_missing_frames", has_missing_frames, "missing_video_frames.size()", missing_video_frames.size(), "checked_count", checked_frames[requested_frame]);
2075 
2076  // Missing frames (sometimes frame #'s are skipped due to invalid or missing timestamps)
2077  std::map<int64_t, int64_t>::iterator itr;
2078  bool found_missing_frame = false;
2079 
2080  // Special MP3 Handling (ignore more than 1 video frame)
2081  if (info.has_audio and info.has_video) {
2082  // If MP3 with single video frame, handle this special case by copying the previously
2083  // decoded image to the new frame. Otherwise, it will spend a huge amount of
2084  // CPU time looking for missing images for all the audio-only frames.
2085  if (checked_frames[requested_frame] > 8 && !missing_video_frames.count(requested_frame) &&
2086  !processing_audio_frames.count(requested_frame) && processed_audio_frames.count(requested_frame) &&
2087  last_video_frame && last_video_frame->has_image_data && HasAlbumArt()) {
2088  missing_video_frames.insert(std::pair<int64_t, int64_t>(requested_frame, last_video_frame->number));
2089  missing_video_frames_source.insert(std::pair<int64_t, int64_t>(last_video_frame->number, requested_frame));
2090  missing_frames.Add(last_video_frame);
2091  }
2092  }
2093 
2094  // Check if requested video frame is a missing
2095  if (missing_video_frames.count(requested_frame)) {
2096  int64_t missing_source_frame = missing_video_frames.find(requested_frame)->second;
2097 
2098  // Increment missing source frame check count (or init to 1)
2099  ++checked_frames[missing_source_frame];
2100 
2101  // Get the previous frame of this missing frame (if it's available in missing cache)
2102  std::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(missing_source_frame);
2103  if (parent_frame == NULL) {
2104  parent_frame = final_cache.GetFrame(missing_source_frame);
2105  if (parent_frame != NULL) {
2106  // Add missing final frame to missing cache
2107  missing_frames.Add(parent_frame);
2108  }
2109  }
2110 
2111  // Create blank missing frame
2112  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
2113 
2114  // Debug output
2115  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame);
2116 
2117  // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
2118  if (parent_frame != NULL) {
2119  // Debug output
2120  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame);
2121 
2122  // Add this frame to the processed map (since it's already done)
2123  std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
2124  if (parent_image) {
2125  missing_frame->AddImage(std::make_shared<QImage>(*parent_image));
2126  processed_video_frames[missing_frame->number] = missing_frame->number;
2127  }
2128  }
2129  }
2130 
2131  // Check if requested audio frame is a missing
2132  if (missing_audio_frames.count(requested_frame)) {
2133 
2134  // Create blank missing frame
2135  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
2136 
2137  // Get Samples per frame (for this frame number)
2138  int samples_per_frame = Frame::GetSamplesPerFrame(missing_frame->number, info.fps, info.sample_rate, info.channels);
2139 
2140  // Debug output
2141  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Add Silence for Missing Audio Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "samples_per_frame", samples_per_frame);
2142 
2143  // Add this frame to the processed map (since it's already done)
2144  missing_frame->AddAudioSilence(samples_per_frame);
2145  processed_audio_frames[missing_frame->number] = missing_frame->number;
2146  }
2147 
2148  return found_missing_frame;
2149 }
2150 
2151 // Check the working queue, and move finished frames to the finished queue
2152 void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_frame) {
2153  // Loop through all working queue frames
2154  bool checked_count_tripped = false;
2155  int max_checked_count = 80;
2156 
2157  // Check if requested frame is 'missing'
2158  CheckMissingFrame(requested_frame);
2159 
2160  while (true) {
2161  // Get the front frame of working cache
2162  std::shared_ptr<Frame> f(working_cache.GetSmallestFrame());
2163 
2164  // Was a frame found?
2165  if (!f)
2166  // No frames found
2167  break;
2168 
2169  // Remove frames which are too old
2170  if (f->number < (requested_frame - (max_concurrent_frames * 2))) {
2171  working_cache.Remove(f->number);
2172  }
2173 
2174  // Check if this frame is 'missing'
2175  CheckMissingFrame(f->number);
2176 
2177  // Init # of times this frame has been checked so far
2178  int checked_count = 0;
2179  int checked_frames_size = 0;
2180 
2181  bool is_video_ready = false;
2182  bool is_audio_ready = false;
2183  { // limit scope of next few lines
2184  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2185  is_video_ready = processed_video_frames.count(f->number);
2186  is_audio_ready = processed_audio_frames.count(f->number);
2187 
2188  // Get check count for this frame
2189  checked_frames_size = checked_frames.size();
2190  if (!checked_count_tripped || f->number >= requested_frame)
2191  checked_count = checked_frames[f->number];
2192  else
2193  // Force checked count over the limit
2194  checked_count = max_checked_count;
2195  }
2196 
2197  if (previous_packet_location.frame == f->number && !end_of_stream)
2198  is_audio_ready = false; // don't finalize the last processed audio frame
2199  bool is_seek_trash = IsPartialFrame(f->number);
2200 
2201  // Adjust for available streams
2202  if (!info.has_video) is_video_ready = true;
2203  if (!info.has_audio) is_audio_ready = true;
2204 
2205  // Make final any frames that get stuck (for whatever reason)
2206  if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
2207  // Debug output
2208  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
2209 
2210  // Trigger checked count tripped mode (clear out all frames before requested frame)
2211  checked_count_tripped = true;
2212 
2213  if (info.has_video && !is_video_ready && last_video_frame) {
2214  // Copy image from last frame
2215  f->AddImage(std::make_shared<QImage>(*last_video_frame->GetImage()));
2216  is_video_ready = true;
2217  }
2218 
2219  if (info.has_audio && !is_audio_ready) {
2220  // Mark audio as processed, and indicate the frame has audio data
2221  is_audio_ready = true;
2222  }
2223  }
2224 
2225  // Debug output
2226  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
2227 
2228  // Check if working frame is final
2229  if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash) {
2230  // Debug output
2231  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "requested_frame", requested_frame, "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "end_of_stream", end_of_stream);
2232 
2233  if (!is_seek_trash) {
2234  // Add missing image (if needed - sometimes end_of_stream causes frames with only audio)
2235  if (info.has_video && !is_video_ready && last_video_frame)
2236  // Copy image from last frame
2237  f->AddImage(std::make_shared<QImage>(*last_video_frame->GetImage()));
2238 
2239  // Reset counter since last 'final' frame
2240  num_checks_since_final = 0;
2241 
2242  // Move frame to final cache
2243  final_cache.Add(f);
2244 
2245  // Add to missing cache (if another frame depends on it)
2246  {
2247  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2248  if (missing_video_frames_source.count(f->number)) {
2249  // Debug output
2250  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count());
2251  missing_frames.Add(f);
2252  }
2253 
2254  // Remove from 'checked' count
2255  checked_frames.erase(f->number);
2256  }
2257 
2258  // Remove frame from working cache
2259  working_cache.Remove(f->number);
2260 
2261  // Update last frame processed
2262  last_frame = f->number;
2263 
2264  } else {
2265  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
2266  working_cache.Remove(f->number);
2267  }
2268 
2269  } else {
2270  // Stop looping
2271  break;
2272  }
2273  }
2274 }
2275 
2276 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
2277 void FFmpegReader::CheckFPS() {
2278  check_fps = true;
2279 
2280 
2281  int first_second_counter = 0;
2282  int second_second_counter = 0;
2283  int third_second_counter = 0;
2284  int forth_second_counter = 0;
2285  int fifth_second_counter = 0;
2286  int frames_detected = 0;
2287  int64_t pts = 0;
2288 
2289  // Loop through the stream
2290  while (true) {
2291  // Get the next packet (if any)
2292  if (GetNextPacket() < 0)
2293  // Break loop when no more packets found
2294  break;
2295 
2296  // Video packet
2297  if (packet->stream_index == videoStream) {
2298  // Check if the AVFrame is finished and set it
2299  if (GetAVFrame()) {
2300  // Update PTS / Frame Offset (if any)
2301  UpdatePTSOffset(true);
2302 
2303  // Get PTS of this packet
2304  pts = GetVideoPTS();
2305 
2306  // Remove pFrame
2307  RemoveAVFrame(pFrame);
2308 
2309  // Apply PTS offset
2310  pts += video_pts_offset;
2311 
2312  // Get the video packet start time (in seconds)
2313  double video_seconds = double(pts) * info.video_timebase.ToDouble();
2314 
2315  // Increment the correct counter
2316  if (video_seconds <= 1.0)
2317  first_second_counter++;
2318  else if (video_seconds > 1.0 && video_seconds <= 2.0)
2319  second_second_counter++;
2320  else if (video_seconds > 2.0 && video_seconds <= 3.0)
2321  third_second_counter++;
2322  else if (video_seconds > 3.0 && video_seconds <= 4.0)
2323  forth_second_counter++;
2324  else if (video_seconds > 4.0 && video_seconds <= 5.0)
2325  fifth_second_counter++;
2326 
2327  // Increment counters
2328  frames_detected++;
2329  }
2330  }
2331  }
2332 
2333  // Double check that all counters have greater than zero (or give up)
2334  if (second_second_counter != 0 && third_second_counter != 0 && forth_second_counter != 0 && fifth_second_counter != 0) {
2335  // Calculate average FPS (average of first few seconds)
2336  int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
2337  int avg_fps = round(sum_fps / 4.0f);
2338 
2339  // Update FPS
2340  info.fps = Fraction(avg_fps, 1);
2341 
2342  // Update Duration and Length
2343  info.video_length = frames_detected;
2344  info.duration = frames_detected / (sum_fps / 4.0f);
2345 
2346  // Update video bit rate
2348  } else if (second_second_counter != 0 && third_second_counter != 0) {
2349  // Calculate average FPS (only on second 2)
2350  int sum_fps = second_second_counter;
2351 
2352  // Update FPS
2353  info.fps = Fraction(sum_fps, 1);
2354 
2355  // Update Duration and Length
2356  info.video_length = frames_detected;
2357  info.duration = frames_detected / float(sum_fps);
2358 
2359  // Update video bit rate
2361  } else {
2362  // Too short to determine framerate, just default FPS
2363  // Set a few important default video settings (so audio can be divided into frames)
2364  info.fps.num = 30;
2365  info.fps.den = 1;
2366 
2367  // Calculate number of frames
2368  info.video_length = frames_detected;
2369  info.duration = frames_detected / info.fps.ToFloat();
2370  }
2371 }
2372 
2373 // Remove AVFrame from cache (and deallocate its memory)
2374 void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
2375  // Remove pFrame (if exists)
2376  if (remove_frame) {
2377  // Free memory
2378  av_freep(&remove_frame->data[0]);
2379 #ifndef WIN32
2380  AV_FREE_FRAME(&remove_frame);
2381 #endif
2382  }
2383 }
2384 
2385 // Remove AVPacket from cache (and deallocate its memory)
2386 void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) {
2387  // deallocate memory for packet
2388  AV_FREE_PACKET(remove_packet);
2389 
2390  // Delete the object
2391  delete remove_packet;
2392 }
2393 
2394 /// Get the smallest video frame that is still being processed
2395 int64_t FFmpegReader::GetSmallestVideoFrame() {
2396  // Loop through frame numbers
2397  std::map<int64_t, int64_t>::iterator itr;
2398  int64_t smallest_frame = -1;
2399  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2400  for (itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr) {
2401  if (itr->first < smallest_frame || smallest_frame == -1)
2402  smallest_frame = itr->first;
2403  }
2404 
2405  // Return frame number
2406  return smallest_frame;
2407 }
2408 
2409 /// Get the smallest audio frame that is still being processed
2410 int64_t FFmpegReader::GetSmallestAudioFrame() {
2411  // Loop through frame numbers
2412  std::map<int64_t, int64_t>::iterator itr;
2413  int64_t smallest_frame = -1;
2414  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2415  for (itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr) {
2416  if (itr->first < smallest_frame || smallest_frame == -1)
2417  smallest_frame = itr->first;
2418  }
2419 
2420  // Return frame number
2421  return smallest_frame;
2422 }
2423 
2424 // Generate JSON string of this object
2425 std::string FFmpegReader::Json() const {
2426 
2427  // Return formatted string
2428  return JsonValue().toStyledString();
2429 }
2430 
2431 // Generate Json::Value for this object
2432 Json::Value FFmpegReader::JsonValue() const {
2433 
2434  // Create root json object
2435  Json::Value root = ReaderBase::JsonValue(); // get parent properties
2436  root["type"] = "FFmpegReader";
2437  root["path"] = path;
2438 
2439  // return JsonValue
2440  return root;
2441 }
2442 
2443 // Load JSON string into this object
2444 void FFmpegReader::SetJson(const std::string value) {
2445 
2446  // Parse JSON string into JSON objects
2447  try {
2448  const Json::Value root = openshot::stringToJson(value);
2449  // Set all values that match
2450  SetJsonValue(root);
2451  }
2452  catch (const std::exception& e) {
2453  // Error parsing JSON (or missing keys)
2454  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
2455  }
2456 }
2457 
2458 // Load Json::Value into this object
2459 void FFmpegReader::SetJsonValue(const Json::Value root) {
2460 
2461  // Set parent data
2463 
2464  // Set data from Json (if key is found)
2465  if (!root["path"].isNull())
2466  path = root["path"].asString();
2467 
2468  // Re-Open path, and re-init everything (if needed)
2469  if (is_open) {
2470  Close();
2471  Open();
2472  }
2473 }
#define AV_RESET_FRAME(av_frame)
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
Definition: Fraction.h:50
#define SWRCONTEXT
Point GetMaxPoint() const
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:248
void Close() override
Close File.
#define AV_FIND_DECODER_CODEC_ID(av_stream)
#define SWR_INIT(ctx)
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:68
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
#define AV_REGISTER_ALL
int preview_width
Optional preview width of timeline image. If your preview window is smaller than the timeline...
Definition: TimelineBase.h:42
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:54
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
float duration
Length of time (in seconds)
Definition: ReaderBase.h:65
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
Json::Value JsonValue() const override
Generate Json::Value for this object.
Header file for FFmpegReader class.
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:84
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:49
Scale the clip until both height and width fill the canvas (cropping the overlap) ...
Definition: Enums.h:54
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define SWR_CLOSE(ctx)
#define SWR_FREE(ctx)
Fraction Reciprocal()
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:94
openshot::Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:269
void SetJson(const std::string value) override
Load JSON string into this object.
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34
#define OPEN_MP_NUM_PROCESSORS
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
#define AVCODEC_REGISTER_ALL
openshot::Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:270
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:337
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:62
#define AV_GET_CODEC_CONTEXT(av_stream, av_codec)
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:66
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:190
std::shared_ptr< openshot::Frame > GetSmallestFrame()
Get the smallest frame number.
AVPixelFormat hw_de_av_pix_fmt_global
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:81
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:536
Header file for all Exception classes.
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:63
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:116
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:107
#define AV_FREE_CONTEXT(av_context)
Exception when no valid codec is found for a file.
Definition: Exceptions.h:157
Exception when memory could not be allocated.
Definition: Exceptions.h:321
double Y
The Y value of the coordinate (usually representing the value of the property being animated) ...
Definition: Coordinate.h:57
int HARDWARE_DECODER
Use video codec for faster video decoding (if supported)
Definition: Settings.h:80
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:85
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:75
#define FF_NUM_PROCESSORS
#define AV_FREE_PACKET(av_packet)
juce::CriticalSection processingCriticalSection
Definition: ReaderBase.h:102
Exception when no streams are found in the file.
Definition: Exceptions.h:269
int height
The height of the video (in pixels)
Definition: ReaderBase.h:67
#define AV_ALLOCATE_FRAME()
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:77
Exception for files that can not be found or opened.
Definition: Exceptions.h:173
int hw_de_on
This class represents a fraction.
Definition: Fraction.h:48
std::map< std::string, std::string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:87
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:84
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
AVHWDeviceType hw_de_av_device_type_global
int HW_DE_DEVICE_SET
Which GPU to use to decode (0 is the first)
Definition: Settings.h:98
#define PixelFormat
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:56
void Clear()
Clear the cache of all frames.
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:171
#define MY_INPUT_BUFFER_PADDING_SIZE
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:254
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:52
int64_t Count()
Count the frames in the queue.
openshot::Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:86
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:74
if(!codec) codec
This namespace is the default namespace for all code in the openshot library.
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
AVDictionary * opts
Coordinate co
This is the primary coordinate.
Definition: Point.h:84
Exception for invalid JSON.
Definition: Exceptions.h:205
#define AV_GET_CODEC_TYPE(av_stream)
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:69
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: ReaderBase.h:73
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:60
openshot::ScaleType scale
The scale determines how a clip should be resized to fit its parent.
Definition: Clip.h:170
openshot::TimelineBase * ParentTimeline()
Get the associated Timeline pointer (if any)
Definition: ClipBase.h:113
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: Settings.cpp:41
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:71
std::string Json() const override
Generate JSON string of this object.
#define SWR_ALLOC()
int DE_LIMIT_WIDTH_MAX
Maximum columns that hardware decode can handle.
Definition: Settings.h:95
#define PIX_FMT_RGBA
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: ReaderBase.h:72
bool GetIsDurationKnown()
Return true if frame can be read with GetFrame()
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
int preview_height
Optional preview width of timeline image. If your preview window is smaller than the timeline...
Definition: TimelineBase.h:43
void Remove(int64_t frame_number)
Remove a specific frame.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:233
void Open() override
Open File - which is called by the constructor automatically.
virtual ~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
Definition: Fraction.h:51
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:83
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:76
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:55
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:70
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:80
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:59
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:82
int DE_LIMIT_HEIGHT_MAX
Maximum rows that hardware decode can handle.
Definition: Settings.h:92