OpenShot Library | libopenshot  0.3.1
Frame.cpp
Go to the documentation of this file.
1 
9 // Copyright (c) 2008-2019 OpenShot Studios, LLC
10 //
11 // SPDX-License-Identifier: LGPL-3.0-or-later
12 
13 #include <thread> // for std::this_thread::sleep_for
14 #include <chrono> // for std::chrono::milliseconds
15 #include <sstream>
16 #include <iomanip>
17 
18 #include "Frame.h"
19 #include "AudioBufferSource.h"
20 #include "AudioResampler.h"
21 #include "QtUtilities.h"
22 
23 #include <AppConfig.h>
24 #include <juce_audio_basics/juce_audio_basics.h>
25 #include <juce_audio_devices/juce_audio_devices.h>
26 
27 #include <QApplication>
28 #include <QImage>
29 #include <QPixmap>
30 #include <QBitmap>
31 #include <QColor>
32 #include <QString>
33 #include <QVector>
34 #include <QPainter>
35 #include <QHBoxLayout>
36 #include <QWidget>
37 #include <QLabel>
38 #include <QPointF>
39 #include <QWidget>
40 
41 using namespace std;
42 using namespace openshot;
43 
44 // Constructor - image & audio
45 Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels)
46  : audio(std::make_shared<juce::AudioBuffer<float>>(channels, samples)),
47  number(number), width(width), height(height),
48  pixel_ratio(1,1), color(color),
49  channels(channels), channel_layout(LAYOUT_STEREO),
50  sample_rate(44100),
51  has_audio_data(false), has_image_data(false),
52  max_audio_sample(0)
53 {
54  // zero (fill with silence) the audio buffer
55  audio->clear();
56 }
57 
58 // Delegating Constructor - blank frame
59 Frame::Frame() : Frame::Frame(1, 1, 1, "#000000", 0, 2) {}
60 
61 // Delegating Constructor - image only
62 Frame::Frame(int64_t number, int width, int height, std::string color)
63  : Frame::Frame(number, width, height, color, 0, 2) {}
64 
65 // Delegating Constructor - audio only
66 Frame::Frame(int64_t number, int samples, int channels)
67  : Frame::Frame(number, 1, 1, "#000000", samples, channels) {}
68 
69 
70 // Copy constructor
71 Frame::Frame ( const Frame &other )
72 {
73  // copy pointers and data
74  DeepCopy(other);
75 }
76 
77 // Assignment operator
79 {
80  // copy pointers and data
81  DeepCopy(other);
82 
83  return *this;
84 }
85 
86 // Copy data and pointers from another Frame instance
87 void Frame::DeepCopy(const Frame& other)
88 {
89  number = other.number;
90  channels = other.channels;
91  width = other.width;
92  height = other.height;
93  channel_layout = other.channel_layout;
96  sample_rate = other.sample_rate;
97  pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
98  color = other.color;
99  max_audio_sample = other.max_audio_sample;
100 
101  if (other.image)
102  image = std::make_shared<QImage>(*(other.image));
103  if (other.audio)
104  audio = std::make_shared<juce::AudioBuffer<float>>(*(other.audio));
105  if (other.wave_image)
106  wave_image = std::make_shared<QImage>(*(other.wave_image));
107 }
108 
109 // Destructor
111  // Clear all pointers
112  image.reset();
113  audio.reset();
114  #ifdef USE_OPENCV
115  imagecv.release();
116  #endif
117 }
118 
119 // Display the frame image to the screen (primarily used for debugging reasons)
121 {
122  if (!QApplication::instance()) {
123  // Only create the QApplication once
124  static int argc = 1;
125  static char* argv[1] = {NULL};
126  previewApp = std::make_shared<QApplication>(argc, argv);
127  }
128 
129  // Get preview image
130  std::shared_ptr<QImage> previewImage = GetImage();
131 
132  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
133  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
134  {
135  // Resize to fix DAR
136  previewImage = std::make_shared<QImage>(previewImage->scaled(
137  previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
138  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
139  }
140 
141  // Create window
142  QWidget previewWindow;
143  previewWindow.setStyleSheet("background-color: #000000;");
144  QHBoxLayout layout;
145 
146  // Create label with current frame's image
147  QLabel previewLabel;
148  previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
149  previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
150  layout.addWidget(&previewLabel);
151 
152  // Show the window
153  previewWindow.setLayout(&layout);
154  previewWindow.show();
155  previewApp->exec();
156 }
157 
158 // Get an audio waveform image
159 std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
160 {
161  // Clear any existing waveform image
162  ClearWaveform();
163 
164  // Init a list of lines
165  QVector<QPointF> lines;
166  QVector<QPointF> labels;
167 
168  // Calculate width of an image based on the # of samples
169  int total_samples = GetAudioSamplesCount();
170  if (total_samples > 0)
171  {
172  // If samples are present...
173  int new_height = 200 * audio->getNumChannels();
174  int height_padding = 20 * (audio->getNumChannels() - 1);
175  int total_height = new_height + height_padding;
176  int total_width = 0;
177  float zero_height = 1.0; // Used to clamp near-zero vales to this value to prevent gaps
178 
179  // Loop through each audio channel
180  float Y = 100.0;
181  for (int channel = 0; channel < audio->getNumChannels(); channel++)
182  {
183  float X = 0.0;
184 
185  // Get audio for this channel
186  const float *samples = audio->getReadPointer(channel);
187 
188  for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
189  {
190  // Sample value (scaled to -100 to 100)
191  float value = samples[sample] * 100.0;
192 
193  // Set threshold near zero (so we don't allow near-zero values)
194  // This prevents empty gaps from appearing in the waveform
195  if (value > -zero_height && value < 0.0) {
196  value = -zero_height;
197  } else if (value > 0.0 && value < zero_height) {
198  value = zero_height;
199  }
200 
201  // Append a line segment for each sample
202  lines.push_back(QPointF(X, Y));
203  lines.push_back(QPointF(X, Y - value));
204  }
205 
206  // Add Channel Label Coordinate
207  labels.push_back(QPointF(5.0, Y - 5.0));
208 
209  // Increment Y
210  Y += (200 + height_padding);
211  total_width = X;
212  }
213 
214  // Create blank image
215  wave_image = std::make_shared<QImage>(
216  total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
217  wave_image->fill(QColor(0,0,0,0));
218 
219  // Load QPainter with wave_image device
220  QPainter painter(wave_image.get());
221 
222  // Set pen color
223  QPen pen;
224  pen.setColor(QColor(Red, Green, Blue, Alpha));
225  pen.setWidthF(1.0);
226  pen.setStyle(Qt::SolidLine);
227  painter.setPen(pen);
228 
229  // Draw the waveform
230  painter.drawLines(lines);
231  painter.end();
232  }
233  else
234  {
235  // No audio samples present
236  wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
237  wave_image->fill(QColor(QString::fromStdString("#000000")));
238  }
239 
240  // Resize Image (if needed)
241  if (wave_image->width() != width || wave_image->height() != height) {
242  QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
243  wave_image = std::make_shared<QImage>(scaled_wave_image);
244  }
245 
246  // Return new image
247  return wave_image;
248 }
249 
250 // Clear the waveform image (and deallocate its memory)
252 {
253  if (wave_image)
254  wave_image.reset();
255 }
256 
257 // Get an audio waveform image pixels
258 const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
259 {
260  // Get audio wave form image
261  wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
262 
263  // Return array of pixel packets
264  return wave_image->constBits();
265 }
266 
267 // Display the wave form
269 {
270  // Get audio wave form image
271  GetWaveform(720, 480, 0, 123, 255, 255);
272 
273  if (!QApplication::instance()) {
274  // Only create the QApplication once
275  static int argc = 1;
276  static char* argv[1] = {NULL};
277  previewApp = std::make_shared<QApplication>(argc, argv);
278  }
279 
280  // Create window
281  QWidget previewWindow;
282  previewWindow.setStyleSheet("background-color: #000000;");
283  QHBoxLayout layout;
284 
285  // Create label with current frame's waveform image
286  QLabel previewLabel;
287  previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
288  previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
289  layout.addWidget(&previewLabel);
290 
291  // Show the window
292  previewWindow.setLayout(&layout);
293  previewWindow.show();
294  previewApp->exec();
295 
296  // Deallocate waveform image
297  ClearWaveform();
298 }
299 
300 // Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
301 float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
302 {
303  if (channel > 0) {
304  // return average magnitude for a specific channel/sample range
305  return audio->getMagnitude(channel, sample, magnitude_range);
306 
307  } else {
308  // Return average magnitude for all channels
309  return audio->getMagnitude(sample, magnitude_range);
310  }
311 }
312 
313 // Get an array of sample data (and optional reverse the sample values)
314 float* Frame::GetAudioSamples(int channel) {
315 
316  // Copy audio data
317  juce::AudioBuffer<float> *buffer(audio.get());
318 
319  // return JUCE audio data for this channel
320  return buffer->getWritePointer(channel);
321 }
322 
323 // Get an array of sample data (all channels interleaved together), using any sample rate
324 float* Frame::GetInterleavedAudioSamples(int* sample_count)
325 {
326  // Copy audio data
327  juce::AudioBuffer<float> *buffer(audio.get());
328 
329  float *output = NULL;
330  int num_of_channels = audio->getNumChannels();
331  int num_of_samples = GetAudioSamplesCount();
332 
333  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
334  output = new float[num_of_channels * num_of_samples];
335  int position = 0;
336 
337  // Loop through samples in each channel (combining them)
338  for (int sample = 0; sample < num_of_samples; sample++)
339  {
340  for (int channel = 0; channel < num_of_channels; channel++)
341  {
342  // Add sample to output array
343  output[position] = buffer->getReadPointer(channel)[sample];
344 
345  // increment position
346  position++;
347  }
348  }
349 
350  // Update sample count (since it might have changed due to resampling)
351  *sample_count = num_of_samples;
352 
353  // return combined array
354  return output;
355 }
356 
357 // Get number of audio channels
359 {
360  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
361  if (audio)
362  return audio->getNumChannels();
363  else
364  return 0;
365 }
366 
367 // Get number of audio samples
369 {
370  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
371  return max_audio_sample;
372 }
373 
375 {
376  return audio.get();
377 }
378 
379 // Get the size in bytes of this frame (rough estimate)
381 {
382  int64_t total_bytes = 0;
383  if (image) {
384  total_bytes += static_cast<int64_t>(
385  width * height * sizeof(char) * 4);
386  }
387  if (audio) {
388  // approximate audio size (sample rate / 24 fps)
389  total_bytes += (sample_rate / 24.0) * sizeof(float);
390  }
391 
392  // return size of this frame
393  return total_bytes;
394 }
395 
396 // Get pixel data (as packets)
397 const unsigned char* Frame::GetPixels()
398 {
399  // Check for blank image
400  if (!image)
401  // Fill with black
402  AddColor(width, height, color);
403 
404  // Return array of pixel packets
405  return image->constBits();
406 }
407 
408 // Get pixel data (for only a single scan-line)
409 const unsigned char* Frame::GetPixels(int row)
410 {
411  // Check for blank image
412  if (!image)
413  // Fill with black
414  AddColor(width, height, color);
415 
416  // Return array of pixel packets
417  return image->constScanLine(row);
418 }
419 
420 // Check a specific pixel color value (returns True/False)
421 bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) {
422  int col_pos = col * 4; // Find column array position
423  if (!image || row < 0 || row >= (height - 1) ||
424  col_pos < 0 || col_pos >= (width - 1) ) {
425  // invalid row / col
426  return false;
427  }
428  // Check pixel color
429  const unsigned char* pixels = GetPixels(row);
430  if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
431  pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
432  pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
433  pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
434  // Pixel color matches successfully
435  return true;
436  } else {
437  // Pixel color does not match
438  return false;
439  }
440 }
441 
442 // Set Pixel Aspect Ratio
443 void Frame::SetPixelRatio(int num, int den)
444 {
445  pixel_ratio.num = num;
446  pixel_ratio.den = den;
447 }
448 
449 // Set frame number
450 void Frame::SetFrameNumber(int64_t new_number)
451 {
452  number = new_number;
453 }
454 
455 // Calculate the # of samples per video frame (for a specific frame number and frame rate)
456 int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
457 {
458  // Get the total # of samples for the previous frame, and the current frame (rounded)
459  double fps_rate = fps.Reciprocal().ToDouble();
460 
461  // Determine previous samples total, and make sure it's evenly divisible by the # of channels
462  double previous_samples = (sample_rate * fps_rate) * (number - 1);
463  double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
464  previous_samples -= previous_samples_remainder;
465 
466  // Determine the current samples total, and make sure it's evenly divisible by the # of channels
467  double total_samples = (sample_rate * fps_rate) * number;
468  double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
469  total_samples -= total_samples_remainder;
470 
471  // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
472  // be evenly divided into frames, so each frame can have have different # of samples.
473  int samples_per_frame = round(total_samples - previous_samples);
474  if (samples_per_frame < 0)
475  samples_per_frame = 0;
476  return samples_per_frame;
477 }
478 
479 // Calculate the # of samples per video frame (for the current frame number)
480 int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
481 {
482  return GetSamplesPerFrame(number, fps, sample_rate, channels);
483 }
484 
485 // Get height of image
487 {
488  return height;
489 }
490 
491 // Get height of image
493 {
494  return width;
495 }
496 
497 // Get the original sample rate of this frame's audio data
499 {
500  return sample_rate;
501 }
502 
503 // Get the original sample rate of this frame's audio data
505 {
506  return channel_layout;
507 }
508 
509 
510 // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
511 void Frame::Save(std::string path, float scale, std::string format, int quality)
512 {
513  // Get preview image
514  std::shared_ptr<QImage> previewImage = GetImage();
515 
516  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
517  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
518  {
519  // Resize to fix DAR
520  previewImage = std::make_shared<QImage>(previewImage->scaled(
521  previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
522  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
523  }
524 
525  // scale image if needed
526  if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
527  {
528  // Resize image
529  previewImage = std::make_shared<QImage>(previewImage->scaled(
530  previewImage->size().width() * scale, previewImage->size().height() * scale,
531  Qt::KeepAspectRatio, Qt::SmoothTransformation));
532  }
533 
534  // Save image
535  previewImage->save(QString::fromStdString(path), format.c_str(), quality);
536 }
537 
538 // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
539 void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
540  std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate) {
541 
542  // Create blank thumbnail image & fill background color
543  auto thumbnail = std::make_shared<QImage>(
544  new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
545  thumbnail->fill(QColor(QString::fromStdString(background_color)));
546 
547  // Create painter
548  QPainter painter(thumbnail.get());
549  painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
550 
551  // Get preview image
552  std::shared_ptr<QImage> previewImage = GetImage();
553 
554  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
555  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
556  {
557  // Calculate correct DAR (display aspect ratio)
558  int aspect_width = previewImage->size().width();
559  int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
560 
561  // Resize to fix DAR
562  previewImage = std::make_shared<QImage>(previewImage->scaled(
563  aspect_width, aspect_height,
564  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
565  }
566 
567  // Resize frame image
568  if (ignore_aspect)
569  // Ignore aspect ratio
570  previewImage = std::make_shared<QImage>(previewImage->scaled(
571  new_width, new_height,
572  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
573  else
574  // Maintain aspect ratio
575  previewImage = std::make_shared<QImage>(previewImage->scaled(
576  new_width, new_height,
577  Qt::KeepAspectRatio, Qt::SmoothTransformation));
578 
579  // Composite frame image onto background (centered)
580  int x = (new_width - previewImage->size().width()) / 2.0; // center
581  int y = (new_height - previewImage->size().height()) / 2.0; // center
582  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
583 
584 
585  // Create transform and rotate (if needed)
586  QTransform transform;
587  float origin_x = previewImage->width() / 2.0;
588  float origin_y = previewImage->height() / 2.0;
589  transform.translate(origin_x, origin_y);
590  transform.rotate(rotate);
591  transform.translate(-origin_x,-origin_y);
592  painter.setTransform(transform);
593 
594  // Draw image onto QImage
595  painter.drawImage(x, y, *previewImage);
596 
597 
598  // Overlay Image (if any)
599  if (overlay_path != "") {
600  // Open overlay
601  auto overlay = std::make_shared<QImage>();
602  overlay->load(QString::fromStdString(overlay_path));
603 
604  // Set pixel format
605  overlay = std::make_shared<QImage>(
606  overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
607 
608  // Resize to fit
609  overlay = std::make_shared<QImage>(overlay->scaled(
610  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
611 
612  // Composite onto thumbnail
613  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
614  painter.drawImage(0, 0, *overlay);
615  }
616 
617 
618  // Mask Image (if any)
619  if (mask_path != "") {
620  // Open mask
621  auto mask = std::make_shared<QImage>();
622  mask->load(QString::fromStdString(mask_path));
623 
624  // Set pixel format
625  mask = std::make_shared<QImage>(
626  mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
627 
628  // Resize to fit
629  mask = std::make_shared<QImage>(mask->scaled(
630  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
631 
632  // Negate mask
633  mask->invertPixels();
634 
635  // Get pixels
636  unsigned char *pixels = static_cast<unsigned char *>(thumbnail->bits());
637  const unsigned char *mask_pixels = static_cast<const unsigned char *>(mask->constBits());
638 
639  // Convert the mask image to grayscale
640  // Loop through pixels
641  for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
642  {
643  // Get the RGB values from the pixel
644  int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
645  int Frame_Alpha = pixels[byte_index + 3];
646  int Mask_Value = constrain(Frame_Alpha - gray_value);
647 
648  // Set all alpha pixels to gray value
649  pixels[byte_index + 3] = Mask_Value;
650  }
651  }
652 
653 
654  // End painter
655  painter.end();
656 
657  // Save image
658  thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
659 }
660 
661 // Constrain a color value from 0 to 255
662 int Frame::constrain(int color_value)
663 {
664  // Constrain new color from 0 to 255
665  if (color_value < 0)
666  color_value = 0;
667  else if (color_value > 255)
668  color_value = 255;
669 
670  return color_value;
671 }
672 
673 void Frame::AddColor(int new_width, int new_height, std::string new_color)
674 {
675  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
676  // Update parameters
677  width = new_width;
678  height = new_height;
679  color = new_color;
680  AddColor(QColor(QString::fromStdString(new_color)));
681 }
682 
683 // Add (or replace) pixel data to the frame (based on a solid color)
684 void Frame::AddColor(const QColor& new_color)
685 {
686  // Create new image object, and fill with pixel data
687  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
688  image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
689 
690  // Fill with solid color
691  image->fill(new_color);
692  has_image_data = true;
693 }
694 
695 // Add (or replace) pixel data to the frame
697  int new_width, int new_height, int bytes_per_pixel,
698  QImage::Format type, const unsigned char *pixels_)
699 {
700  if (has_image_data) {
701  // Delete the previous QImage
702  image.reset();
703  }
704 
705  // Create new image object from pixel data
706  auto new_image = std::make_shared<QImage>(
707  pixels_,
708  new_width, new_height,
709  new_width * bytes_per_pixel,
710  type,
711  (QImageCleanupFunction) &openshot::cleanUpBuffer,
712  (void*) pixels_
713  );
714  AddImage(new_image);
715 }
716 
717 // Add (or replace) pixel data to the frame
718 void Frame::AddImage(std::shared_ptr<QImage> new_image)
719 {
720  // Ignore blank images
721  if (!new_image)
722  return;
723 
724  // assign image data
725  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
726  image = new_image;
727 
728  // Always convert to Format_RGBA8888_Premultiplied (if different)
729  if (image->format() != QImage::Format_RGBA8888_Premultiplied)
730  *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
731 
732  // Update height and width
733  width = image->width();
734  height = image->height();
735  has_image_data = true;
736 }
737 
738 // Add (or replace) pixel data to the frame (for only the odd or even lines)
739 void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
740 {
741  // Ignore blank new_image
742  if (!new_image)
743  return;
744 
745  // Check for blank source image
746  if (!image) {
747  // Replace the blank source image
748  AddImage(new_image);
749 
750  } else {
751  // Ignore image of different sizes or formats
752  bool ret=false;
753  if (image == new_image || image->size() != new_image->size()) {
754  ret = true;
755  }
756  else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
757  new_image = std::make_shared<QImage>(
758  new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
759  }
760  if (ret) {
761  return;
762  }
763 
764  // Get the frame's image
765  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
766  unsigned char *pixels = image->bits();
767  const unsigned char *new_pixels = new_image->constBits();
768 
769  // Loop through the scanlines of the image (even or odd)
770  int start = 0;
771  if (only_odd_lines)
772  start = 1;
773 
774  for (int row = start; row < image->height(); row += 2) {
775  int offset = row * image->bytesPerLine();
776  memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
777  }
778 
779  // Update height and width
780  height = image->height();
781  width = image->width();
782  has_image_data = true;
783  }
784 }
785 
786 
787 // Resize audio container to hold more (or less) samples and channels
788 void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
789 {
790  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
791 
792  // Resize JUCE audio buffer
793  audio->setSize(channels, length, true, true, false);
794  channel_layout = layout;
795  sample_rate = rate;
796 
797  // Calculate max audio sample added
798  max_audio_sample = length;
799 }
800 
801 // Reverse the audio buffer of this frame (will only reverse a single time, regardless of how many times
802 // you invoke this method)
804  if (audio && !audio_reversed) {
805  // Reverse audio buffer
806  audio->reverse(0, audio->getNumSamples());
807  audio_reversed = true;
808  }
809 }
810 
811 // Add audio samples to a specific channel
812 void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
813  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
814 
815  // Clamp starting sample to 0
816  int destStartSampleAdjusted = max(destStartSample, 0);
817 
818  // Extend audio container to hold more (or less) samples and channels.. if needed
819  int new_length = destStartSampleAdjusted + numSamples;
820  int new_channel_length = audio->getNumChannels();
821  if (destChannel >= new_channel_length)
822  new_channel_length = destChannel + 1;
823  if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
824  audio->setSize(new_channel_length, new_length, true, true, false);
825 
826  // Clear the range of samples first (if needed)
827  if (replaceSamples)
828  audio->clear(destChannel, destStartSampleAdjusted, numSamples);
829 
830  // Add samples to frame's audio buffer
831  audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
832  has_audio_data = true;
833 
834  // Calculate max audio sample added
835  if (new_length > max_audio_sample)
836  max_audio_sample = new_length;
837 
838  // Reset audio reverse flag
839  audio_reversed = false;
840 }
841 
842 // Apply gain ramp (i.e. fading volume)
843 void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
844 {
845  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
846 
847  // Apply gain ramp
848  audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
849 }
850 
851 // Get pointer to Magick++ image object
852 std::shared_ptr<QImage> Frame::GetImage()
853 {
854  // Check for blank image
855  if (!image)
856  // Fill with black
857  AddColor(width, height, color);
858 
859  return image;
860 }
861 
862 #ifdef USE_OPENCV
863 
864 // Convert Qimage to Mat
865 cv::Mat Frame::Qimage2mat( std::shared_ptr<QImage>& qimage) {
866 
867  cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
868  cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
869  int from_to[] = { 0,0, 1,1, 2,2 };
870  cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
871  cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
872  return mat2;
873 }
874 
875 // Get pointer to OpenCV image object
877 {
878  // Check for blank image
879  if (!image)
880  // Fill with black
881  AddColor(width, height, color);
882 
883  // if (imagecv.empty())
884  // Convert Qimage to Mat
885  imagecv = Qimage2mat(image);
886 
887  return imagecv;
888 }
889 
890 std::shared_ptr<QImage> Frame::Mat2Qimage(cv::Mat img){
891  cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
892  QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
893 
894  std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
895 
896  // Always convert to RGBA8888 (if different)
897  if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
898  *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
899 
900  return imgIn;
901 }
902 
903 // Set pointer to OpenCV image object
904 void Frame::SetImageCV(cv::Mat _image)
905 {
906  imagecv = _image;
907  image = Mat2Qimage(_image);
908 }
909 #endif
910 
911 // Play audio samples for this frame
913 {
914  // Check if samples are present
915  if (!GetAudioSamplesCount())
916  return;
917 
918  juce::AudioDeviceManager deviceManager;
919  juce::String error = deviceManager.initialise (
920  0, /* number of input channels */
921  2, /* number of output channels */
922  0, /* no XML settings.. */
923  true /* select default device on failure */);
924 
925  // Output error (if any)
926  if (error.isNotEmpty()) {
927  cout << "Error on initialise(): " << error << endl;
928  }
929 
930  juce::AudioSourcePlayer audioSourcePlayer;
931  deviceManager.addAudioCallback (&audioSourcePlayer);
932 
933  std::unique_ptr<AudioBufferSource> my_source;
934  my_source.reset (new AudioBufferSource (audio.get()));
935 
936  // Create TimeSliceThread for audio buffering
937  juce::TimeSliceThread my_thread("Audio buffer thread");
938 
939  // Start thread
940  my_thread.startThread();
941 
942  juce::AudioTransportSource transport1;
943  transport1.setSource (my_source.get(),
944  5000, // tells it to buffer this many samples ahead
945  &my_thread,
946  (double) sample_rate,
947  audio->getNumChannels()); // sample rate of source
948  transport1.setPosition (0);
949  transport1.setGain(1.0);
950 
951 
952  // Create MIXER
953  juce::MixerAudioSource mixer;
954  mixer.addInputSource(&transport1, false);
955  audioSourcePlayer.setSource (&mixer);
956 
957  // Start transports
958  transport1.start();
959 
960  while (transport1.isPlaying())
961  {
962  cout << "playing" << endl;
963  std::this_thread::sleep_for(std::chrono::seconds(1));
964  }
965 
966  cout << "DONE!!!" << endl;
967 
968  transport1.stop();
969  transport1.setSource (0);
970  audioSourcePlayer.setSource (0);
971  my_thread.stopThread(500);
972  deviceManager.removeAudioCallback (&audioSourcePlayer);
973  deviceManager.closeAudioDevice();
974  deviceManager.removeAllChangeListeners();
975  deviceManager.dispatchPendingMessages();
976 
977  cout << "End of Play()" << endl;
978 
979 
980 }
981 
982 // Add audio silence
983 void Frame::AddAudioSilence(int numSamples)
984 {
985  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
986 
987  // Resize audio container
988  audio->setSize(channels, numSamples, false, true, false);
989  audio->clear();
990  has_audio_data = true;
991 
992  // Calculate max audio sample added
993  max_audio_sample = numSamples;
994 
995  // Reset audio reverse flag
996  audio_reversed = false;
997 }
Header file for AudioBufferSource class.
Header file for AudioResampler class.
Header file for Frame class.
Header file for QtUtilities (compatibiity overlay)
This class is used to expose an AudioBuffer<float> as an AudioSource in JUCE.
This class represents a fraction.
Definition: Fraction.h:30
int num
Numerator for the fraction.
Definition: Fraction.h:32
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:78
int den
Denominator for the fraction.
Definition: Fraction.h:33
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:91
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:78
std::shared_ptr< juce::AudioBuffer< float > > audio
Definition: Frame.h:117
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
Definition: Frame.cpp:890
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:673
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:397
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:159
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
Definition: Frame.cpp:511
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:120
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:358
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
Definition: Frame.cpp:865
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:120
int GetWidth()
Get height of image.
Definition: Frame.cpp:492
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:498
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:788
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:87
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
Definition: Frame.cpp:876
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:251
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:912
float * GetInterleavedAudioSamples(int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
Definition: Frame.cpp:324
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:119
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:380
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:504
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:983
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:539
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:268
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:421
float * GetAudioSamples(int channel)
Get an array of sample data (and optional reverse the sample values)
Definition: Frame.cpp:314
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
Definition: Frame.cpp:301
virtual ~Frame()
Destructor.
Definition: Frame.cpp:110
void ReverseAudio()
Definition: Frame.cpp:803
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:480
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:852
Frame()
Constructor - blank frame.
Definition: Frame.cpp:59
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:696
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:843
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:368
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:812
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:258
int GetHeight()
Get height of image.
Definition: Frame.cpp:486
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:443
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:450
juce::AudioBuffer< float > * GetAudioSampleBuffer()
Definition: Frame.cpp:374
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:118
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
Definition: Frame.cpp:904
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:29
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...