OpenShot Library | libopenshot  0.3.1
CacheDisk.cpp
Go to the documentation of this file.
1 
9 // Copyright (c) 2008-2019 OpenShot Studios, LLC
10 //
11 // SPDX-License-Identifier: LGPL-3.0-or-later
12 
13 #include "CacheDisk.h"
14 #include "Exceptions.h"
15 #include "Frame.h"
16 #include "QtUtilities.h"
17 
18 #include <Qt>
19 #include <QString>
20 #include <QTextStream>
21 
22 using namespace std;
23 using namespace openshot;
24 
25 // Default constructor, no max bytes
26 CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale) : CacheBase(0) {
27  // Set cache type name
28  cache_type = "CacheDisk";
29  range_version = 0;
30  needs_range_processing = false;
31  frame_size_bytes = 0;
32  image_format = format;
33  image_quality = quality;
34  image_scale = scale;
35  max_bytes = 0;
36 
37  // Init path directory
38  InitPath(cache_path);
39 }
40 
41 // Constructor that sets the max bytes to cache
42 CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale, int64_t max_bytes) : CacheBase(max_bytes) {
43  // Set cache type name
44  cache_type = "CacheDisk";
45  range_version = 0;
46  needs_range_processing = false;
47  frame_size_bytes = 0;
48  image_format = format;
49  image_quality = quality;
50  image_scale = scale;
51 
52  // Init path directory
53  InitPath(cache_path);
54 }
55 
56 // Initialize cache directory
57 void CacheDisk::InitPath(std::string cache_path) {
58  QString qpath;
59 
60  if (!cache_path.empty()) {
61  // Init QDir with cache directory
62  qpath = QString(cache_path.c_str());
63 
64  } else {
65  // Init QDir with user's temp directory
66  qpath = QDir::tempPath() + QString("/preview-cache/");
67  }
68 
69  // Init QDir with cache directory
70  path = QDir(qpath);
71 
72  // Check if cache directory exists
73  if (!path.exists())
74  // Create
75  path.mkpath(qpath);
76 }
77 
78 // Default destructor
80 {
81  Clear();
82 
83  // remove mutex
84  delete cacheMutex;
85 }
86 
87 // Add a Frame to the cache
88 void CacheDisk::Add(std::shared_ptr<Frame> frame)
89 {
90  // Create a scoped lock, to protect the cache from multiple threads
91  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
92  int64_t frame_number = frame->number;
93 
94  // Freshen frame if it already exists
95  if (frames.count(frame_number))
96  // Move frame to front of queue
97  MoveToFront(frame_number);
98 
99  else
100  {
101  // Add frame to queue and map
102  frames[frame_number] = frame_number;
103  frame_numbers.push_front(frame_number);
104  ordered_frame_numbers.push_back(frame_number);
105  needs_range_processing = true;
106 
107  // Save image to disk (if needed)
108  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
109  frame->Save(frame_path.toStdString(), image_scale, image_format, image_quality);
110  if (frame_size_bytes == 0) {
111  // Get compressed size of frame image (to correctly apply max size against)
112  QFile image_file(frame_path);
113  frame_size_bytes = image_file.size();
114  }
115 
116  // Save audio data (if needed)
117  if (frame->has_audio_data) {
118  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
119  QFile audio_file(audio_path);
120 
121  if (audio_file.open(QIODevice::WriteOnly)) {
122  QTextStream audio_stream(&audio_file);
123  audio_stream << frame->SampleRate() << Qt::endl;
124  audio_stream << frame->GetAudioChannelsCount() << Qt::endl;
125  audio_stream << frame->GetAudioSamplesCount() << Qt::endl;
126  audio_stream << frame->ChannelsLayout() << Qt::endl;
127 
128  // Loop through all samples
129  for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++)
130  {
131  // Get audio for this channel
132  float *samples = frame->GetAudioSamples(channel);
133  for (int sample = 0; sample < frame->GetAudioSamplesCount(); sample++)
134  audio_stream << samples[sample] << Qt::endl;
135  }
136 
137  }
138 
139  }
140 
141  // Clean up old frames
142  CleanUp();
143  }
144 }
145 
146 // Check if frame is already contained in cache
147 bool CacheDisk::Contains(int64_t frame_number) {
148  if (frames.count(frame_number) > 0) {
149  return true;
150  } else {
151  return false;
152  }
153 }
154 
155 // Get a frame from the cache (or NULL shared_ptr if no frame is found)
156 std::shared_ptr<Frame> CacheDisk::GetFrame(int64_t frame_number)
157 {
158  // Create a scoped lock, to protect the cache from multiple threads
159  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
160 
161  // Does frame exists in cache?
162  if (frames.count(frame_number)) {
163  // Does frame exist on disk
164  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
165  if (path.exists(frame_path)) {
166 
167  // Load image file
168  auto image = std::make_shared<QImage>();
169  image->load(frame_path);
170 
171  // Set pixel formatimage->
172  image = std::make_shared<QImage>(image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
173 
174  // Create frame object
175  auto frame = std::make_shared<Frame>();
176  frame->number = frame_number;
177  frame->AddImage(image);
178 
179  // Get audio data (if found)
180  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
181  QFile audio_file(audio_path);
182  if (audio_file.exists()) {
183  // Open audio file
184  QTextStream in(&audio_file);
185  if (audio_file.open(QIODevice::ReadOnly)) {
186  int sample_rate = in.readLine().toInt();
187  int channels = in.readLine().toInt();
188  int sample_count = in.readLine().toInt();
189  int channel_layout = in.readLine().toInt();
190 
191  // Set basic audio properties
192  frame->ResizeAudio(channels, sample_count, sample_rate, (ChannelLayout) channel_layout);
193 
194  // Loop through audio samples and add to frame
195  int current_channel = 0;
196  int current_sample = 0;
197  float *channel_samples = new float[sample_count];
198  while (!in.atEnd()) {
199  // Add sample to channel array
200  channel_samples[current_sample] = in.readLine().toFloat();
201  current_sample++;
202 
203  if (current_sample == sample_count) {
204  // Add audio to frame
205  frame->AddAudio(true, current_channel, 0, channel_samples, sample_count, 1.0);
206 
207  // Increment channel, and reset sample position
208  current_channel++;
209  current_sample = 0;
210  }
211 
212  }
213  }
214  }
215 
216  // return the Frame object
217  return frame;
218  }
219  }
220 
221  // no Frame found
222  return std::shared_ptr<Frame>();
223 }
224 
225 // @brief Get an array of all Frames
226 std::vector<std::shared_ptr<openshot::Frame>> CacheDisk::GetFrames()
227 {
228  // Create a scoped lock, to protect the cache from multiple threads
229  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
230 
231  std::vector<std::shared_ptr<openshot::Frame>> all_frames;
232  std::vector<int64_t>::iterator itr_ordered;
233  for(itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end(); ++itr_ordered)
234  {
235  int64_t frame_number = *itr_ordered;
236  all_frames.push_back(GetFrame(frame_number));
237  }
238 
239  return all_frames;
240 }
241 
242 // Get the smallest frame number (or NULL shared_ptr if no frame is found)
243 std::shared_ptr<Frame> CacheDisk::GetSmallestFrame()
244 {
245  // Create a scoped lock, to protect the cache from multiple threads
246  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
247 
248  // Loop through frame numbers
249  std::deque<int64_t>::iterator itr;
250  int64_t smallest_frame = -1;
251  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
252  {
253  if (*itr < smallest_frame || smallest_frame == -1)
254  smallest_frame = *itr;
255  }
256 
257  // Return frame (if any)
258  if (smallest_frame != -1) {
259  return GetFrame(smallest_frame);
260  } else {
261  return NULL;
262  }
263 }
264 
265 // Gets the maximum bytes value
267 {
268  // Create a scoped lock, to protect the cache from multiple threads
269  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
270 
271  int64_t total_bytes = 0;
272 
273  // Loop through frames, and calculate total bytes
274  std::deque<int64_t>::reverse_iterator itr;
275  for(itr = frame_numbers.rbegin(); itr != frame_numbers.rend(); ++itr)
276  total_bytes += frame_size_bytes;
277 
278  return total_bytes;
279 }
280 
281 // Remove a specific frame
282 void CacheDisk::Remove(int64_t frame_number)
283 {
284  Remove(frame_number, frame_number);
285 }
286 
287 // Remove range of frames
288 void CacheDisk::Remove(int64_t start_frame_number, int64_t end_frame_number)
289 {
290  // Create a scoped lock, to protect the cache from multiple threads
291  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
292 
293  // Loop through frame numbers
294  std::deque<int64_t>::iterator itr;
295  for(itr = frame_numbers.begin(); itr != frame_numbers.end();)
296  {
297  //deque<int64_t>::iterator current = itr++;
298  if (*itr >= start_frame_number && *itr <= end_frame_number)
299  {
300  // erase frame number
301  itr = frame_numbers.erase(itr);
302  } else
303  itr++;
304  }
305 
306  // Loop through ordered frame numbers
307  std::vector<int64_t>::iterator itr_ordered;
308  for(itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end();)
309  {
310  if (*itr_ordered >= start_frame_number && *itr_ordered <= end_frame_number)
311  {
312  // erase frame number
313  frames.erase(*itr_ordered);
314 
315  // Remove the image file (if it exists)
316  QString frame_path(path.path() + "/" + QString("%1.").arg(*itr_ordered) + QString(image_format.c_str()).toLower());
317  QFile image_file(frame_path);
318  if (image_file.exists())
319  image_file.remove();
320 
321  // Remove audio file (if it exists)
322  QString audio_path(path.path() + "/" + QString("%1").arg(*itr_ordered) + ".audio");
323  QFile audio_file(audio_path);
324  if (audio_file.exists())
325  audio_file.remove();
326 
327  itr_ordered = ordered_frame_numbers.erase(itr_ordered);
328  } else
329  itr_ordered++;
330  }
331 
332  // Needs range processing (since cache has changed)
333  needs_range_processing = true;
334 }
335 
336 // Move frame to front of queue (so it lasts longer)
337 void CacheDisk::MoveToFront(int64_t frame_number)
338 {
339  // Does frame exists in cache?
340  if (frames.count(frame_number))
341  {
342  // Create a scoped lock, to protect the cache from multiple threads
343  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
344 
345  // Loop through frame numbers
346  std::deque<int64_t>::iterator itr;
347  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
348  {
349  if (*itr == frame_number)
350  {
351  // erase frame number
352  frame_numbers.erase(itr);
353 
354  // add frame number to 'front' of queue
355  frame_numbers.push_front(frame_number);
356  break;
357  }
358  }
359  }
360 }
361 
362 // Clear the cache of all frames
364 {
365  // Create a scoped lock, to protect the cache from multiple threads
366  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
367 
368  // Clear all containers
369  frames.clear();
370  frame_numbers.clear();
371  frame_numbers.shrink_to_fit();
372  ordered_frame_numbers.clear();
373  ordered_frame_numbers.shrink_to_fit();
374  needs_range_processing = true;
375  frame_size_bytes = 0;
376 
377  // Delete cache directory, and recreate it
378  QString current_path = path.path();
379  path.removeRecursively();
380 
381  // Re-init folder
382  InitPath(current_path.toStdString());
383 }
384 
385 // Count the frames in the queue
387 {
388  // Create a scoped lock, to protect the cache from multiple threads
389  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
390 
391  // Return the number of frames in the cache
392  return frames.size();
393 }
394 
395 // Clean up cached frames that exceed the number in our max_bytes variable
396 void CacheDisk::CleanUp()
397 {
398  // Do we auto clean up?
399  if (max_bytes > 0)
400  {
401  // Create a scoped lock, to protect the cache from multiple threads
402  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
403 
404  while (GetBytes() > max_bytes && frame_numbers.size() > 20)
405  {
406  // Get the oldest frame number.
407  int64_t frame_to_remove = frame_numbers.back();
408 
409  // Remove frame_number and frame
410  Remove(frame_to_remove);
411  }
412  }
413 }
414 
415 // Generate JSON string of this object
416 std::string CacheDisk::Json() {
417 
418  // Return formatted string
419  return JsonValue().toStyledString();
420 }
421 
422 // Generate Json::Value for this object
423 Json::Value CacheDisk::JsonValue() {
424 
425  // Process range data (if anything has changed)
426  CalculateRanges();
427 
428  // Create root json object
429  Json::Value root = CacheBase::JsonValue(); // get parent properties
430  root["type"] = cache_type;
431  root["path"] = path.path().toStdString();
432 
433  Json::Value version;
434  std::stringstream range_version_str;
435  range_version_str << range_version;
436  root["version"] = range_version_str.str();
437 
438  // Parse and append range data (if any)
439  // Parse and append range data (if any)
440  try {
441  const Json::Value ranges = openshot::stringToJson(json_ranges);
442  root["ranges"] = ranges;
443  } catch (...) { }
444 
445  // return JsonValue
446  return root;
447 }
448 
449 // Load JSON string into this object
450 void CacheDisk::SetJson(const std::string value) {
451 
452  // Parse JSON string into JSON objects
453  try
454  {
455  const Json::Value root = openshot::stringToJson(value);
456  // Set all values that match
457  SetJsonValue(root);
458  }
459  catch (const std::exception& e)
460  {
461  // Error parsing JSON (or missing keys)
462  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
463  }
464 }
465 
466 // Load Json::Value into this object
467 void CacheDisk::SetJsonValue(const Json::Value root) {
468 
469  // Close timeline before we do anything (this also removes all open and closing clips)
470  Clear();
471 
472  // Set parent data
474 
475  if (!root["type"].isNull())
476  cache_type = root["type"].asString();
477  if (!root["path"].isNull())
478  // Update duration of timeline
479  InitPath(root["path"].asString());
480 }
Header file for CacheDisk class.
Header file for all Exception classes.
Header file for Frame class.
Header file for QtUtilities (compatibiity overlay)
All cache managers in libopenshot are based on this CacheBase class.
Definition: CacheBase.h:35
int64_t range_version
The version of the JSON range data (incremented with each change)
Definition: CacheBase.h:44
virtual Json::Value JsonValue()=0
Generate Json::Value for this object.
Definition: CacheBase.cpp:102
std::string cache_type
This is a friendly type name of the derived cache instance.
Definition: CacheBase.h:37
void CalculateRanges()
Calculate ranges of frames.
Definition: CacheBase.cpp:38
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: CacheBase.cpp:115
bool needs_range_processing
Something has changed, and the range data needs to be re-calculated.
Definition: CacheBase.h:40
int64_t max_bytes
This is the max number of bytes to cache (0 = no limit)
Definition: CacheBase.h:38
std::recursive_mutex * cacheMutex
Mutex for multiple threads.
Definition: CacheBase.h:47
std::string json_ranges
JSON ranges of frame numbers.
Definition: CacheBase.h:41
std::vector< int64_t > ordered_frame_numbers
Ordered list of frame numbers used by cache.
Definition: CacheBase.h:42
std::vector< std::shared_ptr< openshot::Frame > > GetFrames()
Get an array of all Frames.
Definition: CacheDisk.cpp:226
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheDisk.cpp:156
CacheDisk(std::string cache_path, std::string format, float quality, float scale)
Default constructor, no max bytes.
Definition: CacheDisk.cpp:26
void MoveToFront(int64_t frame_number)
Move frame to front of queue (so it lasts longer)
Definition: CacheDisk.cpp:337
bool Contains(int64_t frame_number)
Check if frame is already contained in cache.
Definition: CacheDisk.cpp:147
std::string Json()
Generate JSON string of this object.
Definition: CacheDisk.cpp:416
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheDisk.cpp:88
std::shared_ptr< openshot::Frame > GetSmallestFrame()
Get the smallest frame number.
Definition: CacheDisk.cpp:243
Json::Value JsonValue()
Generate Json::Value for this object.
Definition: CacheDisk.cpp:423
void Clear()
Clear the cache of all frames.
Definition: CacheDisk.cpp:363
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: CacheDisk.cpp:467
void SetJson(const std::string value)
Load JSON string into this object.
Definition: CacheDisk.cpp:450
int64_t GetBytes()
Gets the maximum bytes value.
Definition: CacheDisk.cpp:266
int64_t Count()
Count the frames in the queue.
Definition: CacheDisk.cpp:386
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheDisk.cpp:282
Exception for invalid JSON.
Definition: Exceptions.h:218
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:29
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16