OpenShot Library | libopenshot  0.3.1
Tracker.cpp
Go to the documentation of this file.
1 
10 // Copyright (c) 2008-2019 OpenShot Studios, LLC
11 //
12 // SPDX-License-Identifier: LGPL-3.0-or-later
13 
14 #include <string>
15 #include <memory>
16 #include <fstream>
17 #include <iostream>
18 
19 #include "effects/Tracker.h"
20 #include "Exceptions.h"
21 #include "Timeline.h"
22 #include "trackerdata.pb.h"
23 
24 #include <google/protobuf/util/time_util.h>
25 
26 #include <QImage>
27 #include <QPainter>
28 #include <QRectF>
29 
30 using namespace std;
31 using namespace openshot;
32 using google::protobuf::util::TimeUtil;
33 
35 Tracker::Tracker(std::string clipTrackerDataPath)
36 {
37  // Init effect properties
38  init_effect_details();
39  // Instantiate a TrackedObjectBBox object and point to it
40  TrackedObjectBBox trackedDataObject;
41  trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
42  // Tries to load the tracked object's data from protobuf file
43  trackedData->LoadBoxData(clipTrackerDataPath);
44  ClipBase* parentClip = this->ParentClip();
45  trackedData->ParentClip(parentClip);
46  trackedData->Id(std::to_string(0));
47  // Insert TrackedObject with index 0 to the trackedObjects map
48  trackedObjects.insert({0, trackedData});
49 }
50 
51 // Default constructor
52 Tracker::Tracker()
53 {
54  // Init effect properties
55  init_effect_details();
56  // Instantiate a TrackedObjectBBox object and point to it
57  TrackedObjectBBox trackedDataObject;
58  trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
59  ClipBase* parentClip = this->ParentClip();
60  trackedData->ParentClip(parentClip);
61  trackedData->Id(std::to_string(0));
62  // Insert TrackedObject with index 0 to the trackedObjects map
63  trackedObjects.insert({0, trackedData});
64 }
65 
66 
67 // Init effect settings
68 void Tracker::init_effect_details()
69 {
71  InitEffectInfo();
72 
74  info.class_name = "Tracker";
75  info.name = "Tracker";
76  info.description = "Track the selected bounding box through the video.";
77  info.has_audio = false;
78  info.has_video = true;
79  info.has_tracked_object = true;
80 
81  this->TimeScale = 1.0;
82 }
83 
84 // This method is required for all derived classes of EffectBase, and returns a
85 // modified openshot::Frame object
86 std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
87 {
88  // Get the frame's image
89  cv::Mat frame_image = frame->GetImageCV();
90 
91  // Initialize the Qt rectangle that will hold the positions of the bounding-box
92  QRectF boxRect;
93  // Initialize the image of the TrackedObject child clip
94  std::shared_ptr<QImage> childClipImage = nullptr;
95 
96  // Check if frame isn't NULL
97  if(!frame_image.empty() &&
98  trackedData->Contains(frame_number) &&
99  trackedData->visible.GetValue(frame_number) == 1)
100  {
101  // Get the width and height of the image
102  float fw = frame_image.size().width;
103  float fh = frame_image.size().height;
104 
105  // Get the bounding-box of given frame
106  BBox fd = trackedData->GetBox(frame_number);
107 
108  // Check if track data exists for the requested frame
109  if (trackedData->draw_box.GetValue(frame_number) == 1)
110  {
111  std::vector<int> stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number);
112  int stroke_width = trackedData->stroke_width.GetValue(frame_number);
113  float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number);
114  std::vector<int> bg_rgba = trackedData->background.GetColorRGBA(frame_number);
115  float bg_alpha = trackedData->background_alpha.GetValue(frame_number);
116 
117  // Create a rotated rectangle object that holds the bounding box
118  cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ),
119  cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ),
120  (int) (fd.angle) );
121 
122  DrawRectangleRGBA(frame_image, box, bg_rgba, bg_alpha, 1, true);
123  DrawRectangleRGBA(frame_image, box, stroke_rgba, stroke_alpha, stroke_width, false);
124  }
125 
126  // Get the image of the Tracked Object' child clip
127  if (trackedData->ChildClipId() != ""){
128  // Cast the parent timeline of this effect
129  Timeline* parentTimeline = static_cast<Timeline *>(ParentTimeline());
130  if (parentTimeline){
131  // Get the Tracked Object's child clip
132  Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId());
133  if (childClip){
134  // Get the image of the child clip for this frame
135  std::shared_ptr<Frame> f(new Frame(1, frame->GetWidth(), frame->GetHeight(), "#00000000"));
136  std::shared_ptr<Frame> childClipFrame = childClip->GetFrame(f, frame_number);
137  childClipImage = childClipFrame->GetImage();
138 
139  // Set the Qt rectangle with the bounding-box properties
140  boxRect.setRect((int)((fd.cx-fd.width/2)*fw),
141  (int)((fd.cy - fd.height/2)*fh),
142  (int)(fd.width*fw),
143  (int)(fd.height*fh) );
144  }
145  }
146  }
147 
148  }
149 
150  // Set image with drawn box to frame
151  // If the input image is NULL or doesn't have tracking data, it's returned as it came
152  frame->SetImageCV(frame_image);
153 
154  // Set the bounding-box image with the Tracked Object's child clip image
155  if (childClipImage){
156  // Get the frame image
157  QImage frameImage = *(frame->GetImage());
158 
159  // Set a Qt painter to the frame image
160  QPainter painter(&frameImage);
161 
162  // Draw the child clip image inside the bounding-box
163  painter.drawImage(boxRect, *childClipImage, QRectF(0, 0, frameImage.size().width(), frameImage.size().height()));
164 
165  // Set the frame image as the composed image
166  frame->AddImage(std::make_shared<QImage>(frameImage));
167  }
168 
169  return frame;
170 }
171 
172 void Tracker::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color, float alpha, int thickness, bool is_background){
173  // Get the bouding box vertices
174  cv::Point2f vertices2f[4];
175  box.points(vertices2f);
176 
177  // TODO: take a rectangle of frame_image by refencence and draw on top of that to improve speed
178  // select min enclosing rectangle to draw on a small portion of the image
179  // cv::Rect rect = box.boundingRect();
180  // cv::Mat image = frame_image(rect)
181 
182  if(is_background){
183  cv::Mat overlayFrame;
184  frame_image.copyTo(overlayFrame);
185 
186  // draw bounding box background
187  cv::Point vertices[4];
188  for(int i = 0; i < 4; ++i){
189  vertices[i] = vertices2f[i];}
190 
191  cv::Rect rect = box.boundingRect();
192  cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
193  // add opacity
194  cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
195  }
196  else{
197  cv::Mat overlayFrame;
198  frame_image.copyTo(overlayFrame);
199 
200  // Draw bounding box
201  for (int i = 0; i < 4; i++)
202  {
203  cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
204  thickness, cv::LINE_AA);
205  }
206 
207  // add opacity
208  cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
209  }
210 }
211 
212 // Get the indexes and IDs of all visible objects in the given frame
213 std::string Tracker::GetVisibleObjects(int64_t frame_number) const{
214 
215  // Initialize the JSON objects
216  Json::Value root;
217  root["visible_objects_index"] = Json::Value(Json::arrayValue);
218  root["visible_objects_id"] = Json::Value(Json::arrayValue);
219 
220  // Iterate through the tracked objects
221  for (const auto& trackedObject : trackedObjects){
222  // Get the tracked object JSON properties for this frame
223  Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number);
224  if (trackedObjectJSON["visible"]["value"].asBool()){
225  // Save the object's index and ID if it's visible in this frame
226  root["visible_objects_index"].append(trackedObject.first);
227  root["visible_objects_id"].append(trackedObject.second->Id());
228  }
229  }
230 
231  return root.toStyledString();
232 }
233 
234 // Generate JSON string of this object
235 std::string Tracker::Json() const {
236 
237  // Return formatted string
238  return JsonValue().toStyledString();
239 }
240 
241 // Generate Json::Value for this object
242 Json::Value Tracker::JsonValue() const {
243 
244  // Create root json object
245  Json::Value root = EffectBase::JsonValue(); // get parent properties
246 
247  // Save the effect's properties on root
248  root["type"] = info.class_name;
249  root["protobuf_data_path"] = protobuf_data_path;
250  root["BaseFPS"]["num"] = BaseFPS.num;
251  root["BaseFPS"]["den"] = BaseFPS.den;
252  root["TimeScale"] = this->TimeScale;
253 
254  // Add trackedObjects IDs to JSON
255  Json::Value objects;
256  for (auto const& trackedObject : trackedObjects){
257  Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
258  // add object json
259  objects[trackedObject.second->Id()] = trackedObjectJSON;
260  }
261  root["objects"] = objects;
262 
263  // return JsonValue
264  return root;
265 }
266 
267 // Load JSON string into this object
268 void Tracker::SetJson(const std::string value) {
269 
270  // Parse JSON string into JSON objects
271  try
272  {
273  const Json::Value root = openshot::stringToJson(value);
274  // Set all values that match
275  SetJsonValue(root);
276  }
277  catch (const std::exception& e)
278  {
279  // Error parsing JSON (or missing keys)
280  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
281  }
282  return;
283 }
284 
285 // Load Json::Value into this object
286 void Tracker::SetJsonValue(const Json::Value root) {
287 
288  // Set parent data
289  EffectBase::SetJsonValue(root);
290 
291  if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject())
292  {
293  if (!root["BaseFPS"]["num"].isNull())
294  {
295  BaseFPS.num = (int) root["BaseFPS"]["num"].asInt();
296  }
297  if (!root["BaseFPS"]["den"].isNull())
298  {
299  BaseFPS.den = (int) root["BaseFPS"]["den"].asInt();
300  }
301  }
302 
303  if (!root["TimeScale"].isNull())
304  TimeScale = (double) root["TimeScale"].asDouble();
305 
306  // Set data from Json (if key is found)
307  if (!root["protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1)
308  {
309  protobuf_data_path = root["protobuf_data_path"].asString();
310  if(!trackedData->LoadBoxData(protobuf_data_path))
311  {
312  std::clog << "Invalid protobuf data path " << protobuf_data_path << '\n';
313  protobuf_data_path = "";
314  }
315  }
316 
317  if (!root["objects"].isNull()){
318  for (auto const& trackedObject : trackedObjects){
319  std::string obj_id = std::to_string(trackedObject.first);
320  if(!root["objects"][obj_id].isNull()){
321  trackedObject.second->SetJsonValue(root["objects"][obj_id]);
322  }
323  }
324  }
325 
326  // Set the tracked object's ids
327  if (!root["objects_id"].isNull()){
328  for (auto const& trackedObject : trackedObjects){
329  Json::Value trackedObjectJSON;
330  trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString();
331  trackedObject.second->SetJsonValue(trackedObjectJSON);
332  }
333  }
334 
335  return;
336 }
337 
338 // Get all properties for a specific frame
339 std::string Tracker::PropertiesJSON(int64_t requested_frame) const {
340 
341  // Generate JSON properties list
342  Json::Value root;
343 
344  // Add trackedObject properties to JSON
345  Json::Value objects;
346  for (auto const& trackedObject : trackedObjects){
347  Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
348  // add object json
349  objects[trackedObject.second->Id()] = trackedObjectJSON;
350  }
351  root["objects"] = objects;
352 
353  // Append effect's properties
354  root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame);
355  root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
356  root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame);
357  root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
358  root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
359  root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame);
360 
361  // Return formatted string
362  return root.toStyledString();
363 }
Header file for all Exception classes.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
Definition: ClipBase.h:33
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:91
std::shared_ptr< openshot::Frame > GetFrame(int64_t clip_frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:389
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:91
Exception for invalid JSON.
Definition: Exceptions.h:218
This class represents a timeline.
Definition: Timeline.h:150
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
Definition: Timeline.cpp:408
This class contains the properties of a tracked object and functions to manipulate it.
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:29
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width
float angle
bounding box rotation angle [degrees]