e955c94ed3
Listing the "Blender Foundation" as copyright holder implied the Blender Foundation holds copyright to files which may include work from many developers. While keeping copyright on headers makes sense for isolated libraries, Blender's own code may be refactored or moved between files in a way that makes the per file copyright holders less meaningful. Copyright references to the "Blender Foundation" have been replaced with "Blender Authors", with the exception of `./extern/` since these this contains libraries which are more isolated, any changed to license headers there can be handled on a case-by-case basis. Some directories in `./intern/` have also been excluded: - `./intern/cycles/` it's own `AUTHORS` file is planned. - `./intern/opensubdiv/`. An "AUTHORS" file has been added, using the chromium projects authors file as a template. Design task: #110784 Ref !110783.
381 lines
13 KiB
C++
381 lines
13 KiB
C++
/* SPDX-FileCopyrightText: 2011 Blender Authors
|
|
*
|
|
* SPDX-License-Identifier: GPL-2.0-or-later */
|
|
|
|
/** \file
|
|
* \ingroup bke
|
|
*
|
|
* This file contains implementation of blender-side region tracker
|
|
* which is used for 2D feature tracking.
|
|
*/
|
|
|
|
#include "MEM_guardedalloc.h"
|
|
|
|
#include "DNA_defaults.h"
|
|
#include "DNA_movieclip_types.h"
|
|
|
|
#include "BLI_threads.h"
|
|
#include "BLI_utildefines.h"
|
|
|
|
#include "BKE_movieclip.h"
|
|
#include "BKE_tracking.h"
|
|
|
|
#include "IMB_imbuf.h"
|
|
#include "IMB_imbuf_types.h"
|
|
|
|
#include "libmv-capi.h"
|
|
#include "tracking_private.h"
|
|
|
|
/* **** utility functions for tracking **** */
|
|
|
|
/** Convert from float and byte RGBA to gray-scale. Supports different coefficients for RGB. */
|
|
static void float_rgba_to_gray(const float *rgba,
|
|
float *gray,
|
|
int num_pixels,
|
|
float weight_red,
|
|
float weight_green,
|
|
float weight_blue)
|
|
{
|
|
for (int i = 0; i < num_pixels; i++) {
|
|
const float *pixel = rgba + 4 * i;
|
|
|
|
gray[i] = weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2];
|
|
}
|
|
}
|
|
|
|
static void uint8_rgba_to_float_gray(const uchar *rgba,
|
|
float *gray,
|
|
int num_pixels,
|
|
float weight_red,
|
|
float weight_green,
|
|
float weight_blue)
|
|
{
|
|
for (int i = 0; i < num_pixels; i++) {
|
|
const uchar *pixel = rgba + i * 4;
|
|
|
|
gray[i] = (weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2]) / 255.0f;
|
|
}
|
|
}
|
|
|
|
/** Get gray-scale float search buffer for given marker and frame. */
|
|
static float *track_get_search_floatbuf(ImBuf *ibuf,
|
|
MovieTrackingTrack *track,
|
|
MovieTrackingMarker *marker,
|
|
int *r_width,
|
|
int *r_height)
|
|
{
|
|
ImBuf *searchibuf;
|
|
float *gray_pixels;
|
|
int width, height;
|
|
|
|
searchibuf = BKE_tracking_get_search_imbuf(ibuf, track, marker, false, true);
|
|
|
|
if (!searchibuf) {
|
|
*r_width = 0;
|
|
*r_height = 0;
|
|
return nullptr;
|
|
}
|
|
|
|
width = searchibuf->x;
|
|
height = searchibuf->y;
|
|
|
|
gray_pixels = MEM_cnew_array<float>(width * height, "tracking floatBuf");
|
|
|
|
if (searchibuf->float_buffer.data) {
|
|
float_rgba_to_gray(
|
|
searchibuf->float_buffer.data, gray_pixels, width * height, 0.2126f, 0.7152f, 0.0722f);
|
|
}
|
|
else {
|
|
uint8_rgba_to_float_gray(
|
|
searchibuf->byte_buffer.data, gray_pixels, width * height, 0.2126f, 0.7152f, 0.0722f);
|
|
}
|
|
|
|
IMB_freeImBuf(searchibuf);
|
|
|
|
*r_width = width;
|
|
*r_height = height;
|
|
|
|
return gray_pixels;
|
|
}
|
|
|
|
/* Get image buffer for a given frame
|
|
*
|
|
* Frame is in clip space.
|
|
*/
|
|
static ImBuf *tracking_context_get_frame_ibuf(MovieClip *clip,
|
|
MovieClipUser *user,
|
|
int clip_flag,
|
|
int framenr)
|
|
{
|
|
ImBuf *ibuf;
|
|
MovieClipUser new_user = *user;
|
|
|
|
new_user.framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, framenr);
|
|
|
|
ibuf = BKE_movieclip_get_ibuf_flag(clip, &new_user, clip_flag, MOVIECLIP_CACHE_SKIP);
|
|
|
|
return ibuf;
|
|
}
|
|
|
|
/* Get image buffer for previous marker's keyframe. */
|
|
static ImBuf *tracking_context_get_keyframed_ibuf(MovieClip *clip,
|
|
MovieClipUser *user,
|
|
int clip_flag,
|
|
MovieTrackingTrack *track,
|
|
int curfra,
|
|
bool backwards,
|
|
MovieTrackingMarker **r_marker_keyed)
|
|
{
|
|
MovieTrackingMarker *marker_keyed;
|
|
int keyed_framenr;
|
|
|
|
marker_keyed = tracking_get_keyframed_marker(track, curfra, backwards);
|
|
if (marker_keyed == nullptr) {
|
|
return nullptr;
|
|
}
|
|
|
|
keyed_framenr = marker_keyed->framenr;
|
|
|
|
*r_marker_keyed = marker_keyed;
|
|
|
|
return tracking_context_get_frame_ibuf(clip, user, clip_flag, keyed_framenr);
|
|
}
|
|
|
|
/* Get image buffer which is used as reference for track. */
|
|
static ImBuf *tracking_context_get_reference_ibuf(MovieClip *clip,
|
|
MovieClipUser *user,
|
|
int clip_flag,
|
|
MovieTrackingTrack *track,
|
|
int curfra,
|
|
bool backwards,
|
|
MovieTrackingMarker **reference_marker)
|
|
{
|
|
ImBuf *ibuf = nullptr;
|
|
|
|
if (track->pattern_match == TRACK_MATCH_KEYFRAME) {
|
|
ibuf = tracking_context_get_keyframed_ibuf(
|
|
clip, user, clip_flag, track, curfra, backwards, reference_marker);
|
|
}
|
|
else {
|
|
ibuf = tracking_context_get_frame_ibuf(clip, user, clip_flag, curfra);
|
|
|
|
/* use current marker as keyframed position */
|
|
*reference_marker = BKE_tracking_marker_get(track, curfra);
|
|
}
|
|
|
|
return ibuf;
|
|
}
|
|
|
|
void tracking_configure_tracker(const MovieTrackingTrack *track,
|
|
float *mask,
|
|
const bool is_backwards,
|
|
libmv_TrackRegionOptions *options)
|
|
{
|
|
options->direction = is_backwards ? LIBMV_TRACK_REGION_BACKWARD : LIBMV_TRACK_REGION_FORWARD;
|
|
|
|
/* TODO(sergey): Use explicit conversion, so that options are decoupled between the Libmv library
|
|
* and enumerator values in DNA. */
|
|
options->motion_model = track->motion_model;
|
|
|
|
options->use_brute = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_BRUTE) != 0);
|
|
|
|
options->use_normalization = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_NORMALIZATION) !=
|
|
0);
|
|
|
|
options->num_iterations = 50;
|
|
options->minimum_correlation = track->minimum_correlation;
|
|
options->sigma = 0.9;
|
|
|
|
if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0) {
|
|
options->image1_mask = mask;
|
|
}
|
|
else {
|
|
options->image1_mask = nullptr;
|
|
}
|
|
}
|
|
|
|
/* Perform tracking from a reference_marker to destination_ibuf.
|
|
* Uses marker as an initial position guess.
|
|
*
|
|
* Returns truth if tracker returned success, puts result
|
|
* to dst_pixel_x and dst_pixel_y.
|
|
*/
|
|
static bool configure_and_run_tracker(ImBuf *destination_ibuf,
|
|
MovieTrackingTrack *track,
|
|
MovieTrackingMarker *reference_marker,
|
|
MovieTrackingMarker *marker,
|
|
float *reference_search_area,
|
|
int reference_search_area_width,
|
|
int reference_search_area_height,
|
|
float *mask,
|
|
const bool is_backward,
|
|
double dst_pixel_x[5],
|
|
double dst_pixel_y[5])
|
|
{
|
|
/* To convert to the x/y split array format for libmv. */
|
|
double src_pixel_x[5], src_pixel_y[5];
|
|
|
|
/* Settings for the tracker */
|
|
libmv_TrackRegionOptions options = {};
|
|
libmv_TrackRegionResult result;
|
|
|
|
float *patch_new;
|
|
|
|
int new_search_area_width, new_search_area_height;
|
|
int frame_width, frame_height;
|
|
|
|
bool tracked;
|
|
|
|
frame_width = destination_ibuf->x;
|
|
frame_height = destination_ibuf->y;
|
|
|
|
/* for now track to the same search area dimension as marker has got for current frame
|
|
* will make all tracked markers in currently tracked segment have the same search area
|
|
* size, but it's quite close to what is actually needed
|
|
*/
|
|
patch_new = track_get_search_floatbuf(
|
|
destination_ibuf, track, marker, &new_search_area_width, &new_search_area_height);
|
|
|
|
/* configure the tracker */
|
|
tracking_configure_tracker(track, mask, is_backward, &options);
|
|
|
|
/* Convert the marker corners and center into pixel coordinates in the
|
|
* search/destination images. */
|
|
tracking_get_marker_coords_for_tracking(
|
|
frame_width, frame_height, reference_marker, src_pixel_x, src_pixel_y);
|
|
tracking_get_marker_coords_for_tracking(
|
|
frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y);
|
|
|
|
if (patch_new == nullptr || reference_search_area == nullptr) {
|
|
return false;
|
|
}
|
|
|
|
/* run the tracker! */
|
|
tracked = libmv_trackRegion(&options,
|
|
reference_search_area,
|
|
reference_search_area_width,
|
|
reference_search_area_height,
|
|
patch_new,
|
|
new_search_area_width,
|
|
new_search_area_height,
|
|
src_pixel_x,
|
|
src_pixel_y,
|
|
&result,
|
|
dst_pixel_x,
|
|
dst_pixel_y);
|
|
|
|
MEM_freeN(patch_new);
|
|
|
|
return tracked;
|
|
}
|
|
|
|
static bool refine_marker_reference_frame_get(MovieTrackingTrack *track,
|
|
MovieTrackingMarker *marker,
|
|
bool backwards,
|
|
int *reference_framenr)
|
|
{
|
|
const MovieTrackingMarker *first_marker = track->markers;
|
|
const MovieTrackingMarker *last_marker = track->markers + track->markersnr - 1;
|
|
MovieTrackingMarker *reference = backwards ? marker + 1 : marker - 1;
|
|
|
|
while (reference >= first_marker && reference <= last_marker &&
|
|
(reference->flag & MARKER_DISABLED) != 0)
|
|
{
|
|
if (backwards) {
|
|
reference++;
|
|
}
|
|
else {
|
|
reference--;
|
|
}
|
|
}
|
|
|
|
if (reference < first_marker || reference > last_marker) {
|
|
return false;
|
|
}
|
|
|
|
*reference_framenr = reference->framenr;
|
|
return (reference->flag & MARKER_DISABLED) == 0;
|
|
}
|
|
|
|
void BKE_tracking_refine_marker(MovieClip *clip,
|
|
MovieTrackingTrack *track,
|
|
MovieTrackingMarker *marker,
|
|
bool backwards)
|
|
{
|
|
MovieTrackingMarker *reference_marker = nullptr;
|
|
ImBuf *reference_ibuf, *destination_ibuf;
|
|
float *search_area, *mask = nullptr;
|
|
int frame_width, frame_height;
|
|
int search_area_height, search_area_width;
|
|
int clip_flag = clip->flag & MCLIP_TIMECODE_FLAGS;
|
|
int reference_framenr;
|
|
MovieClipUser user = *DNA_struct_default_get(MovieClipUser);
|
|
double dst_pixel_x[5], dst_pixel_y[5];
|
|
bool tracked;
|
|
|
|
/* Construct a temporary clip used, used to acquire image buffers. */
|
|
user.framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
|
|
|
|
BKE_movieclip_get_size(clip, &user, &frame_width, &frame_height);
|
|
|
|
/* Get an image buffer for reference frame, also gets reference marker. */
|
|
if (!refine_marker_reference_frame_get(track, marker, backwards, &reference_framenr)) {
|
|
return;
|
|
}
|
|
|
|
reference_ibuf = tracking_context_get_reference_ibuf(
|
|
clip, &user, clip_flag, track, reference_framenr, backwards, &reference_marker);
|
|
if (reference_ibuf == nullptr) {
|
|
return;
|
|
}
|
|
|
|
/* Could not refine with self. */
|
|
if (reference_marker == marker) {
|
|
return;
|
|
}
|
|
|
|
/* Destination image buffer has got frame number corresponding to refining marker. */
|
|
destination_ibuf = BKE_movieclip_get_ibuf_flag(clip, &user, clip_flag, MOVIECLIP_CACHE_SKIP);
|
|
if (destination_ibuf == nullptr) {
|
|
IMB_freeImBuf(reference_ibuf);
|
|
return;
|
|
}
|
|
|
|
/* Get search area from reference image. */
|
|
search_area = track_get_search_floatbuf(
|
|
reference_ibuf, track, reference_marker, &search_area_width, &search_area_height);
|
|
|
|
/* If needed, compute track's mask. */
|
|
if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0) {
|
|
mask = BKE_tracking_track_get_mask(frame_width, frame_height, track, marker);
|
|
}
|
|
|
|
/* Run the tracker from reference frame to current one. */
|
|
tracked = configure_and_run_tracker(destination_ibuf,
|
|
track,
|
|
reference_marker,
|
|
marker,
|
|
search_area,
|
|
search_area_width,
|
|
search_area_height,
|
|
mask,
|
|
backwards,
|
|
dst_pixel_x,
|
|
dst_pixel_y);
|
|
|
|
/* Refine current marker's position if track was successful. */
|
|
if (tracked) {
|
|
tracking_set_marker_coords_from_tracking(
|
|
frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y);
|
|
marker->flag |= MARKER_TRACKED;
|
|
}
|
|
|
|
/* Free memory used for refining */
|
|
MEM_freeN(search_area);
|
|
if (mask) {
|
|
MEM_freeN(mask);
|
|
}
|
|
IMB_freeImBuf(reference_ibuf);
|
|
IMB_freeImBuf(destination_ibuf);
|
|
}
|