Couple of changes to keying node:

- Switch from gaussian-bokeh blur to mean-value which is much faster
  and from matte's quality behaves not worse than gaussian-bokeh.
- Added Edge matte output so areas where black/white clipping happens
  could be easily controlled.

  Currently not sure if it's possible than operation returns two
  buffers, so now implemented as separate clipping operation call
  with specific options.

- Changed logic of detecting edges - check if most pixels within
  kernel are different fits some tolerance interval.

- Added two extra options which controls how edges are detecting:

  * Edge Kernel Radius defines radius of kernel within pixel's value
    is checking.
  * Edge Kernel Tolerance defines maximal difference between value
    of current pixel and it's neigbours which are trated as belng to
    the same plane )background of foreground)

Unfortunately, value for this two sliders for existing files shall
be tweaked manually. Default values are 3 for radius and 0.1 for
tolerance.

source/blender/compositor/nodes/COM_KeyingNode.cpp

@@ -41,18 +41,12 @@
 
 #include "COM_SetAlphaOperation.h"
 
-#define USE_GAUSSIAN_BLUR
-
 KeyingNode::KeyingNode(bNode *editorNode): Node(editorNode)
 {
 }
 
 OutputSocket *KeyingNode::setupPreBlur(ExecutionSystem *graph, InputSocket *inputImage, int size, OutputSocket **originalImage)
 {
-	memset(&preBlurData, 0, sizeof(preBlurData));
-	preBlurData.sizex = size;
-	preBlurData.sizey = size;
-
 	ConvertRGBToYCCOperation *convertRGBToYCCOperation = new ConvertRGBToYCCOperation();
 	convertRGBToYCCOperation->setMode(0);  /* ITU 601 */
 
@@ -72,20 +66,6 @@ OutputSocket *KeyingNode::setupPreBlur(ExecutionSystem *graph, InputSocket *inpu
 			addLink(graph, separateOperation->getOutputSocket(0), combineOperation->getInputSocket(channel));
 		}
 		else {
-#ifdef USE_GAUSSIAN_BLUR
-			SetValueOperation *setValueOperation = new SetValueOperation();
-			setValueOperation->setValue(1.0f);
-			graph->addOperation(setValueOperation);
-
-			GaussianBokehBlurOperation *blurOperation = new GaussianBokehBlurOperation();
-			blurOperation->setData(&preBlurData);
-			blurOperation->setQuality(COM_QUALITY_HIGH);
-
-			addLink(graph, separateOperation->getOutputSocket(0), blurOperation->getInputSocket(0));
-			addLink(graph, setValueOperation->getOutputSocket(0), blurOperation->getInputSocket(1));
-			addLink(graph, blurOperation->getOutputSocket(0), combineOperation->getInputSocket(channel));
-			graph->addOperation(blurOperation);
-#else
 			KeyingBlurOperation *blurOperation = new KeyingBlurOperation();
 
 			blurOperation->setSize(size);
@@ -93,7 +73,6 @@ OutputSocket *KeyingNode::setupPreBlur(ExecutionSystem *graph, InputSocket *inpu
 			addLink(graph, separateOperation->getOutputSocket(0), blurOperation->getInputSocket(0));
 			addLink(graph, blurOperation->getOutputSocket(0), combineOperation->getInputSocket(channel));
 			graph->addOperation(blurOperation);
-#endif
 		}
 	}
 
@@ -109,28 +88,6 @@ OutputSocket *KeyingNode::setupPreBlur(ExecutionSystem *graph, InputSocket *inpu
 
 OutputSocket *KeyingNode::setupPostBlur(ExecutionSystem *graph, OutputSocket *postBLurInput, int size)
 {
-#ifdef USE_GAUSSIAN_BLUR
-	memset(&postBlurData, 0, sizeof(postBlurData));
-
-	postBlurData.sizex = size;
-	postBlurData.sizey = size;
-
-	SetValueOperation *setValueOperation = new SetValueOperation();
-
-	setValueOperation->setValue(1.0f);
-	graph->addOperation(setValueOperation);
-
-	GaussianBokehBlurOperation *blurOperation = new GaussianBokehBlurOperation();
-	blurOperation->setData(&postBlurData);
-	blurOperation->setQuality(COM_QUALITY_HIGH);
-
-	addLink(graph, postBLurInput, blurOperation->getInputSocket(0));
-	addLink(graph, setValueOperation->getOutputSocket(0), blurOperation->getInputSocket(1));
-
-	graph->addOperation(blurOperation);
-
-	return blurOperation->getOutputSocket();
-#else
 	KeyingBlurOperation *blurOperation = new KeyingBlurOperation();
 
 	blurOperation->setSize(size);
@@ -140,7 +97,6 @@ OutputSocket *KeyingNode::setupPostBlur(ExecutionSystem *graph, OutputSocket *po
 	graph->addOperation(blurOperation);
 
 	return blurOperation->getOutputSocket();
-#endif
 }
 
 OutputSocket *KeyingNode::setupDilateErode(ExecutionSystem *graph, OutputSocket *dilateErodeInput, int distance)
@@ -177,12 +133,17 @@ OutputSocket *KeyingNode::setupDespill(ExecutionSystem *graph, OutputSocket *des
 	return despillOperation->getOutputSocket(0);
 }
 
-OutputSocket *KeyingNode::setupClip(ExecutionSystem *graph, OutputSocket *clipInput, float clipBlack, float clipWhite)
+OutputSocket *KeyingNode::setupClip(ExecutionSystem *graph, OutputSocket *clipInput, int kernelRadius, float kernelTolerance,
+                                    float clipBlack, float clipWhite, bool edgeMatte)
 {
 	KeyingClipOperation *clipOperation = new KeyingClipOperation();
 
+	clipOperation->setKernelRadius(kernelRadius);
+	clipOperation->setKernelTolerance(kernelTolerance);
+
 	clipOperation->setClipBlack(clipBlack);
 	clipOperation->setClipWhite(clipWhite);
+	clipOperation->setIsEdgeMatte(edgeMatte);
 
 	addLink(graph, clipInput, clipOperation->getInputSocket(0));
 
@@ -197,7 +158,8 @@ void KeyingNode::convertToOperations(ExecutionSystem *graph, CompositorContext *
 	InputSocket *inputScreen = this->getInputSocket(1);
 	OutputSocket *outputImage = this->getOutputSocket(0);
 	OutputSocket *outputMatte = this->getOutputSocket(1);
-	OutputSocket *postprocessedMatte, *postprocessedImage, *originalImage;
+	OutputSocket *outputEdges = this->getOutputSocket(2);
+	OutputSocket *postprocessedMatte, *postprocessedImage, *originalImage, *edgesMatte;
 
 	bNode *editorNode = this->getbNode();
 	NodeKeyingData *keying_data = (NodeKeyingData *) editorNode->storage;
@@ -223,8 +185,15 @@ void KeyingNode::convertToOperations(ExecutionSystem *graph, CompositorContext *
 
 	postprocessedMatte = keyingOperation->getOutputSocket();
 
-	if (keying_data->clip_black > 0.0f || keying_data->clip_white< 1.0f)
-		postprocessedMatte = setupClip(graph, postprocessedMatte, keying_data->clip_black, keying_data->clip_white);
+	if (keying_data->clip_black > 0.0f || keying_data->clip_white< 1.0f) {
+		postprocessedMatte = setupClip(graph, postprocessedMatte,
+		                               keying_data->edge_kernel_radius, keying_data->edge_kernel_tolerance,
+		                               keying_data->clip_black, keying_data->clip_white, false);
+	}
+
+	edgesMatte = setupClip(graph, postprocessedMatte,
+	                       keying_data->edge_kernel_radius, keying_data->edge_kernel_tolerance,
+	                       keying_data->clip_black, keying_data->clip_white, true);
 
 	/* apply blur on matte if needed */
 	if (keying_data->blur_post)
@@ -250,6 +219,7 @@ void KeyingNode::convertToOperations(ExecutionSystem *graph, CompositorContext *
 	/* connect result to output sockets */
 	outputImage->relinkConnections(postprocessedImage);
 	outputMatte->relinkConnections(postprocessedMatte);
+	outputEdges->relinkConnections(edgesMatte);
 
 	graph->addOperation(alphaOperation);
 }

source/blender/compositor/nodes/COM_KeyingNode.h

@@ -22,7 +22,6 @@
  */
 
 #include "COM_Node.h"
-#include "DNA_node_types.h"
 
 /**
   * @brief KeyingNode
@@ -30,14 +29,12 @@
   */
 class KeyingNode : public Node {
 protected:
-	NodeBlurData preBlurData;
-	NodeBlurData postBlurData;
-
 	OutputSocket *setupPreBlur(ExecutionSystem *graph, InputSocket *inputImage, int size, OutputSocket **originalImage);
 	OutputSocket *setupPostBlur(ExecutionSystem *graph, OutputSocket *postBLurInput, int size);
 	OutputSocket *setupDilateErode(ExecutionSystem *graph, OutputSocket *dilateErodeInput, int distance);
 	OutputSocket *setupDespill(ExecutionSystem *graph, OutputSocket *despillInput, InputSocket *inputSrceen, float factor);
-	OutputSocket *setupClip(ExecutionSystem *graph, OutputSocket *clipInput, float clipBlack, float clipWhite);
+	OutputSocket *setupClip(ExecutionSystem *graph, OutputSocket *clipInput, int kernelRadius, float kernelTolerance,
+	                        float clipBlack, float clipWhite, bool edgeMatte);
 public:
 	KeyingNode(bNode *editorNode);
 	void convertToOperations(ExecutionSystem *graph, CompositorContext *context);

source/blender/compositor/operations/COM_KeyingClipOperation.cpp

@@ -33,9 +33,14 @@ KeyingClipOperation::KeyingClipOperation(): NodeOperation()
 	this->addInputSocket(COM_DT_VALUE);
 	this->addOutputSocket(COM_DT_VALUE);
 
+	this->kernelRadius = 3;
+	this->kernelTolerance = 0.1f;
+
 	this->clipBlack = 0.0f;
 	this->clipWhite = 1.0f;
 
+	this->isEdgeMatte = false;
+
 	this->setComplex(true);
 }
 
@@ -48,7 +53,8 @@ void *KeyingClipOperation::initializeTileData(rcti *rect, MemoryBuffer **memoryB
 
 void KeyingClipOperation::executePixel(float *color, int x, int y, MemoryBuffer *inputBuffers[], void *data)
 {
-	const int delta = 3;
+	const int delta = this->kernelRadius;
+	const float tolerance = this->kernelTolerance;
 
 	MemoryBuffer *inputBuffer = (MemoryBuffer*)data;
 	float *buffer = inputBuffer->getBuffer();
@@ -56,10 +62,11 @@ void KeyingClipOperation::executePixel(float *color, int x, int y, MemoryBuffer
 	int bufferWidth = inputBuffer->getWidth();
 	int bufferHeight = inputBuffer->getHeight();
 
-	int count_black = 0, count_white = 0;
-	int i, j;
+	int i, j, count = 0, totalCount = 0;
 
-	int srcIndex = (y * bufferWidth + x) * 4;
+	float value = buffer[(y * bufferWidth + x) * 4];
+
+	bool ok = false;
 
 	for (i = -delta + 1; i < delta; i++) {
 		for (j = -delta + 1; j < delta; j++) {
@@ -70,24 +77,36 @@ void KeyingClipOperation::executePixel(float *color, int x, int y, MemoryBuffer
 
 			if (cx >= 0 && cx < bufferWidth && cy >= 0 && cy < bufferHeight) {
 				int bufferIndex = (cy * bufferWidth + cx) * 4;
+				float currentValue = buffer[bufferIndex];
 
-				if (buffer[bufferIndex] < 0.4f)
-					count_black++;
-				else if (buffer[bufferIndex] > 0.6f)
-					count_white++;
+				if (fabsf(currentValue - value) < tolerance) {
+					count++;
+				}
+
+				totalCount++;
 			}
 		}
 	}
 
-	color[0] = buffer[srcIndex];
+	ok = count >= (float) totalCount * 0.9f;
 
-	if (count_black >= 22 || count_white >= 22) {
-		if (color[0] < this->clipBlack)
+	if (this->isEdgeMatte) {
+		if (ok)
 			color[0] = 0.0f;
-		else if (color[0] >= this->clipWhite)
-			color[0] = 1.0f;
 		else
-			color[0] = (color[0] - this->clipBlack) / (this->clipWhite - this->clipBlack);
+			color[0] = 1.0f;
+	}
+	else {
+		color[0] = value;
+
+		if (ok) {
+			if (color[0] < this->clipBlack)
+				color[0] = 0.0f;
+			else if (color[0] >= this->clipWhite)
+				color[0] = 1.0f;
+			else
+				color[0] = (color[0] - this->clipBlack) / (this->clipWhite - this->clipBlack);
+		}
 	}
 }
 

source/blender/compositor/operations/COM_KeyingClipOperation.h

@@ -34,12 +34,21 @@ protected:
 	float clipBlack;
 	float clipWhite;
 
+	int kernelRadius;
+	float kernelTolerance;
+
+	bool isEdgeMatte;
 public:
 	KeyingClipOperation();
 
 	void setClipBlack(float value) {this->clipBlack = value;}
 	void setClipWhite(float value) {this->clipWhite = value;}
 
+	void setKernelRadius(int value) {this->kernelRadius = value;}
+	void setKernelTolerance(float value) {this->kernelTolerance = value;}
+
+	void setIsEdgeMatte(bool value) {this->isEdgeMatte = value;}
+
 	void *initializeTileData(rcti *rect, MemoryBuffer **memoryBuffers);
 
 	void executePixel(float *color, int x, int y, MemoryBuffer *inputBuffers[], void *data);

source/blender/editors/space_node/drawnode.c

@@ -2448,6 +2448,8 @@ static void node_composit_buts_keying(uiLayout *layout, bContext *UNUSED(C), Poi
 	uiItemR(layout, ptr, "blur_pre", 0, NULL, ICON_NONE);
 	uiItemR(layout, ptr, "screen_balance", 0, NULL, ICON_NONE);
 	uiItemR(layout, ptr, "despill_factor", 0, NULL, ICON_NONE);
+	uiItemR(layout, ptr, "edge_kernel_radius", 0, NULL, ICON_NONE);
+	uiItemR(layout, ptr, "edge_kernel_tolerance", 0, NULL, ICON_NONE);
 	uiItemR(layout, ptr, "clip_black", 0, NULL, ICON_NONE);
 	uiItemR(layout, ptr, "clip_white", 0, NULL, ICON_NONE);
 	uiItemR(layout, ptr, "dilate_distance", 0, NULL, ICON_NONE);

source/blender/makesdna/DNA_node_types.h

@@ -635,6 +635,8 @@ typedef struct NodeKeyingScreenData {
 typedef struct NodeKeyingData {
 	float screen_balance;
 	float despill_factor;
+	int edge_kernel_radius;
+	float edge_kernel_tolerance;
 	float clip_black, clip_white;
 	int dilate_distance;
 	int blur_pre, blur_post;

source/blender/makesrna/intern/rna_nodetree.c

@@ -3098,43 +3098,55 @@ static void def_cmp_keying(StructRNA *srna)
 	prop = RNA_def_property(srna, "screen_balance", PROP_FLOAT, PROP_FACTOR);
 	RNA_def_property_float_sdna(prop, NULL, "screen_balance");
 	RNA_def_property_range(prop, 0.0f, 1.0f);
-	RNA_def_property_ui_text(prop, "Screen Balance", "");
+	RNA_def_property_ui_text(prop, "Screen Balance", "Balance between two non-primary channels primary channel is comparing against");
 	RNA_def_property_update(prop, NC_NODE|NA_EDITED, "rna_Node_update");
 
 	prop = RNA_def_property(srna, "despill_factor", PROP_FLOAT, PROP_FACTOR);
 	RNA_def_property_float_sdna(prop, NULL, "despill_factor");
 	RNA_def_property_range(prop, 0.0f, 1.0f);
-	RNA_def_property_ui_text(prop, "Despill", "");
+	RNA_def_property_ui_text(prop, "Despill", "Factor of despilling screen color from image");
 	RNA_def_property_update(prop, NC_NODE|NA_EDITED, "rna_Node_update");
 
 	prop = RNA_def_property(srna, "clip_black", PROP_FLOAT, PROP_FACTOR);
 	RNA_def_property_float_sdna(prop, NULL, "clip_black");
 	RNA_def_property_range(prop, 0.0f, 1.0f);
-	RNA_def_property_ui_text(prop, "Clip Black", "");
+	RNA_def_property_ui_text(prop, "Clip Black", "Value of on-scaled matte pixel which considers as fully background pixel");
 	RNA_def_property_update(prop, NC_NODE|NA_EDITED, "rna_Node_update");
 
 	prop = RNA_def_property(srna, "clip_white", PROP_FLOAT, PROP_FACTOR);
 	RNA_def_property_float_sdna(prop, NULL, "clip_white");
 	RNA_def_property_range(prop, 0.0f, 1.0f);
-	RNA_def_property_ui_text(prop, "Clip White", "");
+	RNA_def_property_ui_text(prop, "Clip White", "Value of on-scaled matte pixel which considers as fully foreground pixel");
 	RNA_def_property_update(prop, NC_NODE|NA_EDITED, "rna_Node_update");
 
 	prop = RNA_def_property(srna, "blur_pre", PROP_INT, PROP_NONE);
 	RNA_def_property_int_sdna(prop, NULL, "blur_pre");
 	RNA_def_property_range(prop, 0, 2048);
-	RNA_def_property_ui_text(prop, "Pre Blur", "");
+	RNA_def_property_ui_text(prop, "Pre Blur", "Chroma pre-blur size which applies before running keyer");
 	RNA_def_property_update(prop, NC_NODE|NA_EDITED, "rna_Node_update");
 
 	prop = RNA_def_property(srna, "blur_post", PROP_INT, PROP_NONE);
 	RNA_def_property_int_sdna(prop, NULL, "blur_post");
 	RNA_def_property_range(prop, 0, 2048);
-	RNA_def_property_ui_text(prop, "Post Blur", "");
+	RNA_def_property_ui_text(prop, "Post Blur", "Matte blur size which applies after clipping and dilate/eroding");
 	RNA_def_property_update(prop, NC_NODE|NA_EDITED, "rna_Node_update");
 
 	prop = RNA_def_property(srna, "dilate_distance", PROP_INT, PROP_NONE);
 	RNA_def_property_int_sdna(prop, NULL, "dilate_distance");
 	RNA_def_property_range(prop, -100, 100);
-	RNA_def_property_ui_text(prop, "Dilate/Erode", "");
+	RNA_def_property_ui_text(prop, "Dilate/Erode", "Matte dilate/erode side");
+	RNA_def_property_update(prop, NC_NODE|NA_EDITED, "rna_Node_update");
+
+	prop = RNA_def_property(srna, "edge_kernel_radius", PROP_INT, PROP_NONE);
+	RNA_def_property_int_sdna(prop, NULL, "edge_kernel_radius");
+	RNA_def_property_range(prop, -100, 100);
+	RNA_def_property_ui_text(prop, "Edge Kernel Radius", "Radius of kernel used to detect whether pixel belongs to edge");
+	RNA_def_property_update(prop, NC_NODE|NA_EDITED, "rna_Node_update");
+
+	prop = RNA_def_property(srna, "edge_kernel_tolerance", PROP_FLOAT, PROP_FACTOR);
+	RNA_def_property_float_sdna(prop, NULL, "edge_kernel_tolerance");
+	RNA_def_property_range(prop, 0.0f, 1.0f);
+	RNA_def_property_ui_text(prop, "Edge Kernel Tolerance", "Tolerance to pixels inside kernel which are treating as belonging to the same plane");
 	RNA_def_property_update(prop, NC_NODE|NA_EDITED, "rna_Node_update");
 }
 

source/blender/nodes/composite/nodes/node_composite_keying.c

@@ -54,6 +54,7 @@ static bNodeSocketTemplate cmp_node_keying_in[] = {
 static bNodeSocketTemplate cmp_node_keying_out[] = {
 	{	SOCK_RGBA,  0, "Image"},
 	{	SOCK_FLOAT, 0, "Matte"},
+	{	SOCK_FLOAT, 0, "Edges"},
 	{	-1, 0, ""	}
 };
 
@@ -198,6 +199,9 @@ static void node_composit_init_keying(bNodeTree *UNUSED(ntree), bNode* node, bNo
 
 	data->screen_balance = 0.5f;
 	data->despill_factor = 1.0f;
+	data->edge_kernel_radius = 3;
+	data->edge_kernel_tolerance = 0.1f;
+	data->clip_white = 1.0f;
 	data->clip_black = 0.0f;
 	data->clip_white = 1.0f;