After reviewing the channel keyer algorithm, I found that the chroma keyer algorithm was fundamentally the same. Took the opportunity to implement

a different chroma keying algorithm.  This also solves the problem of the poor UI I had on the chroma key node.

source/blender/blenkernel/intern/node.c

@@ -868,11 +868,11 @@ bNode *nodeAddNodeType(bNodeTree *ntree, int type, bNodeTree *ngroup)
 		else if(type==CMP_NODE_COLOR_SPILL){
 			NodeChroma *c= MEM_callocN(sizeof(NodeChroma), "node chroma");
 			node->storage=c;
-			c->t1= 0.0f;
+			c->t1= 30.0f;
-			c->t2= 0.0f;
+			c->t2= 10.0f;
 			c->t3= 0.0f;
 			c->fsize= 0.0f;
-			c->fstrength= 0.0f;
+			c->fstrength= 1.0f;
 			node->custom1= 1; /* red channel */
 		}
 		else if(type==CMP_NODE_CHROMA){
@@ -883,7 +883,6 @@ bNode *nodeAddNodeType(bNodeTree *ntree, int type, bNodeTree *ngroup)
 			c->t3= 0.0f;
 			c->fsize= 0.0f;
 			c->fstrength= 0.0f;
-			node->custom1= 1; /* green */
 		}
       else if(type==CMP_NODE_CHANNEL_MATTE){
          NodeChroma *c= MEM_callocN(sizeof(NodeChroma), "node chroma");

source/blender/blenkernel/intern/node_composite.c

@@ -4663,6 +4663,7 @@ static bNodeType cmp_node_color_spill={
 /* ******************* Chroma Key ********************************************************** */
 static bNodeSocketType cmp_node_chroma_in[]={
 	{SOCK_RGBA,1,"Image", 0.8f, 0.8f, 0.8f, 1.0f, 0.0f, 1.0f},
+	{SOCK_RGBA,1,"Key Color", 0.8f, 0.8f, 0.8f, 1.0f, 0.0f, 1.0f},
 	{-1,0,""}
 };
 
@@ -4674,69 +4675,83 @@ static bNodeSocketType cmp_node_chroma_out[]={
 
 static void do_rgba_to_ycca_normalized(bNode *node, float *out, float *in)
 {
+	/*normalize to the range -1.0 to 1.0) */
 	rgb_to_ycc(in[0],in[1],in[2], &out[0], &out[1], &out[2]);
-	out[0]=(out[0])/255;
+	out[0]=((out[0])-16)/255.0;
-	out[1]=(out[1])/256;
+	out[1]=((out[1])-128)/255.0;
-	out[2]=(out[2])/256;
+	out[2]=((out[2])-128)/255.0;
 	out[3]=in[3];
 }
 
 static void do_normalized_ycca_to_rgba(bNode *node, float *out, float *in)
 {
-	in[0]=in[0]*255;  
+	/*un-normalize the normalize from above */
-	in[1]=in[1]*256;
+	in[0]=(in[0]*255.0)+16;
-	in[2]=in[2]*256;
+	in[1]=(in[1]*255.0)+128;
+	in[2]=(in[2]*255.0)+128;
 	ycc_to_rgb(in[0],in[1],in[2], &out[0], &out[1], &out[2]);
 	out[3]=in[3];
 }
 
 static void do_chroma_key(bNode *node, float *out, float *in)
 {
-	/* Algorithm of my own design-Bob Holcomb */
-
 	NodeChroma *c;
 	float x, z, alpha;
+	float theta, beta, angle;
+	float kfg, newY, newCb, newCr;
 
 	c=node->storage;
-	switch(node->custom1)
+
-	{
+	/* Algorithm from book "Video Demistified" */
-		case 1:  /*green*/
+
-		{
+	/* find theta, the angle that the color space should be rotated based on key*/
-			x=(atanf((c->t1*in[1])-(c->t1*c->t2))+1)/2;
+	theta=atan2f(c->key[2],c->key[1]);
-			z=(atanf((c->t3*in[2])-(c->t3*c->fsize))+1)/2;
+
-			break;
+	/*rotate the cb and cr into x/z space */
-		}
+	x=in[1]*cosf(theta)+in[2]*sinf(theta);
-		case 2:  /*blue*/
+	z=in[2]*cosf(theta)-in[1]*sinf(theta);
-		{	
+
-			x=(atanf((c->t1*in[1])-(c->t1*c->t2))+1)/2;
+	/*if within the acceptance angle */
-			z=(atanf((c->t3*in[2])-(c->t3*c->fsize))+1)/2;
+	angle=c->t1*M_PI/180.0; /* convert to radians */
-			x=1-x;
+	
-			break;
+	/* if kfg is <0 then the pixel is outside of the key color */
-		}
+	kfg=x-(fabsf(z)/tanf(angle/2.0));
-		default:
+
-		{
+	if(kfg>0.0) {  /* found a pixel that is within key color */
-			x= z= 0.0f;
+
-			break;
+		newY=in[0]-(1-c->t3)*kfg;
-		}
+		newCb=in[1]-kfg*cosf(theta);
+		newCr=in[2]-kfg*sinf(theta);
+		alpha=(kfg+c->fsize)*(c->fstrength);
+
+		beta=atan2f(newCr,newCb);
+		beta=beta*180.0/M_PI; /* convert to degrees for compare*/
+
+		/* if beta is within the clippin angle */
+		if(fabsf(beta)<(c->t2/2.0)) {
+			newCb=0.0;
+			newCr=0.0;
+			alpha=0.0;
 		}
 
-	/*clamp to zero so that negative values don' affect the other channels input */
+		out[0]=newY;
-	if(x<0.0) x=0.0;
+		out[1]=newCb;
-	if(z<0.0) z=0.0;
+		out[2]=newCr;
 
-	/*if chroma values added are less than strength then it is a key value */
+		/* don't make something that was more transparent less transparent */
-	if((x+z) < c->fstrength) {
+		if (alpha<in[3]) {
-		alpha=(x+z);
-		alpha=in[0]+alpha; /*add in the luminence for detail */
-		if(alpha > c->falpha) alpha=0;  /* if below the threshold */
-		if(alpha < in[3]) { /* is it less than the previous alpha */
 			out[3]=alpha;
 		}
 		else {
 			out[3]=in[3];
 		}
 	}
-	
+	else { /*pixel is outside key color */
+		out[0]=in[0];
+		out[1]=in[1];
+		out[2]=in[2];
+		out[3]=in[3]; /* make pixel just as transparent as it was before */
+	}
 }
 
 static void node_composit_exec_chroma(void *data, bNode *node, bNodeStack **in, bNodeStack **out)
@@ -4758,6 +4773,8 @@ static void node_composit_exec_chroma(void *data, bNode *node, bNodeStack **in,
 	
 	/*convert rgbbuf to normalized chroma space*/
 	composit1_pixel_processor(node, chromabuf, inbuf, in[0]->vec, do_rgba_to_ycca_normalized, CB_RGBA);
+	/*convert key to normalized chroma color space */
+	do_rgba_to_ycca_normalized(node, c->key, in[1]->vec);
 	
 	/*per pixel chroma key*/
 	composit1_pixel_processor(node, chromabuf, chromabuf, in[0]->vec, do_chroma_key, CB_RGBA);

source/blender/src/drawnode.c

@@ -1297,41 +1297,31 @@ static int node_composit_buts_chroma_matte(uiBlock *block, bNodeTree *ntree, bNo
 {
 	if(block) {
 		short dx=(butr->xmax-butr->xmin)/2;
-
 		NodeChroma *c= node->storage;
 		uiBlockBeginAlign(block);
 
-		uiDefButS(block, ROW, B_NODE_EXEC+node->nr,"Green",
+		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Acceptance ",
-				butr->xmin,butr->ymin+80,dx,20,
+			butr->xmin, butr->ymin+60, butr->xmax-butr->xmin, 20,
-				&node->custom1,1,1, 0, 0, "Select if using a green background");
+			&c->t1, 1.0f, 80.0f, 100, 0, "Tolerance for colors to be considered a keying color");
-		uiDefButS(block, ROW, B_NODE_EXEC+node->nr,"Blue",
+		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Cutoff ",
-				butr->xmin+dx,butr->ymin+80,dx,20,
+			butr->xmin, butr->ymin+40, butr->xmax-butr->xmin, 20,
-				&node->custom1,1,2, 0, 0, "Select if using a blue background");
+			&c->t2, 0.0f, 30.0f, 100, 0, "Colors below this will be considered as exact matches for keying color");
 
-		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Cb Slope ",
+		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Lift ",
-			butr->xmin, butr->ymin+60, dx, 20,
+			butr->xmin, butr->ymin+20, dx, 20,
-			&c->t1, 0.0f, 20.0f, 100, 0, "Adjusts the weight a pixel's value has in determining if it is a key color");
+			&c->fsize, 0.0f, 1.0f, 100, 0, "Alpha Lift");
+		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Gain ",
+			butr->xmin+dx, butr->ymin+20, dx, 20,
+			&c->fstrength, 0.0f, 1.0f, 100, 0, "Alpha Gain");
 
-		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Cr slope ",
+		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Shadow Adjust ",
-			butr->xmin+dx, butr->ymin+60, dx, 20,
-			&c->t3, 0.0f, 20.0f, 100, 0, "Adjusts the weight a pixel's value has in determining if it is a key color");
-
-		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Cb Pos ",
-			butr->xmin, butr->ymin+40, dx, 20,
-			&c->t2, 0.0f, 1.0f, 100, 0, "Pixel values less than this setting are considered a key color");
-
-		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Cr pos ",
-			butr->xmin+dx, butr->ymin+40, dx, 20,
-			&c->fsize, 0.0f, 1.0f, 100, 0, "Pixel values less than this setting are considered a key color");
-
-		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Alpha Threshold ",
-			butr->xmin, butr->ymin+20, butr->xmax-butr->xmin, 20,
-			&c->fstrength, 0.0f, 0.25f, 100, 0, "Key colored pixels below this setting are considered transparent");
-		uiDefButF(block, NUMSLI, B_NODE_EXEC+node->nr, "Detail Threshold ",
 			butr->xmin, butr->ymin, butr->xmax-butr->xmin, 20,
-			&c->falpha, 0.0f, 1.0f, 100, 0, "Keyed pixels below this setting are not processed for detail alpha adjustment");
+			&c->t3, 0.0f, 1.0f, 100, 0, "Adjusts the brightness of any shadows captured");
+
+		if(c->t2 > c->t1)
+			c->t2=c->t1;
 	}
-	return 100;
+	return 80;
 }
 
 static int node_composit_buts_channel_matte(uiBlock *block, bNodeTree *ntree, bNode *node, rctf *butr)