got triangulator to work, and made it a bmesh operator. also added dissolve faces operator, with no implementation (need to discuss with Geoffry how best to do this). the bmesh test operator, jkey, now maps to the triangulator.

source/blender/bmesh/bmesh_operators.h

@@ -137,8 +137,22 @@ enum {
 #define BMOP_ESUBDIVIDE_EDGES	0
 #define BMOP_ESUBDIVIDE_TOTSLOT	1
 
+/*triangulate*/
+#define BMOP_TRIANGULATE		6
+
+#define BMOP_TRIANG_FACEIN		0
+#define BMOP_NEW_EDGES			1
+#define BMOP_NEW_FACES			2
+#define BMOP_TRIANG_TOTSLOT		3
+
+/*dissolve faces*/
+#define BMOP_DISSOLVE_FACES		7
+
+#define BMOP_DISFACES_FACEIN	0
+#define BMOP_DISFACES_TOTSLOT	1
+
 /*keep this updated!*/
-#define BMOP_TOTAL_OPS				6
+#define BMOP_TOTAL_OPS			8
 /*-------------------------------end operator defines-------------------------------*/
 
 extern BMOpDefine *opdefines[];

source/blender/bmesh/intern/bmesh_opdefines.c

@@ -3,6 +3,22 @@
 
 #include <stdio.h>
 
+BMOpDefine def_dissolvefacesop = {
+	{BMOP_OPSLOT_PNT_BUF},
+	dissolvefaces_exec,
+	BMOP_DISFACES_TOTSLOT,
+	0
+};
+
+BMOpDefine def_triangop = {
+	{BMOP_OPSLOT_PNT_BUF, 
+	 BMOP_OPSLOT_PNT_BUF,
+	 BMOP_OPSLOT_PNT_BUF},
+	triangulate_exec,
+	BMOP_TRIANG_TOTSLOT,
+	0
+};
+
 BMOpDefine def_subdop = {
 	{BMOP_OPSLOT_PNT_BUF},
 	esubdivide_exec,
@@ -52,6 +68,8 @@ BMOpDefine *opdefines[] = {
 	&def_edit2bmesh,
 	&def_bmesh2edit,
 	&def_subdop,
+	&def_triangop,
+	&def_dissolvefacesop,
 };
 
 int bmesh_total_ops = (sizeof(opdefines) / sizeof(void*));

source/blender/bmesh/intern/bmesh_operators.c

@@ -318,9 +318,9 @@ static void *alloc_slot_buffer(BMOperator *op, int slotcode, int len){
 
 /*
  *
- * BMO_FLAG_TO_SLOT
+ * BMO_HEADERFLAG_TO_SLOT
  *
- * Copies elements of a certain type, which have a certain flag set 
+ * Copies elements of a certain type, which have a certain header flag set 
  * into an output slot for an operator.
  *
 */
@@ -366,6 +366,14 @@ void BMO_HeaderFlag_To_Slot(BMesh *bm, BMOperator *op, int slotcode, int flag, i
 	}
 }
 
+/*
+ *
+ * BMO_FLAG_TO_SLOT
+ *
+ * Copies elements of a certain type, which have a certain flag set 
+ * into an output slot for an operator.
+ *
+*/
 void BMO_Flag_To_Slot(BMesh *bm, BMOperator *op, int slotcode, int flag, int type)
 {
 	BMIter elements;

source/blender/bmesh/intern/bmesh_operators_private.h

@@ -13,5 +13,7 @@ void delop_exec(struct BMesh *bm, struct BMOperator *op);
 void esubdivide_exec(BMesh *bmesh, BMOperator *op);
 void edit2bmesh_exec(BMesh *bmesh, BMOperator *op);
 void bmesh2edit_exec(BMesh *bmesh, BMOperator *op);
+void triangulate_exec(BMesh *bmesh, BMOperator *op);
+void dissolvefaces_exec(BMesh *bmesh, BMOperator *op);
 
 #endif

source/blender/bmesh/intern/bmesh_polygon.c

@@ -364,10 +364,10 @@ static BMLoop *find_ear(BMFace *f, float (*verts)[3])
  * -Modify this to try and find ears that will not create a non-manifold face after conversion back to editmesh
  *
 */
-void BM_Triangulate_Face(BMesh *bm, BMFace *f, float (*projectverts)[3])
+void BM_Triangulate_Face(BMesh *bm, BMFace *f, float (*projectverts)[3], int newedgeflag, int newfaceflag)
 {
 	int i, done;
-	BMLoop *l, *nextloop;
+	BMLoop *l, *newl, *nextloop;
 
 	/*copy vertex coordinates to vertspace array*/
 	i = 0;
@@ -386,9 +386,11 @@ void BM_Triangulate_Face(BMesh *bm, BMFace *f, float (*projectverts)[3])
 	while(!done){
 		done = 1;
 		l = find_ear(f, projectverts);
-		if(l){
+		if(l && l->head.prev != l->head.next && f->len > 3){
 			done = 0;
-			f = bmesh_sfme(bm, f, ((BMLoop*)(l->head.prev))->v, ((BMLoop*)(l->head.next))->v, 0);
+			f = bmesh_sfme(bm, f, ((BMLoop*)(l->head.prev))->v, ((BMLoop*)(l->head.next))->v, &newl);
+			BMO_SetFlag(bm, newl->e, newedgeflag);
+			BMO_SetFlag(bm, f, newfaceflag);
 		}
 	}
 
@@ -396,7 +398,9 @@ void BM_Triangulate_Face(BMesh *bm, BMFace *f, float (*projectverts)[3])
 		l = f->loopbase;
 		while (l->f->len > 3){
 			nextloop = ((BMLoop*)(l->head.next->next->next));
-			bmesh_sfme(bm, l->f, l->v,nextloop->v, 0);
+			bmesh_sfme(bm, l->f, l->v,nextloop->v, &newl);
+			BMO_SetFlag(bm, newl->e, newedgeflag);
+			BMO_SetFlag(bm, f, newfaceflag);
 			l = nextloop;
 		}
 	}

source/blender/bmesh/intern/bmesh_private.h

@@ -59,7 +59,8 @@ void bmesh_clear_sysflag(struct BMHeader *element, int flag);
 int bmesh_test_sysflag(struct BMHeader *element, int flag);
 
 /*Polygon Utilities ? FIXME... where do these each go?*/
-void bmesh_triangulate_face(struct BMesh *bm, struct BMFace *f, float (*projectverts)[3]);
+/*newedgeflag sets a flag layer flag, obviously not the header flag.*/
+void BM_Triangulate_Face(BMesh *bm, BMFace *f, float (*projectverts)[3], int newedgeflag, int newfaceflag);
 void bmesh_update_face_normal(struct BMesh *bm, struct BMFace *f, float (*projectverts)[3]);
 void compute_poly_plane(float (*verts)[3], int nverts);
 void poly_rotate_plane(float normal[3], float (*verts)[3], int nverts);

source/blender/bmesh/operators/dissolvefacesop.c

@@ -0,0 +1,26 @@
+#include "MEM_guardedalloc.h"
+
+#include "BKE_utildefines.h"
+
+#include "bmesh.h"
+#include "bmesh_private.h"
+#include "BLI_arithb.h"
+
+#include <stdio.h>
+
+#define FACE_MARK	1
+void dissolvefaces_exec(BMesh *bmesh, BMOperator *op)
+{
+	BMOpSlot *finput;
+	BMFace *face;
+	float projectverts[400][3];
+	void *projverts;
+	int i, count = 0;
+	
+	BMO_Flag_Buffer(bmesh, op, BMOP_DISFACES_FACEIN, FACE_MARK);
+
+	/*TODO: need to discuss with Briggs how best to implement this, seems this would be
+	  a great time to use the walker api, get it up to snuff.  perhaps have a walker
+	  that goes over inner vertices of a contiguously-flagged region?  then you
+	  could just use dissolve disk on them.*/
+}

source/blender/bmesh/operators/triangulateop.c

@@ -0,0 +1,38 @@
+#include "MEM_guardedalloc.h"
+
+#include "BKE_utildefines.h"
+
+#include "bmesh.h"
+#include "bmesh_private.h"
+#include "BLI_arithb.h"
+
+#include <stdio.h>
+
+#define EDGE_NEW	1
+#define FACE_NEW	1
+void triangulate_exec(BMesh *bmesh, BMOperator *op)
+{
+	BMOpSlot *finput;
+	BMFace *face;
+	float projectverts[400][3];
+	void *projverts;
+	int i, count = 0;
+	
+	finput = BMO_GetSlot(op, BMOP_ESUBDIVIDE_EDGES);
+
+	for (i=0; i<finput->len; i++) {
+		face = ((BMFace**)finput->data.p)[i];
+
+		/*HACK! need to discuss with Briggs why the function takes an 
+		  externally-allocated array of vert coordinates in the first place.*/
+		if (face->len > 400) projverts = MEM_callocN(sizeof(float)*3*face->len, "projverts");
+		else projverts = projectverts;
+		
+		BM_Triangulate_Face(bmesh, face, projectverts, EDGE_NEW, FACE_NEW);
+		
+		if (projverts != projectverts) MEM_freeN(projverts);
+	}
+	
+	BMO_Flag_To_Slot(bmesh, op, BMOP_NEW_EDGES, EDGE_NEW, BM_EDGE);
+	BMO_Flag_To_Slot(bmesh, op, BMOP_NEW_FACES, FACE_NEW, BM_FACE);
+}

source/blender/editors/mesh/editmesh_mods.c

@@ -3334,8 +3334,8 @@ static int bmesh_test_exec(bContext *C, wmOperator *op)
 	{
 			BMOperator op;
 
-			BMO_Init_Op(&op, BMOP_ESUBDIVIDE);
-			BMO_HeaderFlag_To_Slot(bm, &op, BMOP_ESUBDIVIDE_EDGES, BM_SELECT, BM_EDGE);
+			BMO_Init_Op(&op, BMOP_TRIANGULATE);
+			BMO_HeaderFlag_To_Slot(bm, &op, BMOP_TRIANG_FACEIN, BM_SELECT, BM_FACE);
 
 			BMO_Exec_Op(bm, &op);
 			BMO_Finish_Op(bm, &op);