Negative zmask rendering: now there is a setting to render everything

behind the zmask instead of in front. Might need tweaks, and doesn't
work with halo yet.

source/blender/makesdna/DNA_scene_types.h

@@ -138,10 +138,11 @@ typedef struct SceneRenderLayer {
 #define SCE_LAY_STRAND	32
 	/* flags between 32 and 0x8000 are set to 1 already, for future options */
 
-#define SCE_LAY_ALL_Z	0x8000
-#define SCE_LAY_XOR		0x10000
-#define SCE_LAY_DISABLE	0x20000
-#define SCE_LAY_ZMASK	0x40000
+#define SCE_LAY_ALL_Z		0x8000
+#define SCE_LAY_XOR			0x10000
+#define SCE_LAY_DISABLE		0x20000
+#define SCE_LAY_ZMASK		0x40000
+#define SCE_LAY_NEG_ZMASK	0x80000
 
 /* srl->passflag */
 #define SCE_PASS_COMBINED	1

source/blender/render/intern/include/render_types.h

@@ -85,6 +85,7 @@ typedef struct RenderPart
 	int *recto;						/* object table for objects */
 	int *rectp;						/* polygon index table */
 	int *rectz;						/* zbuffer */
+	int *rectmask;					/* negative zmask */
 	long *rectdaps;					/* delta acum buffer for pixel structs */
 	int *rectbacko;					/* object table for backside sss */
 	int *rectbackp;					/* polygon index table for backside sss */

source/blender/render/intern/include/rendercore.h

@@ -57,7 +57,7 @@ struct ListBase;
 typedef struct PixStr
 {
 	struct PixStr *next;
-	int obi, facenr, z;
+	int obi, facenr, z, maskz;
 	unsigned short mask;
 	short shadfac;
 } PixStr;

source/blender/render/intern/include/zbuf.h

@@ -96,6 +96,7 @@ typedef struct ZSpan {
 	int *rectz1;							/* seconday z buffer for shadowbuffer (2nd closest z) */
 	int *rectp;								/* polygon index buffer */
 	int *recto;								/* object buffer */
+	int *rectmask;							/* negative zmask buffer */
 	APixstr *apixbuf, *curpstr;				/* apixbuf for transparent */
 	APixstrand *curpstrand;					/* same for strands */
 	struct ListBase *apsmbase;

source/blender/render/intern/source/rendercore.c

@@ -684,7 +684,7 @@ static void freeps(ListBase *lb)
 	lb->first= lb->last= NULL;
 }
 
-static void addps(ListBase *lb, long *rd, int obi, int facenr, int z, unsigned short mask)
+static void addps(ListBase *lb, long *rd, int obi, int facenr, int z, int maskz, unsigned short mask)
 {
 	PixStrMain *psm;
 	PixStr *ps, *last= NULL;
@@ -717,6 +717,7 @@ static void addps(ListBase *lb, long *rd, int obi, int facenr, int z, unsigned s
 	ps->obi= obi;
 	ps->facenr= facenr;
 	ps->z= z;
+	ps->maskz= maskz;
 	ps->mask = mask;
 	ps->shadfac= 0;
 }
@@ -905,16 +906,16 @@ void make_pixelstructs(RenderPart *pa, ZSpan *zspan, int sample, void *data)
 	int *ro= zspan->recto;
 	int *rp= zspan->rectp;
 	int *rz= zspan->rectz;
+	int *rm= zspan->rectmask;
 	int x, y;
 	int mask= 1<<sample;
 
 	for(y=0; y<pa->recty; y++) {
-		for(x=0; x<pa->rectx; x++, rd++, rp++, ro++) {
+		for(x=0; x<pa->rectx; x++, rd++, rp++, ro++, rz++, rm++) {
 			if(*rp) {
-				addps(lb, rd, *ro, *rp, *(rz+x), mask);
+				addps(lb, rd, *ro, *rp, *rz, (zspan->rectmask)? *rm: 0, mask);
 			}
 		}
-		rz+= pa->rectx;
 	}
 
 	if(sdata->rl->layflag & SCE_LAY_EDGE) 
@@ -936,9 +937,10 @@ void zbufshadeDA_tile(RenderPart *pa)
 	pa->recto= MEM_mallocN(sizeof(int)*pa->rectx*pa->recty, "recto");
 	pa->rectp= MEM_mallocN(sizeof(int)*pa->rectx*pa->recty, "rectp");
 	pa->rectz= MEM_mallocN(sizeof(int)*pa->rectx*pa->recty, "rectz");
-	
 	for(rl= rr->layers.first; rl; rl= rl->next) {
-
+		if((rl->layflag & SCE_LAY_ZMASK) && (rl->layflag & SCE_LAY_NEG_ZMASK))
+			pa->rectmask= MEM_mallocN(sizeof(int)*pa->rectx*pa->recty, "rectmask");
+	
 		/* initialize pixelstructs and edge buffer */
 		addpsmain(&psmlist);
 		pa->rectdaps= MEM_callocN(sizeof(long)*pa->rectx*pa->recty+4, "zbufDArectd");
@@ -1041,6 +1043,11 @@ void zbufshadeDA_tile(RenderPart *pa)
 		
 		if(edgerect) MEM_freeN(edgerect);
 		edgerect= NULL;
+
+		if(pa->rectmask) {
+			MEM_freeN(pa->rectmask);
+			pa->rectmask= NULL;
+		}
 	}
 	
 	/* free all */
@@ -1076,7 +1083,9 @@ void zbufshade_tile(RenderPart *pa)
 	pa->rectz= MEM_mallocN(sizeof(int)*pa->rectx*pa->recty, "rectz");
 
 	for(rl= rr->layers.first; rl; rl= rl->next) {
-		
+		if((rl->layflag & SCE_LAY_ZMASK) && (rl->layflag & SCE_LAY_NEG_ZMASK))
+			pa->rectmask= MEM_mallocN(sizeof(int)*pa->rectx*pa->recty, "rectmask");
+
 		/* general shader info, passes */
 		shade_sample_initialize(&ssamp, pa, rl);
 		
@@ -1190,6 +1199,11 @@ void zbufshade_tile(RenderPart *pa)
 		
 		if(edgerect) MEM_freeN(edgerect);
 		edgerect= NULL;
+
+		if(pa->rectmask) {
+			MEM_freeN(pa->rectmask);
+			pa->rectmask= NULL;
+		}
 	}
 
 	/* display active layer */
@@ -1223,7 +1237,7 @@ static void addps_sss(void *cb_handle, int obi, int facenr, int x, int y, int z)
 	if(pa->rectall) {
 		long *rs= pa->rectall + pa->rectx*y + x;
 
-		addps(&handle->psmlist, rs, obi, facenr, z, 0);
+		addps(&handle->psmlist, rs, obi, facenr, z, 0, 0);
 		handle->totps++;
 	}
 	if(pa->rectz) {

source/blender/render/intern/source/strand.c

@@ -415,6 +415,7 @@ typedef struct StrandPart {
 	APixstrand *apixbuf;
 	int *totapixbuf;
 	int *rectz;
+	int *rectmask;
 	long *rectdaps;
 	int rectx, recty;
 	int sample;
@@ -495,7 +496,7 @@ static void do_strand_fillac(void *handle, int x, int y, float u, float v, float
 	StrandSegment *sseg= spart->segment;
 	APixstrand *apn, *apnew;
 	float t, s;
-	int offset, mask, obi, strnr, seg, zverg, bufferz;
+	int offset, mask, obi, strnr, seg, zverg, bufferz, maskz=0;
 
 	offset = y*spart->rectx + x;
 	obi= sseg->obi - spart->re->objectinstance;
@@ -512,18 +513,24 @@ static void do_strand_fillac(void *handle, int x, int y, float u, float v, float
 		long *rd= spart->rectdaps + offset;
 		
 		bufferz= 0x7FFFFFFF;
+		if(spart->rectmask) maskz= 0x7FFFFFFF;
 		
 		if(*rd) {	
 			for(ps= (PixStr *)(*rd); ps; ps= ps->next) {
 				if(mask & ps->mask) {
 					bufferz= ps->z;
+					if(spart->rectmask)
+						maskz= ps->maskz;
 					break;
 				}
 			}
 		}
 	}
-	else
+	else {
 		bufferz= spart->rectz[offset];
+		if(spart->rectmask)
+			maskz= spart->rectmask[offset];
+	}
 
 #define CHECK_ADD(n) \
 	if(apn->p[n]==strnr && apn->obi[n]==obi && apn->seg[n]==seg) \
@@ -534,29 +541,31 @@ static void do_strand_fillac(void *handle, int x, int y, float u, float v, float
 
 	/* add to pixel list */
 	if(zverg < bufferz && (spart->totapixbuf[offset] < MAX_ZROW)) {
-		t = u*spart->t[0] + v*spart->t[1] + (1.0f-u-v)*spart->t[2];
-		s = fabs(u*spart->s[0] + v*spart->s[1] + (1.0f-u-v)*spart->s[2]);
+		if(!spart->rectmask || zverg > maskz) {
+			t = u*spart->t[0] + v*spart->t[1] + (1.0f-u-v)*spart->t[2];
+			s = fabs(u*spart->s[0] + v*spart->s[1] + (1.0f-u-v)*spart->s[2]);
 
-		apn= spart->apixbuf + offset;
-		while(apn) {
-			CHECK_ADD(0);
-			CHECK_ADD(1);
-			CHECK_ADD(2);
-			CHECK_ADD(3);
-			CHECK_ASSIGN(0);
-			CHECK_ASSIGN(1);
-			CHECK_ASSIGN(2);
-			CHECK_ASSIGN(3);
+			apn= spart->apixbuf + offset;
+			while(apn) {
+				CHECK_ADD(0);
+				CHECK_ADD(1);
+				CHECK_ADD(2);
+				CHECK_ADD(3);
+				CHECK_ASSIGN(0);
+				CHECK_ASSIGN(1);
+				CHECK_ASSIGN(2);
+				CHECK_ASSIGN(3);
 
-			apnew= addpsAstrand(spart->zspan);
-			SWAP(APixstrand, *apnew, *apn);
-			apn->next= apnew;
-			CHECK_ASSIGN(0);
+				apnew= addpsAstrand(spart->zspan);
+				SWAP(APixstrand, *apnew, *apn);
+				apn->next= apnew;
+				CHECK_ASSIGN(0);
+			}
+
+			strand_shade_refcount(cache, sseg->v[1]);
+			strand_shade_refcount(cache, sseg->v[2]);
+			spart->totapixbuf[offset]++;
 		}
-
-		strand_shade_refcount(cache, sseg->v[1]);
-		strand_shade_refcount(cache, sseg->v[2]);
-		spart->totapixbuf[offset]++;
 	}
 }
 
@@ -779,6 +788,7 @@ int zbuffer_strands_abuf(Render *re, RenderPart *pa, RenderLayer *rl, APixstrand
 	spart.zspan= &zspan;
 	spart.rectdaps= pa->rectdaps;
 	spart.rectz= pa->rectz;
+	spart.rectmask= pa->rectmask;
 	spart.cache= cache;
 
 	zbuf_alloc_span(&zspan, pa->rectx, pa->recty, re->clipcrop);

source/blender/render/intern/source/zbuf.c

@@ -306,8 +306,8 @@ static void zbuffillAc4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v2,
 	float x0,y0,z0;
 	float x1,y1,z1,x2,y2,z2,xx1;
 	float *span1, *span2;
-	int *rz, x, y;
-	int sn1, sn2, rectx, *rectzofs, my0, my2, mask;
+	int *rz, *rm, x, y;
+	int sn1, sn2, rectx, *rectzofs, *rectmaskofs, my0, my2, mask;
 	
 	/* init */
 	zbuf_init_span(zspan);
@@ -352,6 +352,7 @@ static void zbuffillAc4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v2,
 	/* start-offset in rect */
 	rectx= zspan->rectx;
 	rectzofs= (int *)(zspan->arectz+rectx*(my2));
+	rectmaskofs= (int *)(zspan->rectmask+rectx*(my2));
 	apofs= (zspan->apixbuf+ rectx*(my2));
 	mask= zspan->mask;
 
@@ -378,6 +379,7 @@ static void zbuffillAc4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v2,
 		if(sn2>=sn1) {
 			zverg= (double)sn1*zxd + zy0;
 			rz= rectzofs+sn1;
+			rm= rectmaskofs+sn1;
 			ap= apofs+sn1;
 			x= sn2-sn1;
 			
@@ -385,26 +387,29 @@ static void zbuffillAc4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v2,
 			
 			while(x>=0) {
 				if( (int)zverg < *rz) {
-//					int i= zvlnr & 3;
-					
-					apn= ap;
-					while(apn) {
-						if(apn->p[0]==0) {apn->obi[0]= obi; apn->p[0]= zvlnr; apn->z[0]= zverg; apn->mask[0]= mask; break; }
-						if(apn->p[0]==zvlnr && apn->obi[0]==obi) {apn->mask[0]|= mask; break; }
-						if(apn->p[1]==0) {apn->obi[1]= obi; apn->p[1]= zvlnr; apn->z[1]= zverg; apn->mask[1]= mask; break; }
-						if(apn->p[1]==zvlnr && apn->obi[1]==obi) {apn->mask[1]|= mask; break; }
-						if(apn->p[2]==0) {apn->obi[2]= obi; apn->p[2]= zvlnr; apn->z[2]= zverg; apn->mask[2]= mask; break; }
-						if(apn->p[2]==zvlnr && apn->obi[2]==obi) {apn->mask[2]|= mask; break; }
-						if(apn->p[3]==0) {apn->obi[3]= obi; apn->p[3]= zvlnr; apn->z[3]= zverg; apn->mask[3]= mask; break; }
-						if(apn->p[3]==zvlnr && apn->obi[3]==obi) {apn->mask[3]|= mask; break; }
-//						if(apn->p[i]==0) {apn->obi[i]= obi; apn->p[i]= zvlnr; apn->z[i]= zverg; apn->mask[i]= mask; break; }
-//						if(apn->p[i]==zvlnr && apn->obi[i]==obi) {apn->mask[i]|= mask; break; }
-						if(apn->next==NULL) apn->next= addpsA(zspan);
-						apn= apn->next;
-					}				
+					if(!zspan->rectmask || (int)zverg > *rm) {
+	//					int i= zvlnr & 3;
+						
+						apn= ap;
+						while(apn) {
+							if(apn->p[0]==0) {apn->obi[0]= obi; apn->p[0]= zvlnr; apn->z[0]= zverg; apn->mask[0]= mask; break; }
+							if(apn->p[0]==zvlnr && apn->obi[0]==obi) {apn->mask[0]|= mask; break; }
+							if(apn->p[1]==0) {apn->obi[1]= obi; apn->p[1]= zvlnr; apn->z[1]= zverg; apn->mask[1]= mask; break; }
+							if(apn->p[1]==zvlnr && apn->obi[1]==obi) {apn->mask[1]|= mask; break; }
+							if(apn->p[2]==0) {apn->obi[2]= obi; apn->p[2]= zvlnr; apn->z[2]= zverg; apn->mask[2]= mask; break; }
+							if(apn->p[2]==zvlnr && apn->obi[2]==obi) {apn->mask[2]|= mask; break; }
+							if(apn->p[3]==0) {apn->obi[3]= obi; apn->p[3]= zvlnr; apn->z[3]= zverg; apn->mask[3]= mask; break; }
+							if(apn->p[3]==zvlnr && apn->obi[3]==obi) {apn->mask[3]|= mask; break; }
+	//						if(apn->p[i]==0) {apn->obi[i]= obi; apn->p[i]= zvlnr; apn->z[i]= zverg; apn->mask[i]= mask; break; }
+	//						if(apn->p[i]==zvlnr && apn->obi[i]==obi) {apn->mask[i]|= mask; break; }
+							if(apn->next==NULL) apn->next= addpsA(zspan);
+							apn= apn->next;
+						}				
+					}
 				}
 				zverg+= zxd;
 				rz++; 
+				rm++;
 				ap++; 
 				x--;
 			}
@@ -412,6 +417,7 @@ static void zbuffillAc4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v2,
 		
 		zy0-=zyd;
 		rectzofs-= rectx;
+		rectmaskofs-= rectx;
 		apofs-= rectx;
 	}
 }
@@ -421,7 +427,7 @@ static void zbuffillAc4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v2,
 static void zbuflineAc(ZSpan *zspan, int obi, int zvlnr, float *vec1, float *vec2)
 {
 	APixstr *ap, *apn;
-	int *rectz;
+	int *rectz, *rectmask;
 	int start, end, x, y, oldx, oldy, ofs;
 	int dz, vergz, mask, maxtest=0;
 	float dx, dy;
@@ -458,37 +464,40 @@ static void zbuflineAc(ZSpan *zspan, int obi, int zvlnr, float *vec1, float *vec
 		if(vergz>0x50000000 && dz>0) maxtest= 1;		// prevent overflow
 		
 		rectz= (int *)(zspan->arectz+zspan->rectx*(oldy) +start);
+		rectmask= (int *)(zspan->rectmask+zspan->rectx*(oldy) +start);
 		ap= (zspan->apixbuf+ zspan->rectx*(oldy) +start);
 
 		if(dy<0) ofs= -zspan->rectx;
 		else ofs= zspan->rectx;
 		
-		for(x= start; x<=end; x++, rectz++, ap++) {
+		for(x= start; x<=end; x++, rectz++, rectmask++, ap++) {
 			
 			y= floor(v1[1]);
 			if(y!=oldy) {
 				oldy= y;
 				rectz+= ofs;
+				rectmask+= ofs;
 				ap+= ofs;
 			}
 			
 			if(x>=0 && y>=0 && y<zspan->recty) {
 				if(vergz<*rectz) {
-				
-					apn= ap;
-					while(apn) {	/* loop unrolled */
-						if(apn->p[0]==0) {apn->obi[0]= obi; apn->p[0]= zvlnr; apn->z[0]= vergz; apn->mask[0]= mask; break; }
-						if(apn->p[0]==zvlnr && apn->obi[0]==obi) {apn->mask[0]|= mask; break; }
-						if(apn->p[1]==0) {apn->obi[1]= obi; apn->p[1]= zvlnr; apn->z[1]= vergz; apn->mask[1]= mask; break; }
-						if(apn->p[1]==zvlnr && apn->obi[1]==obi) {apn->mask[1]|= mask; break; }
-						if(apn->p[2]==0) {apn->obi[2]= obi; apn->p[2]= zvlnr; apn->z[2]= vergz; apn->mask[2]= mask; break; }
-						if(apn->p[2]==zvlnr && apn->obi[2]==obi) {apn->mask[2]|= mask; break; }
-						if(apn->p[3]==0) {apn->obi[3]= obi; apn->p[3]= zvlnr; apn->z[3]= vergz; apn->mask[3]= mask; break; }
-						if(apn->p[3]==zvlnr && apn->obi[3]==obi) {apn->mask[3]|= mask; break; }
-						if(apn->next==0) apn->next= addpsA(zspan);
-						apn= apn->next;
-					}				
+					if(!zspan->rectmask || vergz>*rectmask) {
 					
+						apn= ap;
+						while(apn) {	/* loop unrolled */
+							if(apn->p[0]==0) {apn->obi[0]= obi; apn->p[0]= zvlnr; apn->z[0]= vergz; apn->mask[0]= mask; break; }
+							if(apn->p[0]==zvlnr && apn->obi[0]==obi) {apn->mask[0]|= mask; break; }
+							if(apn->p[1]==0) {apn->obi[1]= obi; apn->p[1]= zvlnr; apn->z[1]= vergz; apn->mask[1]= mask; break; }
+							if(apn->p[1]==zvlnr && apn->obi[1]==obi) {apn->mask[1]|= mask; break; }
+							if(apn->p[2]==0) {apn->obi[2]= obi; apn->p[2]= zvlnr; apn->z[2]= vergz; apn->mask[2]= mask; break; }
+							if(apn->p[2]==zvlnr && apn->obi[2]==obi) {apn->mask[2]|= mask; break; }
+							if(apn->p[3]==0) {apn->obi[3]= obi; apn->p[3]= zvlnr; apn->z[3]= vergz; apn->mask[3]= mask; break; }
+							if(apn->p[3]==zvlnr && apn->obi[3]==obi) {apn->mask[3]|= mask; break; }
+							if(apn->next==0) apn->next= addpsA(zspan);
+							apn= apn->next;
+						}				
+					}
 				}
 			}
 			
@@ -526,37 +535,40 @@ static void zbuflineAc(ZSpan *zspan, int obi, int zvlnr, float *vec1, float *vec
 		if(vergz>0x50000000 && dz>0) maxtest= 1;		// prevent overflow
 
 		rectz= (int *)( zspan->arectz+ (start)*zspan->rectx+ oldx );
+		rectmask= (int *)( zspan->rectmask+ (start)*zspan->rectx+ oldx );
 		ap= (zspan->apixbuf+ zspan->rectx*(start) +oldx);
 				
 		if(dx<0) ofs= -1;
 		else ofs= 1;
 
-		for(y= start; y<=end; y++, rectz+=zspan->rectx, ap+=zspan->rectx) {
+		for(y= start; y<=end; y++, rectz+=zspan->rectx, rectmask+=zspan->rectx, ap+=zspan->rectx) {
 			
 			x= floor(v1[0]);
 			if(x!=oldx) {
 				oldx= x;
 				rectz+= ofs;
+				rectmask+= ofs;
 				ap+= ofs;
 			}
 			
 			if(x>=0 && y>=0 && x<zspan->rectx) {
 				if(vergz<*rectz) {
-					
-					apn= ap;
-					while(apn) {	/* loop unrolled */
-						if(apn->p[0]==0) {apn->obi[0]= obi; apn->p[0]= zvlnr; apn->z[0]= vergz; apn->mask[0]= mask; break; }
-						if(apn->p[0]==zvlnr) {apn->mask[0]|= mask; break; }
-						if(apn->p[1]==0) {apn->obi[1]= obi; apn->p[1]= zvlnr; apn->z[1]= vergz; apn->mask[1]= mask; break; }
-						if(apn->p[1]==zvlnr) {apn->mask[1]|= mask; break; }
-						if(apn->p[2]==0) {apn->obi[2]= obi; apn->p[2]= zvlnr; apn->z[2]= vergz; apn->mask[2]= mask; break; }
-						if(apn->p[2]==zvlnr) {apn->mask[2]|= mask; break; }
-						if(apn->p[3]==0) {apn->obi[3]= obi; apn->p[3]= zvlnr; apn->z[3]= vergz; apn->mask[3]= mask; break; }
-						if(apn->p[3]==zvlnr) {apn->mask[3]|= mask; break; }
-						if(apn->next==0) apn->next= addpsA(zspan);
-						apn= apn->next;
-					}	
-					
+					if(!zspan->rectmask || vergz>*rectmask) {
+						
+						apn= ap;
+						while(apn) {	/* loop unrolled */
+							if(apn->p[0]==0) {apn->obi[0]= obi; apn->p[0]= zvlnr; apn->z[0]= vergz; apn->mask[0]= mask; break; }
+							if(apn->p[0]==zvlnr) {apn->mask[0]|= mask; break; }
+							if(apn->p[1]==0) {apn->obi[1]= obi; apn->p[1]= zvlnr; apn->z[1]= vergz; apn->mask[1]= mask; break; }
+							if(apn->p[1]==zvlnr) {apn->mask[1]|= mask; break; }
+							if(apn->p[2]==0) {apn->obi[2]= obi; apn->p[2]= zvlnr; apn->z[2]= vergz; apn->mask[2]= mask; break; }
+							if(apn->p[2]==zvlnr) {apn->mask[2]|= mask; break; }
+							if(apn->p[3]==0) {apn->obi[3]= obi; apn->p[3]= zvlnr; apn->z[3]= vergz; apn->mask[3]= mask; break; }
+							if(apn->p[3]==zvlnr) {apn->mask[3]|= mask; break; }
+							if(apn->next==0) apn->next= addpsA(zspan);
+							apn= apn->next;
+						}	
+					}
 				}
 			}
 			
@@ -571,7 +583,7 @@ static void zbuflineAc(ZSpan *zspan, int obi, int zvlnr, float *vec1, float *vec
 
 static void zbufline(ZSpan *zspan, int obi, int zvlnr, float *vec1, float *vec2)
 {
-	int *rectz, *rectp, *recto;
+	int *rectz, *rectp, *recto, *rectmask;
 	int start, end, x, y, oldx, oldy, ofs;
 	int dz, vergz, maxtest= 0;
 	float dx, dy;
@@ -607,11 +619,12 @@ static void zbufline(ZSpan *zspan, int obi, int zvlnr, float *vec1, float *vec2)
 		rectz= zspan->rectz + oldy*zspan->rectx+ start;
 		rectp= zspan->rectp + oldy*zspan->rectx+ start;
 		recto= zspan->recto + oldy*zspan->rectx+ start;
+		rectmask= zspan->rectmask + oldy*zspan->rectx+ start;
 		
 		if(dy<0) ofs= -zspan->rectx;
 		else ofs= zspan->rectx;
 		
-		for(x= start; x<=end; x++, rectz++, rectp++, recto++) {
+		for(x= start; x<=end; x++, rectz++, rectp++, recto++, rectmask++) {
 			
 			y= floor(v1[1]);
 			if(y!=oldy) {
@@ -619,13 +632,16 @@ static void zbufline(ZSpan *zspan, int obi, int zvlnr, float *vec1, float *vec2)
 				rectz+= ofs;
 				rectp+= ofs;
 				recto+= ofs;
+				rectmask+= ofs;
 			}
 			
 			if(x>=0 && y>=0 && y<zspan->recty) {
 				if(vergz<*rectz) {
-					*recto= obi;
-					*rectz= vergz;
-					*rectp= zvlnr;
+					if(!zspan->rectmask || vergz>*rectmask) {
+						*recto= obi;
+						*rectz= vergz;
+						*rectp= zvlnr;
+					}
 				}
 			}
 			
@@ -662,11 +678,12 @@ static void zbufline(ZSpan *zspan, int obi, int zvlnr, float *vec1, float *vec2)
 		rectz= zspan->rectz + start*zspan->rectx+ oldx;
 		rectp= zspan->rectp + start*zspan->rectx+ oldx;
 		recto= zspan->recto + start*zspan->rectx+ oldx;
+		rectmask= zspan->rectmask + start*zspan->rectx+ oldx;
 		
 		if(dx<0) ofs= -1;
 		else ofs= 1;
 
-		for(y= start; y<=end; y++, rectz+=zspan->rectx, rectp+=zspan->rectx, recto+=zspan->rectx) {
+		for(y= start; y<=end; y++, rectz+=zspan->rectx, rectp+=zspan->rectx, recto+=zspan->rectx, rectmask+=zspan->rectx) {
 			
 			x= floor(v1[0]);
 			if(x!=oldx) {
@@ -674,13 +691,16 @@ static void zbufline(ZSpan *zspan, int obi, int zvlnr, float *vec1, float *vec2)
 				rectz+= ofs;
 				rectp+= ofs;
 				recto+= ofs;
+				rectmask+= ofs;
 			}
 			
 			if(x>=0 && y>=0 && x<zspan->rectx) {
 				if(vergz<*rectz) {
-					*rectz= vergz;
-					*rectp= zvlnr;
-					*recto= obi;
+					if(!zspan->rectmask || vergz>*rectmask) {
+						*rectz= vergz;
+						*rectp= zvlnr;
+						*recto= obi;
+					}
 				}
 			}
 			
@@ -1021,6 +1041,7 @@ static void zbuffillGLinv4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v
 	float *span1, *span2;
 	int *rectoofs, *ro;
 	int *rectpofs, *rp;
+	int *rectmaskofs, *rm;
 	int *rz, x, y;
 	int sn1, sn2, rectx, *rectzofs, my0, my2;
 	
@@ -1071,6 +1092,7 @@ static void zbuffillGLinv4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v
 	rectzofs= (zspan->rectz+rectx*my2);
 	rectpofs= (zspan->rectp+rectx*my2);
 	rectoofs= (zspan->recto+rectx*my2);
+	rectmaskofs= (zspan->rectmask+rectx*my2);
 	
 	/* correct span */
 	sn1= (my0 + my2)/2;
@@ -1097,18 +1119,22 @@ static void zbuffillGLinv4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v
 			rz= rectzofs+sn1;
 			rp= rectpofs+sn1;
 			ro= rectoofs+sn1;
+			rm= rectmaskofs+sn1;
 			x= sn2-sn1;
 			
 			while(x>=0) {
 				if( (int)zverg > *rz || *rz==0x7FFFFFFF) {
-					*ro= obi;
-					*rz= (int)zverg;
-					*rp= zvlnr;
+					if(!zspan->rectmask || (int)zverg > *rm) {
+						*ro= obi;
+						*rz= (int)zverg;
+						*rp= zvlnr;
+					}
 				}
 				zverg+= zxd;
 				rz++; 
 				rp++; 
 				ro++;
+				rm++;
 				x--;
 			}
 		}
@@ -1117,6 +1143,7 @@ static void zbuffillGLinv4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v
 		rectzofs-= rectx;
 		rectpofs-= rectx;
 		rectoofs-= rectx;
+		rectmaskofs-= rectx;
 	}
 }
 
@@ -1130,6 +1157,7 @@ static void zbuffillGL4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v2,
 	float *span1, *span2;
 	int *rectoofs, *ro;
 	int *rectpofs, *rp;
+	int *rectmaskofs, *rm;
 	int *rz, x, y;
 	int sn1, sn2, rectx, *rectzofs, my0, my2;
 	
@@ -1180,6 +1208,7 @@ static void zbuffillGL4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v2,
 	rectzofs= (zspan->rectz+rectx*my2);
 	rectpofs= (zspan->rectp+rectx*my2);
 	rectoofs= (zspan->recto+rectx*my2);
+	rectmaskofs= (zspan->rectmask+rectx*my2);
 
 	/* correct span */
 	sn1= (my0 + my2)/2;
@@ -1206,18 +1235,22 @@ static void zbuffillGL4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v2,
 			rz= rectzofs+sn1;
 			rp= rectpofs+sn1;
 			ro= rectoofs+sn1;
+			rm= rectmaskofs+sn1;
 			x= sn2-sn1;
 			
 			while(x>=0) {
-				if( (int)zverg < *rz) {
-					*rz= (int)zverg;
-					*rp= zvlnr;
-					*ro= obi;
+				if((int)zverg < *rz) {
+					if(!zspan->rectmask || (int)zverg > *rm) {
+						*rz= (int)zverg;
+						*rp= zvlnr;
+						*ro= obi;
+					}
 				}
 				zverg+= zxd;
 				rz++; 
 				rp++; 
 				ro++; 
+				rm++;
 				x--;
 			}
 		}
@@ -1226,6 +1259,7 @@ static void zbuffillGL4(ZSpan *zspan, int obi, int zvlnr, float *v1, float *v2,
 		rectzofs-= rectx;
 		rectpofs-= rectx;
 		rectoofs-= rectx;
+		rectmaskofs-= rectx;
 	}
 }
 
@@ -1926,7 +1960,7 @@ void zbufclip4(ZSpan *zspan, int obi, int zvlnr, float *f1, float *f2, float *f3
 #define EXTEND_PIXEL(a)	if(temprectp[a]) {z+= rectz[a]; tot++;}
 
 /* changes the zbuffer to be ready for z-masking: applies an extend-filter, and then clears */
-static void zmask_rect(int *rectz, int *rectp, int xs, int ys)
+static void zmask_rect(int *rectz, int *rectp, int xs, int ys, int neg)
 {
 	int len=0, x, y;
 	int *temprectp;
@@ -1976,14 +2010,18 @@ static void zmask_rect(int *rectz, int *rectp, int xs, int ys)
 			}
 		}
 	}
+
 	MEM_freeN(temprectp);
 	
-	/* clear not filled z values */
-	for(len= xs*ys -1; len>=0; len--) {
-		if(rectp[len]==0) {
-			rectz[len] = -0x7FFFFFFF;
-			rectp[len]= -1;	/* env code */
-		}	
+	if(neg); /* z values for negative are already correct */
+	else {
+		/* clear not filled z values */
+		for(len= xs*ys -1; len>=0; len--) {
+			if(rectp[len]==0) {
+				rectz[len] = -0x7FFFFFFF;
+				rectp[len]= -1;	/* env code */
+			}	
+		}
 	}
 }
 
@@ -2005,7 +2043,8 @@ void zbuffer_solid(RenderPart *pa, RenderLayer *rl, void(*fillfunc)(RenderPart*,
 	unsigned int lay= rl->lay, lay_zmask= rl->lay_zmask;
 	int i, v, zvlnr, zsample, samples, c1, c2, c3, c4=0;
 	short nofill=0, env=0, wire=0, zmaskpass=0;
-	short all_z= rl->layflag & SCE_LAY_ALL_Z;
+	short all_z= (rl->layflag & SCE_LAY_ALL_Z) && !(rl->layflag & SCE_LAY_ZMASK);
+	short neg_zmask= (rl->layflag & SCE_LAY_ZMASK) && (rl->layflag & SCE_LAY_NEG_ZMASK);
 	
 	samples= (R.osa? R.osa: 1);
 	samples= MIN2(4, samples-pa->sample);
@@ -2038,9 +2077,13 @@ void zbuffer_solid(RenderPart *pa, RenderLayer *rl, void(*fillfunc)(RenderPart*,
 		
 		/* the buffers */
 		if(zsample == samples-1) {
-			zspan->rectz= pa->rectz;
 			zspan->rectp= pa->rectp;
 			zspan->recto= pa->recto;
+
+			if(neg_zmask)
+				zspan->rectz= pa->rectmask;
+			else
+				zspan->rectz= pa->rectz;
 		}
 		else {
 			zspan->recto= MEM_mallocN(sizeof(int)*pa->rectx*pa->recty, "recto");
@@ -2051,10 +2094,6 @@ void zbuffer_solid(RenderPart *pa, RenderLayer *rl, void(*fillfunc)(RenderPart*,
 		fillrect(zspan->rectz, pa->rectx, pa->recty, 0x7FFFFFFF);
 		fillrect(zspan->rectp, pa->rectx, pa->recty, 0);
 		fillrect(zspan->recto, pa->rectx, pa->recty, 0);
-		
-		/* filling methods */
-		zspan->zbuffunc= zbuffillGL4;
-		zspan->zbuflinefunc= zbufline;
 	}
 
 	/* in case zmask we fill Z for objects in lay_zmask first, then clear Z, and then do normal zbuffering */
@@ -2062,6 +2101,19 @@ void zbuffer_solid(RenderPart *pa, RenderLayer *rl, void(*fillfunc)(RenderPart*,
 		zmaskpass= 1;
 	
 	for(; zmaskpass >=0; zmaskpass--) {
+		ma= NULL;
+
+		/* filling methods */
+		for(zsample=0; zsample<samples; zsample++) {
+			zspan= &zspans[zsample];
+
+			if(zmaskpass && neg_zmask)
+				zspan->zbuffunc= zbuffillGLinv4;
+			else
+				zspan->zbuffunc= zbuffillGL4;
+			zspan->zbuflinefunc= zbufline;
+		}
+
 		/* regular zbuffering loop, does all sample buffers */
 		for(i=0, obi=R.instancetable.first; obi; i++, obi=obi->next) {
 			obr= obi->obr;
@@ -2071,10 +2123,8 @@ void zbuffer_solid(RenderPart *pa, RenderLayer *rl, void(*fillfunc)(RenderPart*,
 				if((obr->lay & lay_zmask)==0)
 					continue;
 			}
-			else {
-				if(!all_z && !(obr->lay & (lay|lay_zmask)))
-					continue;
-			}
+			else if(!all_z && !(obr->lay & (lay|lay_zmask)))
+				continue;
 			
 			if(obi->flag & R_TRANSFORMED)
 				zbuf_make_winmat(&R, obi->mat, winmat);
@@ -2099,8 +2149,10 @@ void zbuffer_solid(RenderPart *pa, RenderLayer *rl, void(*fillfunc)(RenderPart*,
 						wire= (ma->mode & MA_WIRE);
 						
 						for(zsample=0; zsample<samples; zsample++) {
-							if(ma->mode & MA_ZINV) zspans[zsample].zbuffunc= zbuffillGLinv4;
-							else zspans[zsample].zbuffunc= zbuffillGL4;
+							if(ma->mode & MA_ZINV || (zmaskpass && neg_zmask))
+								zspans[zsample].zbuffunc= zbuffillGLinv4;
+							else
+								zspans[zsample].zbuffunc= zbuffillGL4;
 						}
 					}
 				}
@@ -2174,7 +2226,19 @@ void zbuffer_solid(RenderPart *pa, RenderLayer *rl, void(*fillfunc)(RenderPart*,
 		if(zmaskpass) {
 			for(zsample=0; zsample<samples; zsample++) {
 				zspan= &zspans[zsample];
-				zmask_rect(zspan->rectz, zspan->rectp, pa->rectx, pa->recty);
+
+				if(neg_zmask) {
+					zspan->rectmask= zspan->rectz;
+					if(zsample == samples-1)
+						zspan->rectz= pa->rectz;
+					else
+						zspan->rectz= MEM_mallocN(sizeof(int)*pa->rectx*pa->recty, "rectz");
+					fillrect(zspan->rectz, pa->rectx, pa->recty, 0x7FFFFFFF);
+
+					zmask_rect(zspan->rectmask, zspan->rectp, pa->rectx, pa->recty, 1);
+				}
+				else
+					zmask_rect(zspan->rectz, zspan->rectp, pa->rectx, pa->recty, 0);
 			}
 		}
 	}
@@ -2189,6 +2253,8 @@ void zbuffer_solid(RenderPart *pa, RenderLayer *rl, void(*fillfunc)(RenderPart*,
 			MEM_freeN(zspan->rectz);
 			MEM_freeN(zspan->rectp);
 			MEM_freeN(zspan->recto);
+			if(zspan->rectmask)
+				MEM_freeN(zspan->rectmask);
 		}
 
 		zbuf_free_span(zspan);
@@ -3193,18 +3259,21 @@ void RE_zbuf_accumulate_vecblur(NodeBlurData *nbd, int xsize, int ysize, float *
  * Copy results from the solid face z buffering to the transparent
  * buffer.
  */
-static void copyto_abufz(RenderPart *pa, int *arectz, int sample)
+static void copyto_abufz(RenderPart *pa, int *arectz, int *rectmask, int sample)
 {
 	PixStr *ps;
-	int x, y, *rza;
+	int x, y, *rza, *rma;
 	long *rd;
 	
 	if(R.osa==0) {
-		memcpy(arectz, pa->rectz, 4*pa->rectx*pa->recty);
+		memcpy(arectz, pa->rectz, sizeof(int)*pa->rectx*pa->recty);
+		if(rectmask && pa->rectmask)
+			memcpy(rectmask, pa->rectmask, sizeof(int)*pa->rectx*pa->recty);
 		return;
 	}
 	
 	rza= arectz;
+	rma= rectmask;
 	rd= pa->rectdaps;
 
 	sample= (1<<sample);
@@ -3213,17 +3282,19 @@ static void copyto_abufz(RenderPart *pa, int *arectz, int sample)
 		for(x=0; x<pa->rectx; x++) {
 			
 			*rza= 0x7FFFFFFF;
+			if(rectmask) *rma= 0x7FFFFFFF;
 			if(*rd) {	
 				/* when there's a sky pixstruct, fill in sky-Z, otherwise solid Z */
 				for(ps= (PixStr *)(*rd); ps; ps= ps->next) {
 					if(sample & ps->mask) {
 						*rza= ps->z;
+						if(rectmask) *rma= ps->maskz;
 						break;
 					}
 				}
 			}
 			
-			rd++; rza++;
+			rd++; rza++, rma++;
 		}
 	}
 }
@@ -3235,7 +3306,7 @@ static void copyto_abufz(RenderPart *pa, int *arectz, int sample)
  * Do accumulation z buffering.
  */
 
-static int zbuffer_abuf(RenderPart *pa, APixstr *APixbuf, ListBase *apsmbase, unsigned int lay)
+static int zbuffer_abuf(RenderPart *pa, APixstr *APixbuf, ListBase *apsmbase, RenderLayer *rl, unsigned int lay)
 {
 	ZbufProjectCache cache[ZBUF_PROJECT_CACHE_SIZE];
 	ZSpan zspans[16], *zspan;	/* MAX_OSA */
@@ -3266,11 +3337,14 @@ static int zbuffer_abuf(RenderPart *pa, APixstr *APixbuf, ListBase *apsmbase, un
 		zspan->apixbuf= APixbuf;
 		zspan->apsmbase= apsmbase;
 		
+		if((rl->layflag & SCE_LAY_ZMASK) && (rl->layflag & SCE_LAY_NEG_ZMASK))
+			zspan->rectmask= MEM_mallocN(sizeof(int)*pa->rectx*pa->recty, "Arectmask");
+
 		/* filling methods */
 		zspan->zbuffunc= zbuffillAc4;
 		zspan->zbuflinefunc= zbuflineAc;
 
-		copyto_abufz(pa, zspan->arectz, zsample);	/* init zbuffer */
+		copyto_abufz(pa, zspan->arectz, zspan->rectmask, zsample);	/* init zbuffer */
 		zspan->mask= 1<<zsample;
 
 		if(R.osa) {
@@ -3398,6 +3472,8 @@ static int zbuffer_abuf(RenderPart *pa, APixstr *APixbuf, ListBase *apsmbase, un
 	for(zsample=0; zsample<samples; zsample++) {
 		zspan= &zspans[zsample];
 		MEM_freeN(zspan->arectz);
+		if(zspan->rectmask)
+			MEM_freeN(zspan->rectmask);
 		zbuf_free_span(zspan);
 	}
 	
@@ -3908,7 +3984,7 @@ unsigned short *zbuffer_transp_shade(RenderPart *pa, RenderLayer *rl, float *pas
 	/* fill the Apixbuf */
 	doztra= 0;
 	if(rl->layflag & SCE_LAY_ZTRA)
-		doztra+= zbuffer_abuf(pa, APixbuf, &apsmbase, rl->lay);
+		doztra+= zbuffer_abuf(pa, APixbuf, &apsmbase, rl, rl->lay);
 	if((rl->layflag & SCE_LAY_STRAND) && APixbufstrand)
 		doztra+= zbuffer_strands_abuf(&R, pa, rl, APixbufstrand, &apsmbase, sscache);
 

source/blender/src/buttons_scene.c

@@ -2824,8 +2824,11 @@ static void render_panel_layers(void)
 	draw_3d_layer_buttons(block, BUT_TOGDUAL, &srl->lay,		130,110, 35, 30, "Scene-layers included in this render-layer (Hold CTRL for Z-mask)");
 	
 	uiBlockBeginAlign(block);
-	uiDefButBitI(block, TOG, SCE_LAY_ALL_Z, B_NOP,"AllZ",	10, 85, 40, 20, &srl->layflag, 0, 0, 0, 0, "Fill in Z values for solid faces in invisible layers, for masking");	
-	uiDefButBitI(block, TOG, SCE_LAY_ZMASK, B_NOP,"Zmask",	10, 65, 40, 20, &srl->layflag, 0, 0, 0, 0, "Only render what's in front of the solid z values");	
+	uiDefButBitI(block, TOG, SCE_LAY_ZMASK, B_REDR,"Zmask",	10, 85, 40, 20, &srl->layflag, 0, 0, 0, 0, "Only render what's in front of the solid z values");	
+	if(srl->layflag & SCE_LAY_ZMASK)
+		uiDefButBitI(block, TOG, SCE_LAY_NEG_ZMASK, B_NOP,"Neg",	10, 65, 40, 20, &srl->layflag, 0, 0, 0, 0, "For Zmask, only render what is behind solid z values instead of in front");
+	else
+		uiDefButBitI(block, TOG, SCE_LAY_ALL_Z, B_NOP,"AllZ",	10, 65, 40, 20, &srl->layflag, 0, 0, 0, 0, "Fill in Z values for solid faces in invisible layers, for masking");	
 	uiBlockBeginAlign(block);
 	uiDefButBitI(block, TOG, SCE_LAY_SOLID, B_NOP,"Solid",	50,  85, 45, 20, &srl->layflag, 0, 0, 0, 0, "Render Solid faces in this Layer");	
 	uiDefButBitI(block, TOG, SCE_LAY_HALO, B_NOP,"Halo",	95,  85, 40, 20, &srl->layflag, 0, 0, 0, 0, "Render Halos in this Layer (on top of Solid)");