Fix T37177, sculpting can act on opposite side of mesh in orthographic camera.

Summary:
Issue here most probably is that the start point in ray-casting is too
far away from the mesh. As a result the triangle intersection code can
sometimes miss the ray intersection. To solve this, we project the ray
segment to the boundary of the root node.

Reviewers: brecht, sergey, campbellbarton

Reviewed By: brecht

Maniphest Tasks: T37177

Differential Revision: http://developer.blender.org/D115

source/blender/blenkernel/BKE_pbvh.h

@@ -97,6 +97,12 @@ int BKE_pbvh_node_raycast(PBVH *bvh, PBVHNode *node, float (*origco)[3], int use
                           const float ray_start[3], const float ray_normal[3],
                           float *dist);
 
+/* for orthographic cameras, project the far away ray segment points to the root node so
+ * we can have better precision. Warning, this function assumes that ray begins and ends outside
+ * bounding box! */
+void BKE_pbvh_raycast_project_ray_root(PBVH *bvh, bool original, float ray_start[3],
+                                       float ray_end[3], float ray_normal[3]);
+
 /* Drawing */
 
 void BKE_pbvh_node_draw(PBVHNode *node, void *data);

source/blender/blenkernel/intern/pbvh.c

@@ -1541,6 +1541,49 @@ int BKE_pbvh_node_raycast(PBVH *bvh, PBVHNode *node, float (*origco)[3], int use
 	return hit;
 }
 
+void BKE_pbvh_raycast_project_ray_root (PBVH *bvh, bool original, float ray_start[3], float ray_end[3], float ray_normal[3])
+{
+	if (bvh->nodes) {
+		float rootmin_start, rootmin_end;
+		float bb_min_root[3], bb_max_root[3], bb_center[3], bb_diff[3];
+		IsectRayAABBData ray;
+		float ray_normal_inv[3];
+		float offset = 1.0f + 1e-3f;
+		float offset_vec[3] = {1e-3f, 1e-3f, 1e-3f};
+
+		if (original)
+			BKE_pbvh_node_get_original_BB(bvh->nodes, bb_min_root, bb_max_root);
+		else
+			BKE_pbvh_node_get_BB(bvh->nodes, bb_min_root, bb_max_root);
+
+		/* slightly offset min and max in case we have a zero width node (due to a plane mesh for instance),
+		 * or faces very close to the bounding box boundary. */
+		mid_v3_v3v3(bb_center, bb_max_root, bb_min_root);
+		/* diff should be same for both min/max since it's calculated from center */
+		sub_v3_v3v3(bb_diff, bb_max_root, bb_center);
+		/* handles case of zero width bb */
+		add_v3_v3(bb_diff, offset_vec);
+		madd_v3_v3v3fl(bb_max_root, bb_center, bb_diff, offset);
+		madd_v3_v3v3fl(bb_min_root, bb_center, bb_diff, -offset);
+
+		/* first project start ray */
+		isect_ray_aabb_initialize(&ray, ray_start, ray_normal);
+		if (!isect_ray_aabb(&ray, bb_min_root, bb_max_root, &rootmin_start))
+			return;
+
+		/* then the end ray */
+		mul_v3_v3fl(ray_normal_inv, ray_normal, -1.0);
+		isect_ray_aabb_initialize(&ray, ray_end, ray_normal_inv);
+		/* unlikely to fail exiting if entering succeeded, still keep this here */
+		if (!isect_ray_aabb(&ray, bb_min_root, bb_max_root, &rootmin_end))
+			return;
+
+		madd_v3_v3v3fl(ray_start, ray_start, ray_normal, rootmin_start);
+		madd_v3_v3v3fl(ray_end, ray_end, ray_normal_inv, rootmin_end);
+	}
+}
+
+
 //#include <GL/glew.h>
 
 typedef struct {

source/blender/editors/sculpt_paint/sculpt.c

@@ -4290,12 +4290,16 @@ int sculpt_stroke_get_location(bContext *C, float out[3], const float mouse[2])
 	float ray_start[3], ray_end[3], ray_normal[3], dist;
 	float obimat[4][4];
 	SculptRaycastData srd;
+	bool original;
+	RegionView3D *rv3d;
 
 	view3d_set_viewcontext(C, &vc);
 	
+	rv3d = vc.ar->regiondata;
 	ob = vc.obact;
 	ss = ob->sculpt;
 	cache = ss->cache;
+	original = (cache) ? cache->original : 0;
 
 	sculpt_stroke_modifiers_check(C, ob);
 
@@ -4309,15 +4313,24 @@ int sculpt_stroke_get_location(bContext *C, float out[3], const float mouse[2])
 	sub_v3_v3v3(ray_normal, ray_end, ray_start);
 	dist = normalize_v3(ray_normal);
 
+	if (!rv3d->is_persp) {
+		BKE_pbvh_raycast_project_ray_root(ss->pbvh, srd.original, ray_start, ray_end, ray_normal);
+
+		/* recalculate the normal */
+		sub_v3_v3v3(ray_normal, ray_end, ray_start);
+		dist = normalize_v3(ray_normal);
+	}
+
+	srd.original = original;
 	srd.ss = vc.obact->sculpt;
+	srd.hit = 0;
 	srd.ray_start = ray_start;
 	srd.ray_normal = ray_normal;
 	srd.dist = dist;
-	srd.hit = 0;
-	srd.original = (cache) ? cache->original : 0;
+
 	BKE_pbvh_raycast(ss->pbvh, sculpt_raycast_cb, &srd,
 	                 ray_start, ray_normal, srd.original);
-	
+
 	copy_v3_v3(out, ray_normal);
 	mul_v3_fl(out, srd.dist);
 	add_v3_v3(out, ray_start);