Fix #35960, #36044: blender internal viewport rendering crash while editing data.

Now the viewport rendering thread will lock the main thread while it is exporting
objects to render data. This is not ideal if you have big scenes that might block
the UI, but Cycles does the same, and it's fairly quick because the same evaluated
mesh can be used as for viewport drawing. It's the only way to get things stable
until the thread safe dependency graph is here.

This adds a mechanism to the job system for jobs to lock the main thread, using a
new 'ticket mutex lock' which is a mutex lock that gives priority to the first
thread that tries to lock the mutex.

Still to solve: undo/redo crashes.

source/blender/blenlib/BLI_threads.h

@@ -131,6 +131,18 @@ void BLI_rw_mutex_free(ThreadRWMutex *mutex);
 void BLI_rw_mutex_lock(ThreadRWMutex *mutex, int mode);
 void BLI_rw_mutex_unlock(ThreadRWMutex *mutex);
 
+/* Ticket Mutex Lock
+ *
+ * This is a 'fair' mutex in that it will grant the lock to the first thread
+ * that requests it. */
+
+typedef struct TicketMutex TicketMutex;
+
+TicketMutex *BLI_ticket_mutex_alloc(void);
+void BLI_ticket_mutex_free(TicketMutex *ticket);
+void BLI_ticket_mutex_lock(TicketMutex *ticket);
+void BLI_ticket_mutex_unlock(TicketMutex *ticket);
+
 /* ThreadedWorker
  *
  * A simple tool for dispatching work to a limited number of threads

source/blender/blenlib/intern/threads.c

@@ -508,6 +508,52 @@ void BLI_rw_mutex_free(ThreadRWMutex *mutex)
 	MEM_freeN(mutex);
 }
 
+/* Ticket Mutex Lock */
+
+struct TicketMutex {
+	pthread_cond_t cond;
+	pthread_mutex_t mutex;
+	unsigned int queue_head, queue_tail;
+};
+
+TicketMutex *BLI_ticket_mutex_alloc(void)
+{
+	TicketMutex *ticket = MEM_callocN(sizeof(TicketMutex), "TicketMutex");
+
+	pthread_cond_init(&ticket->cond, NULL);
+	pthread_mutex_init(&ticket->mutex, NULL);
+
+	return ticket;
+}
+
+void BLI_ticket_mutex_free(TicketMutex *ticket)
+{
+	pthread_mutex_destroy(&ticket->mutex);
+	pthread_cond_destroy(&ticket->cond);
+	MEM_freeN(ticket);
+}
+
+void BLI_ticket_mutex_lock(TicketMutex *ticket)
+{
+	unsigned int queue_me;
+
+	pthread_mutex_lock(&ticket->mutex);
+	queue_me = ticket->queue_tail++;
+
+	while (queue_me != ticket->queue_head)
+		pthread_cond_wait(&ticket->cond, &ticket->mutex);
+
+	pthread_mutex_unlock(&ticket->mutex);
+}
+
+void BLI_ticket_mutex_unlock(TicketMutex *ticket)
+{
+	pthread_mutex_lock(&ticket->mutex);
+	ticket->queue_head++;
+	pthread_cond_broadcast(&ticket->cond);
+	pthread_mutex_unlock(&ticket->mutex);
+}
+
 /* ************************************************ */
 
 typedef struct ThreadedWorker {

source/blender/editors/render/render_internal.c

@@ -733,6 +733,7 @@ typedef struct RenderPreview {
 	/* from wmJob */
 	void *owner;
 	short *stop, *do_update;
+	wmJob *job;
 	
 	Scene *scene;
 	ScrArea *sa;
@@ -913,8 +914,15 @@ static void render_view3d_startjob(void *customdata, short *stop, short *do_upda
 		else lay = rp->v3d->lay;
 		
 		RE_SetView(re, rp->viewmat);
-		
+
+		/* copying blender data while main thread is locked, to avoid crashes */
+		WM_job_main_thread_lock_acquire(rp->job);
 		RE_Database_FromScene(re, rp->bmain, rp->scene, lay, 0);		// 0= dont use camera view
+		WM_job_main_thread_lock_release(rp->job);
+
+		/* do preprocessing like building raytree, shadows, volumes, SSS */
+		RE_Database_Preprocess(re);
+
 		// printf("dbase update\n");
 	}
 	else {
@@ -958,7 +966,8 @@ static void render_view3d_do(RenderEngine *engine, const bContext *C, int keep_d
 	wm_job = WM_jobs_get(CTX_wm_manager(C), CTX_wm_window(C), CTX_wm_region(C), "Render Preview",
 	                     WM_JOB_EXCL_RENDER, WM_JOB_TYPE_RENDER_PREVIEW);
 	rp = MEM_callocN(sizeof(RenderPreview), "render preview");
-	
+	rp->job = wm_job;
+
 	/* customdata for preview thread */
 	rp->scene = scene;
 	rp->engine = engine;

source/blender/render/extern/include/RE_pipeline.h

@@ -207,6 +207,7 @@ void RE_GetViewPlane(struct Render *re, rctf *viewplane, rcti *disprect);
 
 /* make or free the dbase */
 void RE_Database_FromScene(struct Render *re, struct Main *bmain, struct Scene *scene, unsigned int lay, int use_camera_view);
+void RE_Database_Preprocess(struct Render *re);
 void RE_Database_Free(struct Render *re);
 
 /* project dbase again, when viewplane/perspective changed */

source/blender/render/intern/source/convertblender.c

@@ -5321,14 +5321,10 @@ void RE_Database_FromScene(Render *re, Main *bmain, Scene *scene, unsigned int l
 	database_init_objects(re, lay, 0, 0, 0, 0);
 	
 	if (!re->test_break(re->tbh)) {
-		int tothalo;
-
 		set_material_lightgroups(re);
 		for (sce= re->scene; sce; sce= sce->set)
 			set_renderlayer_lightgroups(re, sce);
 		
-		slurph_opt= 1;
-		
 		/* for now some clumsy copying still */
 		re->i.totvert= re->totvert;
 		re->i.totface= re->totvlak;
@@ -5336,7 +5332,16 @@ void RE_Database_FromScene(Render *re, Main *bmain, Scene *scene, unsigned int l
 		re->i.tothalo= re->tothalo;
 		re->i.totlamp= re->totlamp;
 		re->stats_draw(re->sdh, &re->i);
-		
+	}
+
+	slurph_opt= 1;
+}
+
+void RE_Database_Preprocess(Render *re)
+{
+	if (!re->test_break(re->tbh)) {
+		int tothalo;
+
 		/* don't sort stars */
 		tothalo= re->tothalo;
 		if (!re->test_break(re->tbh)) {
@@ -5392,7 +5397,6 @@ void RE_Database_FromScene(Render *re, Main *bmain, Scene *scene, unsigned int l
 		if (!re->test_break(re->tbh))
 			if (re->r.mode & R_RAYTRACE)
 				volume_precache(re);
-		
 	}
 	
 	if (re->test_break(re->tbh))
@@ -5866,8 +5870,10 @@ void RE_Database_FromScene_Vectors(Render *re, Main *bmain, Scene *sce, unsigned
 	RE_Database_Free(re);
 	re->strandsurface= strandsurface;
 	
-	if (!re->test_break(re->tbh))
+	if (!re->test_break(re->tbh)) {
 		RE_Database_FromScene(re, bmain, sce, lay, 1);
+		RE_Database_Preprocess(re);
+	}
 	
 	if (!re->test_break(re->tbh)) {
 		int vectorlay= get_vector_renderlayers(re->scene);

source/blender/render/intern/source/pipeline.c

@@ -1142,10 +1142,13 @@ static void do_render_3d(Render *re)
 		re->draw_lock(re->dlh, 1);
 	
 	/* make render verts/faces/halos/lamps */
-	if (render_scene_needs_vector(re))
+	if (render_scene_needs_vector(re)) {
 		RE_Database_FromScene_Vectors(re, re->main, re->scene, re->lay);
-	else
+	}
+	else {
 		RE_Database_FromScene(re, re->main, re->scene, re->lay, 1);
+		RE_Database_Preprocess(re);
+	}
 	
 	/* clear UI drawing locks */
 	if (re->draw_lock)

source/blender/windowmanager/WM_api.h

@@ -404,6 +404,9 @@ void		WM_jobs_kill_type(struct wmWindowManager *wm, int job_type);
 
 int			WM_jobs_has_running(struct wmWindowManager *wm);
 
+void		WM_job_main_thread_lock_acquire(struct wmJob *job);
+void		WM_job_main_thread_lock_release(struct wmJob *job);
+
 			/* clipboard */
 char       *WM_clipboard_text_get(int selection);
 void        WM_clipboard_text_set(char *buf, int selection);

source/blender/windowmanager/intern/wm_jobs.c

@@ -128,8 +128,43 @@ struct wmJob {
 	ListBase threads;
 
 	double start_time;
+
+	/* ticket mutex for main thread locking while some job accesses
+	 * data that the main thread might modify at the same time */
+	TicketMutex *main_thread_mutex;
+	bool main_thread_mutex_ending;
 };
 
+/* Main thread locking */
+
+void WM_job_main_thread_lock_acquire(wmJob *wm_job)
+{
+	BLI_ticket_mutex_lock(wm_job->main_thread_mutex);
+
+	/* if BLI_end_threads is being called to stop the job before it's finished,
+	 * we no longer need to lock to get access to the main thread as it's
+	 * waiting and can't respond */
+	if (wm_job->main_thread_mutex_ending)
+		BLI_ticket_mutex_unlock(wm_job->main_thread_mutex);
+}
+
+void WM_job_main_thread_lock_release(wmJob *wm_job)
+{
+	if (!wm_job->main_thread_mutex_ending)
+		BLI_ticket_mutex_unlock(wm_job->main_thread_mutex);
+}
+
+static void wm_job_main_thread_yield(wmJob *wm_job, bool ending)
+{
+	if (ending)
+		wm_job->main_thread_mutex_ending = true;
+
+	/* unlock and lock the ticket mutex. because it's a fair mutex any job that
+	 * is waiting to acquire the lock will get it first, before we can lock */
+	BLI_ticket_mutex_unlock(wm_job->main_thread_mutex);
+	BLI_ticket_mutex_lock(wm_job->main_thread_mutex);
+}
+
 /* finds:
  * if type, compare for it, otherwise any matching job 
  */
@@ -162,13 +197,16 @@ wmJob *WM_jobs_get(wmWindowManager *wm, wmWindow *win, void *owner, const char *
 	
 	if (wm_job == NULL) {
 		wm_job = MEM_callocN(sizeof(wmJob), "new job");
-	
+
 		BLI_addtail(&wm->jobs, wm_job);
 		wm_job->win = win;
 		wm_job->owner = owner;
 		wm_job->flag = flag;
 		wm_job->job_type = job_type;
 		BLI_strncpy(wm_job->name, name, sizeof(wm_job->name));
+
+		wm_job->main_thread_mutex = BLI_ticket_mutex_alloc();
+		BLI_ticket_mutex_lock(wm_job->main_thread_mutex);
 	}
 	/* else: a running job, be careful */
 	
@@ -369,12 +407,21 @@ void WM_jobs_start(wmWindowManager *wm, wmJob *wm_job)
 	}
 }
 
+static void wm_job_free(wmWindowManager *wm, wmJob *wm_job)
+{
+	BLI_remlink(&wm->jobs, wm_job);
+	BLI_ticket_mutex_unlock(wm_job->main_thread_mutex);
+	BLI_ticket_mutex_free(wm_job->main_thread_mutex);
+	MEM_freeN(wm_job);
+}
+
 /* stop job, end thread, free data completely */
 static void wm_jobs_kill_job(wmWindowManager *wm, wmJob *wm_job)
 {
 	if (wm_job->running) {
 		/* signal job to end */
 		wm_job->stop = TRUE;
+		wm_job_main_thread_yield(wm_job, true);
 		BLI_end_threads(&wm_job->threads);
 
 		if (wm_job->endjob)
@@ -389,9 +436,7 @@ static void wm_jobs_kill_job(wmWindowManager *wm, wmJob *wm_job)
 		wm_job->run_free(wm_job->run_customdata);
 	
 	/* remove wm_job */
-	BLI_remlink(&wm->jobs, wm_job);
-	MEM_freeN(wm_job);
-	
+	wm_job_free(wm, wm_job);
 }
 
 /* wait until every job ended */
@@ -483,7 +528,6 @@ void wm_jobs_timer(const bContext *C, wmWindowManager *wm, wmTimer *wt)
 	float total_progress = 0.f;
 	float jobs_progress = 0;
 	
-	
 	for (wm_job = wm->jobs.first; wm_job; wm_job = wm_jobnext) {
 		wm_jobnext = wm_job->next;
 		
@@ -491,6 +535,9 @@ void wm_jobs_timer(const bContext *C, wmWindowManager *wm, wmTimer *wt)
 			
 			/* running threads */
 			if (wm_job->threads.first) {
+
+				/* let threads get temporary lock over main thread if needed */
+				wm_job_main_thread_yield(wm_job, false);
 				
 				/* always call note and update when ready */
 				if (wm_job->do_update || wm_job->ready) {
@@ -522,7 +569,9 @@ void wm_jobs_timer(const bContext *C, wmWindowManager *wm, wmTimer *wt)
 					}
 
 					wm_job->running = FALSE;
+					wm_job_main_thread_yield(wm_job, true);
 					BLI_end_threads(&wm_job->threads);
+					wm_job->main_thread_mutex_ending = false;
 					
 					if (wm_job->endnote)
 						WM_event_add_notifier(C, wm_job->endnote, NULL);
@@ -539,8 +588,7 @@ void wm_jobs_timer(const bContext *C, wmWindowManager *wm, wmTimer *wt)
 						wm_job->wt = NULL;
 						
 						/* remove wm_job */
-						BLI_remlink(&wm->jobs, wm_job);
-						MEM_freeN(wm_job);
+						wm_job_free(wm, wm_job);
 					}
 				}
 				else if (wm_job->flag & WM_JOB_PROGRESS) {