Fix some inconsistencies in object visibility/selectability tests.
[blender.git] / source / blender / editors / object / object_bake.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2004 by Blender Foundation
19  * All rights reserved.
20  *
21  * The Original Code is: all of this file.
22  *
23  * Contributor(s): Morten Mikkelsen,
24  *                 Sergey Sharybin
25  *
26  * ***** END GPL LICENSE BLOCK *****
27  */
28
29 /** \file blender/editors/object/object_bake.c
30  *  \ingroup edobj
31  */
32
33 #include <string.h>
34
35 #include "MEM_guardedalloc.h"
36
37 #include "DNA_scene_types.h"
38 #include "DNA_screen_types.h"
39 #include "DNA_space_types.h"
40 #include "DNA_world_types.h"
41 #include "DNA_object_types.h"
42 #include "DNA_mesh_types.h"
43 #include "DNA_meshdata_types.h"
44
45 #include "BLI_blenlib.h"
46 #include "BLI_threads.h"
47 #include "BLI_utildefines.h"
48
49 #include "BKE_blender.h"
50 #include "BKE_screen.h"
51 #include "BKE_context.h"
52 #include "BKE_global.h"
53 #include "BKE_image.h"
54 #include "BKE_main.h"
55 #include "BKE_material.h"
56 #include "BKE_multires.h"
57 #include "BKE_report.h"
58 #include "BKE_cdderivedmesh.h"
59 #include "BKE_modifier.h"
60 #include "BKE_DerivedMesh.h"
61 #include "BKE_mesh.h"
62 #include "BKE_scene.h"
63
64 #include "DEG_depsgraph.h"
65
66 #include "RE_pipeline.h"
67 #include "RE_shader_ext.h"
68 #include "RE_multires_bake.h"
69
70 #include "PIL_time.h"
71
72 #include "IMB_imbuf_types.h"
73 #include "IMB_imbuf.h"
74
75 #include "GPU_draw.h" /* GPU_free_image */
76
77 #include "WM_api.h"
78 #include "WM_types.h"
79
80 #include "ED_object.h"
81 #include "ED_screen.h"
82 #include "ED_uvedit.h"
83
84 #include "object_intern.h"
85
86 static Image *bake_object_image_get(Object *ob, int mat_nr)
87 {
88         Image *image = NULL;
89         ED_object_get_active_image(ob, mat_nr + 1, &image, NULL, NULL, NULL);
90         return image;
91 }
92
93 static Image **bake_object_image_get_array(Object *ob)
94 {
95         Image **image_array = MEM_mallocN(sizeof(Material *) * ob->totcol, __func__);
96         for (int i = 0; i < ob->totcol; i++) {
97                 image_array[i] = bake_object_image_get(ob, i);
98         }
99         return image_array;
100 }
101
102 /* ****************** multires BAKING ********************** */
103
104 /* holder of per-object data needed for bake job
105  * needed to make job totally thread-safe */
106 typedef struct MultiresBakerJobData {
107         struct MultiresBakerJobData *next, *prev;
108         /* material aligned image array (for per-face bake image) */
109         struct {
110                 Image **array;
111                 int     len;
112         } ob_image;
113         DerivedMesh *lores_dm, *hires_dm;
114         bool simple;
115         int lvl, tot_lvl;
116         ListBase images;
117 } MultiresBakerJobData;
118
119 /* data passing to multires-baker job */
120 typedef struct {
121         Scene *scene;
122         ListBase data;
123         bool bake_clear;      /* Clear the images before baking */
124         int bake_filter;      /* Bake-filter, aka margin */
125         short mode;           /* mode of baking (displacement, normals, AO) */
126         bool use_lores_mesh;  /* Use low-resolution mesh when baking displacement maps */
127         int number_of_rays;   /* Number of rays to be cast when doing AO baking */
128         float bias;           /* Bias between object and start ray point when doing AO baking */
129         int threads;             /* Number of threads to be used for baking */
130         float user_scale;          /* User scale used to scale displacement when baking derivative map. */
131 } MultiresBakeJob;
132
133 static bool multiresbake_check(bContext *C, wmOperator *op)
134 {
135         Scene *scene = CTX_data_scene(C);
136         Object *ob;
137         Mesh *me;
138         MultiresModifierData *mmd;
139         bool ok = true;
140         int a;
141
142         CTX_DATA_BEGIN (C, Base *, base, selected_editable_bases)
143         {
144                 ob = base->object;
145
146                 if (ob->type != OB_MESH) {
147                         BKE_report(op->reports, RPT_ERROR, "Baking of multires data only works with an active mesh object");
148
149                         ok = false;
150                         break;
151                 }
152
153                 me = (Mesh *)ob->data;
154                 mmd = get_multires_modifier(scene, ob, 0);
155
156                 /* Multi-resolution should be and be last in the stack */
157                 if (ok && mmd) {
158                         ModifierData *md;
159
160                         ok = mmd->totlvl > 0;
161
162                         for (md = (ModifierData *)mmd->modifier.next; md && ok; md = md->next) {
163                                 if (modifier_isEnabled(scene, md, eModifierMode_Realtime)) {
164                                         ok = false;
165                                 }
166                         }
167                 }
168                 else {
169                         ok = false;
170                 }
171
172                 if (!ok) {
173                         BKE_report(op->reports, RPT_ERROR, "Multires data baking requires multi-resolution object");
174
175                         break;
176                 }
177
178                 if (!me->mloopuv) {
179                         BKE_report(op->reports, RPT_ERROR, "Mesh should be unwrapped before multires data baking");
180
181                         ok = false;
182                 }
183                 else {
184                         a = me->totpoly;
185                         while (ok && a--) {
186                                 Image *ima = bake_object_image_get(ob, me->mpoly[a].mat_nr);
187
188                                 if (!ima) {
189                                         BKE_report(op->reports, RPT_ERROR, "You should have active texture to use multires baker");
190
191                                         ok = false;
192                                 }
193                                 else {
194                                         ImBuf *ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
195
196                                         if (!ibuf) {
197                                                 BKE_report(op->reports, RPT_ERROR, "Baking should happen to image with image buffer");
198
199                                                 ok = false;
200                                         }
201                                         else {
202                                                 if (ibuf->rect == NULL && ibuf->rect_float == NULL)
203                                                         ok = false;
204
205                                                 if (ibuf->rect_float && !(ibuf->channels == 0 || ibuf->channels == 4))
206                                                         ok = false;
207
208                                                 if (!ok)
209                                                         BKE_report(op->reports, RPT_ERROR, "Baking to unsupported image type");
210                                         }
211
212                                         BKE_image_release_ibuf(ima, ibuf, NULL);
213                                 }
214                         }
215                 }
216
217                 if (!ok)
218                         break;
219         }
220         CTX_DATA_END;
221
222         return ok;
223 }
224
225 static DerivedMesh *multiresbake_create_loresdm(Scene *scene, Object *ob, int *lvl)
226 {
227         DerivedMesh *dm;
228         MultiresModifierData *mmd = get_multires_modifier(scene, ob, 0);
229         Mesh *me = (Mesh *)ob->data;
230         MultiresModifierData tmp_mmd = *mmd;
231         DerivedMesh *cddm = CDDM_from_mesh(me);
232
233         DM_set_only_copy(cddm, CD_MASK_BAREMESH);
234
235         if (mmd->lvl == 0) {
236                 dm = CDDM_copy(cddm);
237         }
238         else {
239                 tmp_mmd.lvl = mmd->lvl;
240                 tmp_mmd.sculptlvl = mmd->lvl;
241                 dm = multires_make_derived_from_derived(cddm, &tmp_mmd, scene, ob, 0);
242         }
243
244         cddm->release(cddm);
245
246         *lvl = mmd->lvl;
247
248         return dm;
249 }
250
251 static DerivedMesh *multiresbake_create_hiresdm(Scene *scene, Object *ob, int *lvl, bool *simple)
252 {
253         Mesh *me = (Mesh *)ob->data;
254         MultiresModifierData *mmd = get_multires_modifier(scene, ob, 0);
255         MultiresModifierData tmp_mmd = *mmd;
256         DerivedMesh *cddm = CDDM_from_mesh(me);
257         DerivedMesh *dm;
258
259         DM_set_only_copy(cddm, CD_MASK_BAREMESH);
260
261         /* TODO: DM_set_only_copy wouldn't set mask for loop and poly data,
262          *       but we really need BAREMESH only to save lots of memory
263          */
264         CustomData_set_only_copy(&cddm->loopData, CD_MASK_BAREMESH);
265         CustomData_set_only_copy(&cddm->polyData, CD_MASK_BAREMESH);
266
267         *lvl = mmd->totlvl;
268         *simple = mmd->simple != 0;
269
270         tmp_mmd.lvl = mmd->totlvl;
271         tmp_mmd.sculptlvl = mmd->totlvl;
272         dm = multires_make_derived_from_derived(cddm, &tmp_mmd, scene, ob, 0);
273         cddm->release(cddm);
274
275         return dm;
276 }
277
278 typedef enum ClearFlag {
279         CLEAR_TANGENT_NORMAL = 1,
280         CLEAR_DISPLACEMENT = 2
281 } ClearFlag;
282
283
284 static void clear_single_image(Image *image, ClearFlag flag)
285 {
286         const float vec_alpha[4] = {0.0f, 0.0f, 0.0f, 0.0f};
287         const float vec_solid[4] = {0.0f, 0.0f, 0.0f, 1.0f};
288         const float nor_alpha[4] = {0.5f, 0.5f, 1.0f, 0.0f};
289         const float nor_solid[4] = {0.5f, 0.5f, 1.0f, 1.0f};
290         const float disp_alpha[4] = {0.5f, 0.5f, 0.5f, 0.0f};
291         const float disp_solid[4] = {0.5f, 0.5f, 0.5f, 1.0f};
292
293         if ((image->id.tag & LIB_TAG_DOIT) == 0) {
294                 ImBuf *ibuf = BKE_image_acquire_ibuf(image, NULL, NULL);
295
296                 if (flag == CLEAR_TANGENT_NORMAL)
297                         IMB_rectfill(ibuf, (ibuf->planes == R_IMF_PLANES_RGBA) ? nor_alpha : nor_solid);
298                 else if (flag == CLEAR_DISPLACEMENT)
299                         IMB_rectfill(ibuf, (ibuf->planes == R_IMF_PLANES_RGBA) ? disp_alpha : disp_solid);
300                 else
301                         IMB_rectfill(ibuf, (ibuf->planes == R_IMF_PLANES_RGBA) ? vec_alpha : vec_solid);
302
303                 image->id.tag |= LIB_TAG_DOIT;
304
305                 BKE_image_release_ibuf(image, ibuf, NULL);
306         }
307 }
308
309 static void clear_images_poly(Image **ob_image_array, int ob_image_array_len, ClearFlag flag)
310 {
311         for (int i = 0; i < ob_image_array_len; i++) {
312                 Image *image = ob_image_array[i];
313                 if (image) {
314                         image->id.tag &= ~LIB_TAG_DOIT;
315                 }
316         }
317
318         for (int i = 0; i < ob_image_array_len; i++) {
319                 Image *image = ob_image_array[i];
320                 if (image) {
321                         clear_single_image(image, flag);
322                 }
323         }
324
325         for (int i = 0; i < ob_image_array_len; i++) {
326                 Image *image = ob_image_array[i];
327                 if (image) {
328                         image->id.tag &= ~LIB_TAG_DOIT;
329                 }
330         }
331 }
332
333 static int multiresbake_image_exec_locked(bContext *C, wmOperator *op)
334 {
335         Object *ob;
336         Scene *scene = CTX_data_scene(C);
337         int objects_baked = 0;
338
339         if (!multiresbake_check(C, op))
340                 return OPERATOR_CANCELLED;
341
342         if (scene->r.bake_flag & R_BAKE_CLEAR) {  /* clear images */
343                 CTX_DATA_BEGIN (C, Base *, base, selected_editable_bases)
344                 {
345                         ClearFlag clear_flag = 0;
346
347                         ob = base->object;
348                         // me = (Mesh *)ob->data;
349
350                         if (scene->r.bake_mode == RE_BAKE_NORMALS) {
351                                 clear_flag = CLEAR_TANGENT_NORMAL;
352                         }
353                         else if (scene->r.bake_mode == RE_BAKE_DISPLACEMENT) {
354                                 clear_flag = CLEAR_DISPLACEMENT;
355                         }
356
357                         {
358                                 Image **ob_image_array = bake_object_image_get_array(ob);
359                                 clear_images_poly(ob_image_array, ob->totcol, clear_flag);
360                                 MEM_freeN(ob_image_array);
361                         }
362                 }
363                 CTX_DATA_END;
364         }
365
366         CTX_DATA_BEGIN (C, Base *, base, selected_editable_bases)
367         {
368                 MultiresBakeRender bkr = {NULL};
369
370                 ob = base->object;
371
372                 multires_force_update(ob);
373
374                 /* copy data stored in job descriptor */
375                 bkr.scene = scene;
376                 bkr.bake_filter = scene->r.bake_filter;
377                 bkr.mode = scene->r.bake_mode;
378                 bkr.use_lores_mesh = scene->r.bake_flag & R_BAKE_LORES_MESH;
379                 bkr.bias = scene->r.bake_biasdist;
380                 bkr.number_of_rays = scene->r.bake_samples;
381                 bkr.threads = BKE_scene_num_threads(scene);
382                 bkr.user_scale = (scene->r.bake_flag & R_BAKE_USERSCALE) ? scene->r.bake_user_scale : -1.0f;
383                 //bkr.reports= op->reports;
384
385                 /* create low-resolution DM (to bake to) and hi-resolution DM (to bake from) */
386                 bkr.ob_image.array = bake_object_image_get_array(ob);
387                 bkr.ob_image.len = ob->totcol;
388
389                 bkr.hires_dm = multiresbake_create_hiresdm(scene, ob, &bkr.tot_lvl, &bkr.simple);
390                 bkr.lores_dm = multiresbake_create_loresdm(scene, ob, &bkr.lvl);
391
392                 RE_multires_bake_images(&bkr);
393
394                 MEM_freeN(bkr.ob_image.array);
395
396                 BLI_freelistN(&bkr.image);
397
398                 bkr.lores_dm->release(bkr.lores_dm);
399                 bkr.hires_dm->release(bkr.hires_dm);
400
401                 objects_baked++;
402         }
403         CTX_DATA_END;
404
405         if (!objects_baked)
406                 BKE_report(op->reports, RPT_ERROR, "No objects found to bake from");
407
408         return OPERATOR_FINISHED;
409 }
410
411 /* Multiresbake adopted for job-system executing */
412 static void init_multiresbake_job(bContext *C, MultiresBakeJob *bkj)
413 {
414         Scene *scene = CTX_data_scene(C);
415         Object *ob;
416
417         /* backup scene settings, so their changing in UI would take no effect on baker */
418         bkj->scene = scene;
419         bkj->bake_filter = scene->r.bake_filter;
420         bkj->mode = scene->r.bake_mode;
421         bkj->use_lores_mesh = scene->r.bake_flag & R_BAKE_LORES_MESH;
422         bkj->bake_clear = scene->r.bake_flag & R_BAKE_CLEAR;
423         bkj->bias = scene->r.bake_biasdist;
424         bkj->number_of_rays = scene->r.bake_samples;
425         bkj->threads = BKE_scene_num_threads(scene);
426         bkj->user_scale = (scene->r.bake_flag & R_BAKE_USERSCALE) ? scene->r.bake_user_scale : -1.0f;
427         //bkj->reports = op->reports;
428
429         CTX_DATA_BEGIN (C, Base *, base, selected_editable_bases)
430         {
431                 MultiresBakerJobData *data;
432                 int lvl;
433
434                 ob = base->object;
435
436                 multires_force_update(ob);
437
438                 data = MEM_callocN(sizeof(MultiresBakerJobData), "multiresBaker derivedMesh_data");
439
440                 data->ob_image.array = bake_object_image_get_array(ob);
441                 data->ob_image.len = ob->totcol;
442
443                 /* create low-resolution DM (to bake to) and hi-resolution DM (to bake from) */
444                 data->hires_dm = multiresbake_create_hiresdm(scene, ob, &data->tot_lvl, &data->simple);
445                 data->lores_dm = multiresbake_create_loresdm(scene, ob, &lvl);
446                 data->lvl = lvl;
447
448                 BLI_addtail(&bkj->data, data);
449         }
450         CTX_DATA_END;
451 }
452
453 static void multiresbake_startjob(void *bkv, short *stop, short *do_update, float *progress)
454 {
455         MultiresBakerJobData *data;
456         MultiresBakeJob *bkj = bkv;
457         int baked_objects = 0, tot_obj;
458
459         tot_obj = BLI_listbase_count(&bkj->data);
460
461         if (bkj->bake_clear) {  /* clear images */
462                 for (data = bkj->data.first; data; data = data->next) {
463                         ClearFlag clear_flag = 0;
464
465                         if (bkj->mode == RE_BAKE_NORMALS) {
466                                 clear_flag = CLEAR_TANGENT_NORMAL;
467                         }
468                         else if (bkj->mode == RE_BAKE_DISPLACEMENT) {
469                                 clear_flag = CLEAR_DISPLACEMENT;
470                         }
471
472                         clear_images_poly(data->ob_image.array, data->ob_image.len, clear_flag);
473                 }
474         }
475
476         for (data = bkj->data.first; data; data = data->next) {
477                 MultiresBakeRender bkr = {NULL};
478
479                 /* copy data stored in job descriptor */
480                 bkr.scene = bkj->scene;
481                 bkr.bake_filter = bkj->bake_filter;
482                 bkr.mode = bkj->mode;
483                 bkr.use_lores_mesh = bkj->use_lores_mesh;
484                 bkr.user_scale = bkj->user_scale;
485                 //bkr.reports = bkj->reports;
486                 bkr.ob_image.array = data->ob_image.array;
487                 bkr.ob_image.len   = data->ob_image.len;
488
489                 /* create low-resolution DM (to bake to) and hi-resolution DM (to bake from) */
490                 bkr.lores_dm = data->lores_dm;
491                 bkr.hires_dm = data->hires_dm;
492                 bkr.tot_lvl = data->tot_lvl;
493                 bkr.lvl = data->lvl;
494                 bkr.simple = data->simple;
495
496                 /* needed for proper progress bar */
497                 bkr.tot_obj = tot_obj;
498                 bkr.baked_objects = baked_objects;
499
500                 bkr.stop = stop;
501                 bkr.do_update = do_update;
502                 bkr.progress = progress;
503
504                 bkr.bias = bkj->bias;
505                 bkr.number_of_rays = bkj->number_of_rays;
506                 bkr.threads = bkj->threads;
507
508                 RE_multires_bake_images(&bkr);
509
510                 data->images = bkr.image;
511
512                 baked_objects++;
513         }
514 }
515
516 static void multiresbake_freejob(void *bkv)
517 {
518         MultiresBakeJob *bkj = bkv;
519         MultiresBakerJobData *data, *next;
520         LinkData *link;
521
522         data = bkj->data.first;
523         while (data) {
524                 next = data->next;
525                 data->lores_dm->release(data->lores_dm);
526                 data->hires_dm->release(data->hires_dm);
527
528                 /* delete here, since this delete will be called from main thread */
529                 for (link = data->images.first; link; link = link->next) {
530                         Image *ima = (Image *)link->data;
531                         GPU_free_image(ima);
532                 }
533
534                 MEM_freeN(data->ob_image.array);
535
536                 BLI_freelistN(&data->images);
537
538                 MEM_freeN(data);
539                 data = next;
540         }
541
542         MEM_freeN(bkj);
543 }
544
545 static int multiresbake_image_exec(bContext *C, wmOperator *op)
546 {
547         Scene *scene = CTX_data_scene(C);
548         MultiresBakeJob *bkr;
549         wmJob *wm_job;
550
551         if (!multiresbake_check(C, op))
552                 return OPERATOR_CANCELLED;
553
554         bkr = MEM_callocN(sizeof(MultiresBakeJob), "MultiresBakeJob data");
555         init_multiresbake_job(C, bkr);
556
557         if (!bkr->data.first) {
558                 BKE_report(op->reports, RPT_ERROR, "No objects found to bake from");
559                 return OPERATOR_CANCELLED;
560         }
561
562         /* setup job */
563         wm_job = WM_jobs_get(CTX_wm_manager(C), CTX_wm_window(C), scene, "Multires Bake",
564                              WM_JOB_EXCL_RENDER | WM_JOB_PRIORITY | WM_JOB_PROGRESS, WM_JOB_TYPE_OBJECT_BAKE_TEXTURE);
565         WM_jobs_customdata_set(wm_job, bkr, multiresbake_freejob);
566         WM_jobs_timer(wm_job, 0.5, NC_IMAGE, 0); /* TODO - only draw bake image, can we enforce this */
567         WM_jobs_callbacks(wm_job, multiresbake_startjob, NULL, NULL, NULL);
568
569         G.is_break = false;
570
571         WM_jobs_start(CTX_wm_manager(C), wm_job);
572         WM_cursor_wait(0);
573
574         /* add modal handler for ESC */
575         WM_event_add_modal_handler(C, op);
576
577         return OPERATOR_RUNNING_MODAL;
578 }
579
580 /* ****************** render BAKING ********************** */
581
582 /* catch esc */
583 static int objects_bake_render_modal(bContext *C, wmOperator *UNUSED(op), const wmEvent *event)
584 {
585         /* no running blender, remove handler and pass through */
586         if (0 == WM_jobs_test(CTX_wm_manager(C), CTX_data_scene(C), WM_JOB_TYPE_OBJECT_BAKE_TEXTURE))
587                 return OPERATOR_FINISHED | OPERATOR_PASS_THROUGH;
588
589         /* running render */
590         switch (event->type) {
591                 case ESCKEY:
592                         return OPERATOR_RUNNING_MODAL;
593         }
594         return OPERATOR_PASS_THROUGH;
595 }
596
597 static bool is_multires_bake(Scene *scene)
598 {
599         if (ELEM(scene->r.bake_mode, RE_BAKE_NORMALS, RE_BAKE_DISPLACEMENT, RE_BAKE_AO))
600                 return scene->r.bake_flag & R_BAKE_MULTIRES;
601
602         return 0;
603 }
604
605 static int objects_bake_render_invoke(bContext *C, wmOperator *op, const wmEvent *UNUSED(_event))
606 {
607         Scene *scene = CTX_data_scene(C);
608         int result = OPERATOR_CANCELLED;
609
610         result = multiresbake_image_exec(C, op);
611
612         WM_event_add_notifier(C, NC_SCENE | ND_RENDER_RESULT, scene);
613
614         return result;
615 }
616
617
618 static int bake_image_exec(bContext *C, wmOperator *op)
619 {
620         Scene *scene = CTX_data_scene(C);
621         int result = OPERATOR_CANCELLED;
622
623         if (!is_multires_bake(scene)) {
624                 BLI_assert(0);
625                 return result;
626         }
627
628         result = multiresbake_image_exec_locked(C, op);
629
630         WM_event_add_notifier(C, NC_SCENE | ND_RENDER_RESULT, scene);
631
632         return result;
633 }
634
635 void OBJECT_OT_bake_image(wmOperatorType *ot)
636 {
637         /* identifiers */
638         ot->name = "Bake";
639         ot->description = "Bake image textures of selected objects";
640         ot->idname = "OBJECT_OT_bake_image";
641
642         /* api callbacks */
643         ot->exec = bake_image_exec;
644         ot->invoke = objects_bake_render_invoke;
645         ot->modal = objects_bake_render_modal;
646         ot->poll = ED_operator_object_active;
647 }