BLI: improve Map.add_new
[blender.git] / source / blender / editors / object / object_bake_api.c
1 /*
2  * This program is free software; you can redistribute it and/or
3  * modify it under the terms of the GNU General Public License
4  * as published by the Free Software Foundation; either version 2
5  * of the License, or (at your option) any later version.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * along with this program; if not, write to the Free Software Foundation,
14  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15  *
16  * The Original Code is Copyright (C) 2004 by Blender Foundation
17  * All rights reserved.
18  */
19
20 /** \file
21  * \ingroup edobj
22  */
23
24 #include "MEM_guardedalloc.h"
25
26 #include "DNA_material_types.h"
27 #include "DNA_mesh_types.h"
28 #include "DNA_object_types.h"
29
30 #include "RNA_access.h"
31 #include "RNA_define.h"
32 #include "RNA_enum_types.h"
33
34 #include "BLI_fileops.h"
35 #include "BLI_listbase.h"
36 #include "BLI_path_util.h"
37
38 #include "BKE_context.h"
39 #include "BKE_global.h"
40 #include "BKE_image.h"
41 #include "BKE_layer.h"
42 #include "BKE_lib_id.h"
43 #include "BKE_main.h"
44 #include "BKE_material.h"
45 #include "BKE_mesh.h"
46 #include "BKE_modifier.h"
47 #include "BKE_node.h"
48 #include "BKE_object.h"
49 #include "BKE_report.h"
50 #include "BKE_scene.h"
51 #include "BKE_screen.h"
52
53 #include "DEG_depsgraph.h"
54 #include "DEG_depsgraph_build.h"
55 #include "DEG_depsgraph_query.h"
56
57 #include "RE_engine.h"
58 #include "RE_pipeline.h"
59
60 #include "IMB_colormanagement.h"
61 #include "IMB_imbuf.h"
62 #include "IMB_imbuf_types.h"
63
64 #include "WM_api.h"
65 #include "WM_types.h"
66
67 #include "ED_object.h"
68 #include "ED_screen.h"
69 #include "ED_uvedit.h"
70
71 #include "object_intern.h"
72
73 /* prototypes */
74 static void bake_set_props(wmOperator *op, Scene *scene);
75
76 typedef struct BakeAPIRender {
77   Object *ob;
78   Main *main;
79   Scene *scene;
80   ViewLayer *view_layer;
81   ReportList *reports;
82   ListBase selected_objects;
83
84   eScenePassType pass_type;
85   int pass_filter;
86   int margin;
87
88   int save_mode;
89
90   bool is_clear;
91   bool is_split_materials;
92   bool is_automatic_name;
93   bool is_selected_to_active;
94   bool is_cage;
95
96   float cage_extrusion;
97   float max_ray_distance;
98   int normal_space;
99   eBakeNormalSwizzle normal_swizzle[3];
100
101   char uv_layer[MAX_CUSTOMDATA_LAYER_NAME];
102   char custom_cage[MAX_NAME];
103   char filepath[FILE_MAX];
104
105   int width;
106   int height;
107   const char *identifier;
108
109   int result;
110   bool ready;
111
112   /* callbacks */
113   Render *render;
114   float *progress;
115   short *do_update;
116
117   /* for redrawing */
118   ScrArea *area;
119 } BakeAPIRender;
120
121 /* callbacks */
122
123 static void bake_progress_update(void *bjv, float progress)
124 {
125   BakeAPIRender *bj = bjv;
126
127   if (bj->progress && *bj->progress != progress) {
128     *bj->progress = progress;
129
130     /* make jobs timer to send notifier */
131     *(bj->do_update) = true;
132   }
133 }
134
135 /* catch esc */
136 static int bake_modal(bContext *C, wmOperator *UNUSED(op), const wmEvent *event)
137 {
138   /* no running blender, remove handler and pass through */
139   if (0 == WM_jobs_test(CTX_wm_manager(C), CTX_data_scene(C), WM_JOB_TYPE_OBJECT_BAKE)) {
140     return OPERATOR_FINISHED | OPERATOR_PASS_THROUGH;
141   }
142
143   /* running render */
144   switch (event->type) {
145     case EVT_ESCKEY: {
146       G.is_break = true;
147       return OPERATOR_RUNNING_MODAL;
148     }
149   }
150   return OPERATOR_PASS_THROUGH;
151 }
152
153 /* for exec() when there is no render job
154  * note: this wont check for the escape key being pressed, but doing so isnt threadsafe */
155 static int bake_break(void *UNUSED(rjv))
156 {
157   if (G.is_break) {
158     return 1;
159   }
160   return 0;
161 }
162
163 static void bake_update_image(ScrArea *area, Image *image)
164 {
165   if (area && area->spacetype == SPACE_IMAGE) { /* in case the user changed while baking */
166     SpaceImage *sima = area->spacedata.first;
167     if (sima) {
168       sima->image = image;
169     }
170   }
171 }
172
173 static bool write_internal_bake_pixels(Image *image,
174                                        BakePixel pixel_array[],
175                                        float *buffer,
176                                        const int width,
177                                        const int height,
178                                        const int margin,
179                                        const bool is_clear,
180                                        const bool is_noncolor)
181 {
182   ImBuf *ibuf;
183   void *lock;
184   bool is_float;
185   char *mask_buffer = NULL;
186   const size_t num_pixels = (size_t)width * (size_t)height;
187
188   ibuf = BKE_image_acquire_ibuf(image, NULL, &lock);
189
190   if (!ibuf) {
191     return false;
192   }
193
194   if (margin > 0 || !is_clear) {
195     mask_buffer = MEM_callocN(sizeof(char) * num_pixels, "Bake Mask");
196     RE_bake_mask_fill(pixel_array, num_pixels, mask_buffer);
197   }
198
199   is_float = (ibuf->rect_float != NULL);
200
201   /* colormanagement conversions */
202   if (!is_noncolor) {
203     const char *from_colorspace;
204     const char *to_colorspace;
205
206     from_colorspace = IMB_colormanagement_role_colorspace_name_get(COLOR_ROLE_SCENE_LINEAR);
207
208     if (is_float) {
209       to_colorspace = IMB_colormanagement_get_float_colorspace(ibuf);
210     }
211     else {
212       to_colorspace = IMB_colormanagement_get_rect_colorspace(ibuf);
213     }
214
215     if (from_colorspace != to_colorspace) {
216       IMB_colormanagement_transform(
217           buffer, ibuf->x, ibuf->y, ibuf->channels, from_colorspace, to_colorspace, false);
218     }
219   }
220
221   /* populates the ImBuf */
222   if (is_clear) {
223     if (is_float) {
224       IMB_buffer_float_from_float(ibuf->rect_float,
225                                   buffer,
226                                   ibuf->channels,
227                                   IB_PROFILE_LINEAR_RGB,
228                                   IB_PROFILE_LINEAR_RGB,
229                                   false,
230                                   ibuf->x,
231                                   ibuf->y,
232                                   ibuf->x,
233                                   ibuf->x);
234     }
235     else {
236       IMB_buffer_byte_from_float((uchar *)ibuf->rect,
237                                  buffer,
238                                  ibuf->channels,
239                                  ibuf->dither,
240                                  IB_PROFILE_SRGB,
241                                  IB_PROFILE_SRGB,
242                                  false,
243                                  ibuf->x,
244                                  ibuf->y,
245                                  ibuf->x,
246                                  ibuf->x);
247     }
248   }
249   else {
250     if (is_float) {
251       IMB_buffer_float_from_float_mask(ibuf->rect_float,
252                                        buffer,
253                                        ibuf->channels,
254                                        ibuf->x,
255                                        ibuf->y,
256                                        ibuf->x,
257                                        ibuf->x,
258                                        mask_buffer);
259     }
260     else {
261       IMB_buffer_byte_from_float_mask((uchar *)ibuf->rect,
262                                       buffer,
263                                       ibuf->channels,
264                                       ibuf->dither,
265                                       false,
266                                       ibuf->x,
267                                       ibuf->y,
268                                       ibuf->x,
269                                       ibuf->x,
270                                       mask_buffer);
271     }
272   }
273
274   /* margins */
275   if (margin > 0) {
276     RE_bake_margin(ibuf, mask_buffer, margin);
277   }
278
279   ibuf->userflags |= IB_DISPLAY_BUFFER_INVALID;
280   BKE_image_mark_dirty(image, ibuf);
281
282   if (ibuf->rect_float) {
283     ibuf->userflags |= IB_RECT_INVALID;
284   }
285
286   /* force mipmap recalc */
287   if (ibuf->mipmap[0]) {
288     ibuf->userflags |= IB_MIPMAP_INVALID;
289     imb_freemipmapImBuf(ibuf);
290   }
291
292   BKE_image_release_ibuf(image, ibuf, NULL);
293
294   if (mask_buffer) {
295     MEM_freeN(mask_buffer);
296   }
297
298   return true;
299 }
300
301 /* force OpenGL reload */
302 static void refresh_images(BakeImages *bake_images)
303 {
304   for (int i = 0; i < bake_images->size; i++) {
305     Image *ima = bake_images->data[i].image;
306     LISTBASE_FOREACH (ImageTile *, tile, &ima->tiles) {
307       if (tile->ok == IMA_OK_LOADED) {
308         BKE_image_free_gputextures(ima);
309         DEG_id_tag_update(&ima->id, 0);
310         break;
311       }
312     }
313   }
314 }
315
316 static bool write_external_bake_pixels(const char *filepath,
317                                        BakePixel pixel_array[],
318                                        float *buffer,
319                                        const int width,
320                                        const int height,
321                                        const int margin,
322                                        ImageFormatData *im_format,
323                                        const bool is_noncolor)
324 {
325   ImBuf *ibuf = NULL;
326   bool ok = false;
327   bool is_float;
328
329   is_float = im_format->depth > 8;
330
331   /* create a new ImBuf */
332   ibuf = IMB_allocImBuf(width, height, im_format->planes, (is_float ? IB_rectfloat : IB_rect));
333
334   if (!ibuf) {
335     return false;
336   }
337
338   /* populates the ImBuf */
339   if (is_float) {
340     IMB_buffer_float_from_float(ibuf->rect_float,
341                                 buffer,
342                                 ibuf->channels,
343                                 IB_PROFILE_LINEAR_RGB,
344                                 IB_PROFILE_LINEAR_RGB,
345                                 false,
346                                 ibuf->x,
347                                 ibuf->y,
348                                 ibuf->x,
349                                 ibuf->x);
350   }
351   else {
352     if (!is_noncolor) {
353       const char *from_colorspace = IMB_colormanagement_role_colorspace_name_get(
354           COLOR_ROLE_SCENE_LINEAR);
355       const char *to_colorspace = IMB_colormanagement_get_rect_colorspace(ibuf);
356       IMB_colormanagement_transform(
357           buffer, ibuf->x, ibuf->y, ibuf->channels, from_colorspace, to_colorspace, false);
358     }
359
360     IMB_buffer_byte_from_float((uchar *)ibuf->rect,
361                                buffer,
362                                ibuf->channels,
363                                ibuf->dither,
364                                IB_PROFILE_SRGB,
365                                IB_PROFILE_SRGB,
366                                false,
367                                ibuf->x,
368                                ibuf->y,
369                                ibuf->x,
370                                ibuf->x);
371   }
372
373   /* margins */
374   if (margin > 0) {
375     char *mask_buffer = NULL;
376     const size_t num_pixels = (size_t)width * (size_t)height;
377
378     mask_buffer = MEM_callocN(sizeof(char) * num_pixels, "Bake Mask");
379     RE_bake_mask_fill(pixel_array, num_pixels, mask_buffer);
380     RE_bake_margin(ibuf, mask_buffer, margin);
381
382     if (mask_buffer) {
383       MEM_freeN(mask_buffer);
384     }
385   }
386
387   if ((ok = BKE_imbuf_write(ibuf, filepath, im_format))) {
388 #ifndef WIN32
389     chmod(filepath, S_IRUSR | S_IWUSR);
390 #endif
391     // printf("%s saving bake map: '%s'\n", __func__, filepath);
392   }
393
394   /* garbage collection */
395   IMB_freeImBuf(ibuf);
396
397   return ok;
398 }
399
400 static bool is_noncolor_pass(eScenePassType pass_type)
401 {
402   return ELEM(pass_type,
403               SCE_PASS_Z,
404               SCE_PASS_NORMAL,
405               SCE_PASS_VECTOR,
406               SCE_PASS_INDEXOB,
407               SCE_PASS_UV,
408               SCE_PASS_RAYHITS,
409               SCE_PASS_INDEXMA);
410 }
411
412 /* if all is good tag image and return true */
413 static bool bake_object_check(ViewLayer *view_layer, Object *ob, ReportList *reports)
414 {
415   Base *base = BKE_view_layer_base_find(view_layer, ob);
416
417   if (base == NULL) {
418     BKE_reportf(reports, RPT_ERROR, "Object \"%s\" is not in view layer", ob->id.name + 2);
419     return false;
420   }
421
422   if (!(base->flag & BASE_ENABLED_RENDER)) {
423     BKE_reportf(reports, RPT_ERROR, "Object \"%s\" is not enabled for rendering", ob->id.name + 2);
424     return false;
425   }
426
427   if (ob->type != OB_MESH) {
428     BKE_reportf(reports, RPT_ERROR, "Object \"%s\" is not a mesh", ob->id.name + 2);
429     return false;
430   }
431
432   Mesh *me = (Mesh *)ob->data;
433   if (CustomData_get_active_layer_index(&me->ldata, CD_MLOOPUV) == -1) {
434     BKE_reportf(
435         reports, RPT_ERROR, "No active UV layer found in the object \"%s\"", ob->id.name + 2);
436     return false;
437   }
438
439   for (int i = 0; i < ob->totcol; i++) {
440     bNodeTree *ntree = NULL;
441     bNode *node = NULL;
442     const int mat_nr = i + 1;
443     Image *image;
444     ED_object_get_active_image(ob, mat_nr, &image, NULL, &node, &ntree);
445
446     if (image) {
447       ImBuf *ibuf;
448
449       if (node) {
450         if (BKE_node_is_connected_to_output(ntree, node)) {
451           /* we don't return false since this may be a false positive
452            * this can't be RPT_ERROR though, otherwise it prevents
453            * multiple highpoly objects to be baked at once */
454           BKE_reportf(reports,
455                       RPT_INFO,
456                       "Circular dependency for image \"%s\" from object \"%s\"",
457                       image->id.name + 2,
458                       ob->id.name + 2);
459         }
460       }
461
462       void *lock;
463       ibuf = BKE_image_acquire_ibuf(image, NULL, &lock);
464
465       if (ibuf) {
466         BKE_image_release_ibuf(image, ibuf, lock);
467       }
468       else {
469         BKE_reportf(reports,
470                     RPT_ERROR,
471                     "Uninitialized image \"%s\" from object \"%s\"",
472                     image->id.name + 2,
473                     ob->id.name + 2);
474
475         BKE_image_release_ibuf(image, ibuf, lock);
476         return false;
477       }
478     }
479     else {
480       Material *mat = BKE_object_material_get(ob, mat_nr);
481       if (mat != NULL) {
482         BKE_reportf(reports,
483                     RPT_INFO,
484                     "No active image found in material \"%s\" (%d) for object \"%s\"",
485                     mat->id.name + 2,
486                     i,
487                     ob->id.name + 2);
488       }
489       else {
490         BKE_reportf(reports,
491                     RPT_INFO,
492                     "No active image found in material slot (%d) for object \"%s\"",
493                     i,
494                     ob->id.name + 2);
495       }
496       continue;
497     }
498
499     image->id.tag |= LIB_TAG_DOIT;
500   }
501   return true;
502 }
503
504 static bool bake_pass_filter_check(eScenePassType pass_type,
505                                    const int pass_filter,
506                                    ReportList *reports)
507 {
508   switch (pass_type) {
509     case SCE_PASS_COMBINED:
510       if ((pass_filter & R_BAKE_PASS_FILTER_EMIT) != 0) {
511         return true;
512       }
513
514       if (((pass_filter & R_BAKE_PASS_FILTER_DIRECT) != 0) ||
515           ((pass_filter & R_BAKE_PASS_FILTER_INDIRECT) != 0)) {
516         if (((pass_filter & R_BAKE_PASS_FILTER_DIFFUSE) != 0) ||
517             ((pass_filter & R_BAKE_PASS_FILTER_GLOSSY) != 0) ||
518             ((pass_filter & R_BAKE_PASS_FILTER_TRANSM) != 0) ||
519             ((pass_filter & R_BAKE_PASS_FILTER_SUBSURFACE) != 0)) {
520           return true;
521         }
522
523         if ((pass_filter & R_BAKE_PASS_FILTER_AO) != 0) {
524           BKE_report(
525               reports,
526               RPT_ERROR,
527               "Combined bake pass Ambient Occlusion contribution requires an enabled light pass "
528               "(bake the Ambient Occlusion pass type instead)");
529         }
530         else {
531           BKE_report(reports,
532                      RPT_ERROR,
533                      "Combined bake pass requires Emit, or a light pass with "
534                      "Direct or Indirect contributions enabled");
535         }
536
537         return false;
538       }
539       BKE_report(reports,
540                  RPT_ERROR,
541                  "Combined bake pass requires Emit, or a light pass with "
542                  "Direct or Indirect contributions enabled");
543       return false;
544     case SCE_PASS_DIFFUSE_COLOR:
545     case SCE_PASS_GLOSSY_COLOR:
546     case SCE_PASS_TRANSM_COLOR:
547     case SCE_PASS_SUBSURFACE_COLOR:
548       if (((pass_filter & R_BAKE_PASS_FILTER_COLOR) != 0) ||
549           ((pass_filter & R_BAKE_PASS_FILTER_DIRECT) != 0) ||
550           ((pass_filter & R_BAKE_PASS_FILTER_INDIRECT) != 0)) {
551         return true;
552       }
553       else {
554         BKE_report(reports,
555                    RPT_ERROR,
556                    "Bake pass requires Direct, Indirect, or Color contributions to be enabled");
557         return false;
558       }
559       break;
560     default:
561       return true;
562       break;
563   }
564 }
565
566 /* before even getting in the bake function we check for some basic errors */
567 static bool bake_objects_check(Main *bmain,
568                                ViewLayer *view_layer,
569                                Object *ob,
570                                ListBase *selected_objects,
571                                ReportList *reports,
572                                const bool is_selected_to_active)
573 {
574   CollectionPointerLink *link;
575
576   /* error handling and tag (in case multiple materials share the same image) */
577   BKE_main_id_tag_idcode(bmain, ID_IM, LIB_TAG_DOIT, false);
578
579   if (is_selected_to_active) {
580     int tot_objects = 0;
581
582     if (!bake_object_check(view_layer, ob, reports)) {
583       return false;
584     }
585
586     for (link = selected_objects->first; link; link = link->next) {
587       Object *ob_iter = (Object *)link->ptr.data;
588
589       if (ob_iter == ob) {
590         continue;
591       }
592
593       if (ELEM(ob_iter->type, OB_MESH, OB_FONT, OB_CURVE, OB_SURF, OB_MBALL) == false) {
594         BKE_reportf(reports,
595                     RPT_ERROR,
596                     "Object \"%s\" is not a mesh or can't be converted to a mesh (Curve, Text, "
597                     "Surface or Metaball)",
598                     ob_iter->id.name + 2);
599         return false;
600       }
601       tot_objects += 1;
602     }
603
604     if (tot_objects == 0) {
605       BKE_report(reports, RPT_ERROR, "No valid selected objects");
606       return false;
607     }
608   }
609   else {
610     if (BLI_listbase_is_empty(selected_objects)) {
611       BKE_report(reports, RPT_ERROR, "No valid selected objects");
612       return false;
613     }
614
615     for (link = selected_objects->first; link; link = link->next) {
616       if (!bake_object_check(view_layer, link->ptr.data, reports)) {
617         return false;
618       }
619     }
620   }
621   return true;
622 }
623
624 /* it needs to be called after bake_objects_check since the image tagging happens there */
625 static void bake_images_clear(Main *bmain, const bool is_tangent)
626 {
627   Image *image;
628   for (image = bmain->images.first; image; image = image->id.next) {
629     if ((image->id.tag & LIB_TAG_DOIT) != 0) {
630       RE_bake_ibuf_clear(image, is_tangent);
631     }
632   }
633 }
634
635 static void build_image_lookup(Main *bmain, Object *ob, BakeImages *bake_images)
636 {
637   const int tot_mat = ob->totcol;
638   int tot_images = 0;
639
640   /* error handling and tag (in case multiple materials share the same image) */
641   BKE_main_id_tag_idcode(bmain, ID_IM, LIB_TAG_DOIT, false);
642
643   for (int i = 0; i < tot_mat; i++) {
644     Image *image;
645     ED_object_get_active_image(ob, i + 1, &image, NULL, NULL, NULL);
646
647     /* Some materials have no image, we just ignore those cases. */
648     if (image == NULL) {
649       bake_images->lookup[i] = -1;
650     }
651     else if (image->id.tag & LIB_TAG_DOIT) {
652       for (int j = 0; j < i; j++) {
653         if (bake_images->data[j].image == image) {
654           bake_images->lookup[i] = j;
655           break;
656         }
657       }
658     }
659     else {
660       bake_images->lookup[i] = tot_images;
661       bake_images->data[tot_images].image = image;
662       image->id.tag |= LIB_TAG_DOIT;
663       tot_images++;
664     }
665   }
666
667   bake_images->size = tot_images;
668 }
669
670 /*
671  * returns the total number of pixels
672  */
673 static size_t init_internal_images(BakeImages *bake_images, ReportList *reports)
674 {
675   size_t tot_size = 0;
676
677   for (int i = 0; i < bake_images->size; i++) {
678     BakeImage *bk_image = &bake_images->data[i];
679     void *lock;
680     ImBuf *ibuf = BKE_image_acquire_ibuf(bk_image->image, NULL, &lock);
681
682     if (ibuf) {
683       bk_image->width = ibuf->x;
684       bk_image->height = ibuf->y;
685       bk_image->offset = tot_size;
686
687       tot_size += (size_t)ibuf->x * (size_t)ibuf->y;
688     }
689     else {
690       BKE_image_release_ibuf(bk_image->image, ibuf, lock);
691       BKE_reportf(reports, RPT_ERROR, "Uninitialized image %s", bk_image->image->id.name + 2);
692       return 0;
693     }
694     BKE_image_release_ibuf(bk_image->image, ibuf, lock);
695   }
696   return tot_size;
697 }
698
699 /* create new mesh with edit mode changes and modifiers applied */
700 static Mesh *bake_mesh_new_from_object(Object *object)
701 {
702   Mesh *me = BKE_mesh_new_from_object(NULL, object, false);
703
704   if (me->flag & ME_AUTOSMOOTH) {
705     BKE_mesh_split_faces(me, true);
706   }
707
708   return me;
709 }
710
711 static int bake(Render *re,
712                 Main *bmain,
713                 Scene *scene,
714                 ViewLayer *view_layer,
715                 Object *ob_low,
716                 ListBase *selected_objects,
717                 ReportList *reports,
718                 const eScenePassType pass_type,
719                 const int pass_filter,
720                 const int margin,
721                 const eBakeSaveMode save_mode,
722                 const bool is_clear,
723                 const bool is_split_materials,
724                 const bool is_automatic_name,
725                 const bool is_selected_to_active,
726                 const bool is_cage,
727                 const float cage_extrusion,
728                 const float max_ray_distance,
729                 const int normal_space,
730                 const eBakeNormalSwizzle normal_swizzle[],
731                 const char *custom_cage,
732                 const char *filepath,
733                 const int width,
734                 const int height,
735                 const char *identifier,
736                 ScrArea *area,
737                 const char *uv_layer)
738 {
739   /* We build a depsgraph for the baking,
740    * so we don't need to change the original data to adjust visibility and modifiers. */
741   Depsgraph *depsgraph = DEG_graph_new(bmain, scene, view_layer, DAG_EVAL_RENDER);
742   DEG_graph_build_from_view_layer(depsgraph);
743
744   int op_result = OPERATOR_CANCELLED;
745   bool ok = false;
746
747   Object *ob_cage = NULL;
748   Object *ob_cage_eval = NULL;
749   Object *ob_low_eval = NULL;
750
751   BakeHighPolyData *highpoly = NULL;
752   int tot_highpoly = 0;
753
754   Mesh *me_low = NULL;
755   Mesh *me_cage = NULL;
756
757   MultiresModifierData *mmd_low = NULL;
758   int mmd_flags_low = 0;
759
760   float *result = NULL;
761
762   BakePixel *pixel_array_low = NULL;
763   BakePixel *pixel_array_high = NULL;
764
765   const bool is_save_internal = (save_mode == R_BAKE_SAVE_INTERNAL);
766   const bool is_noncolor = is_noncolor_pass(pass_type);
767   const int depth = RE_pass_depth(pass_type);
768
769   BakeImages bake_images = {NULL};
770
771   size_t num_pixels;
772   int tot_materials;
773
774   RE_bake_engine_set_engine_parameters(re, bmain, scene);
775
776   if (!RE_bake_has_engine(re)) {
777     BKE_report(reports, RPT_ERROR, "Current render engine does not support baking");
778     goto cleanup;
779   }
780
781   tot_materials = ob_low->totcol;
782
783   if (uv_layer && uv_layer[0] != '\0') {
784     Mesh *me = (Mesh *)ob_low->data;
785     if (CustomData_get_named_layer(&me->ldata, CD_MLOOPUV, uv_layer) == -1) {
786       BKE_reportf(reports,
787                   RPT_ERROR,
788                   "No UV layer named \"%s\" found in the object \"%s\"",
789                   uv_layer,
790                   ob_low->id.name + 2);
791       goto cleanup;
792     }
793   }
794
795   if (tot_materials == 0) {
796     if (is_save_internal) {
797       BKE_report(
798           reports, RPT_ERROR, "No active image found, add a material or bake to an external file");
799
800       goto cleanup;
801     }
802     else if (is_split_materials) {
803       BKE_report(
804           reports,
805           RPT_ERROR,
806           "No active image found, add a material or bake without the Split Materials option");
807
808       goto cleanup;
809     }
810     else {
811       /* baking externally without splitting materials */
812       tot_materials = 1;
813     }
814   }
815
816   /* we overallocate in case there is more materials than images */
817   bake_images.data = MEM_mallocN(sizeof(BakeImage) * tot_materials,
818                                  "bake images dimensions (width, height, offset)");
819   bake_images.lookup = MEM_mallocN(sizeof(int) * tot_materials,
820                                    "bake images lookup (from material to BakeImage)");
821
822   build_image_lookup(bmain, ob_low, &bake_images);
823
824   if (is_save_internal) {
825     num_pixels = init_internal_images(&bake_images, reports);
826
827     if (num_pixels == 0) {
828       goto cleanup;
829     }
830   }
831   else {
832     /* when saving externally always use the size specified in the UI */
833
834     num_pixels = (size_t)width * (size_t)height * bake_images.size;
835
836     for (int i = 0; i < bake_images.size; i++) {
837       bake_images.data[i].width = width;
838       bake_images.data[i].height = height;
839       bake_images.data[i].offset = (is_split_materials ? num_pixels : 0);
840       bake_images.data[i].image = NULL;
841     }
842
843     if (!is_split_materials) {
844       /* saving a single image */
845       for (int i = 0; i < tot_materials; i++) {
846         bake_images.lookup[i] = 0;
847       }
848     }
849   }
850
851   if (is_selected_to_active) {
852     CollectionPointerLink *link;
853     tot_highpoly = 0;
854
855     for (link = selected_objects->first; link; link = link->next) {
856       Object *ob_iter = link->ptr.data;
857
858       if (ob_iter == ob_low) {
859         continue;
860       }
861
862       tot_highpoly++;
863     }
864
865     if (is_cage && custom_cage[0] != '\0') {
866       ob_cage = BLI_findstring(&bmain->objects, custom_cage, offsetof(ID, name) + 2);
867
868       if (ob_cage == NULL || ob_cage->type != OB_MESH) {
869         BKE_report(reports, RPT_ERROR, "No valid cage object");
870         goto cleanup;
871       }
872       else {
873         ob_cage_eval = DEG_get_evaluated_object(depsgraph, ob_cage);
874         ob_cage_eval->restrictflag |= OB_RESTRICT_RENDER;
875         ob_cage_eval->base_flag &= ~(BASE_VISIBLE_DEPSGRAPH | BASE_ENABLED_RENDER);
876       }
877     }
878   }
879
880   pixel_array_low = MEM_mallocN(sizeof(BakePixel) * num_pixels, "bake pixels low poly");
881   pixel_array_high = MEM_mallocN(sizeof(BakePixel) * num_pixels, "bake pixels high poly");
882   result = MEM_callocN(sizeof(float) * depth * num_pixels, "bake return pixels");
883
884   /* for multires bake, use linear UV subdivision to match low res UVs */
885   if (pass_type == SCE_PASS_NORMAL && normal_space == R_BAKE_SPACE_TANGENT &&
886       !is_selected_to_active) {
887     mmd_low = (MultiresModifierData *)BKE_modifiers_findby_type(ob_low, eModifierType_Multires);
888     if (mmd_low) {
889       mmd_flags_low = mmd_low->flags;
890       mmd_low->uv_smooth = SUBSURF_UV_SMOOTH_NONE;
891     }
892   }
893
894   /* Make sure depsgraph is up to date. */
895   BKE_scene_graph_update_tagged(depsgraph, bmain);
896   ob_low_eval = DEG_get_evaluated_object(depsgraph, ob_low);
897
898   /* get the mesh as it arrives in the renderer */
899   me_low = bake_mesh_new_from_object(ob_low_eval);
900
901   /* populate the pixel array with the face data */
902   if ((is_selected_to_active && (ob_cage == NULL) && is_cage) == false) {
903     RE_bake_pixels_populate(me_low, pixel_array_low, num_pixels, &bake_images, uv_layer);
904   }
905   /* else populate the pixel array with the 'cage' mesh (the smooth version of the mesh)  */
906
907   if (is_selected_to_active) {
908     CollectionPointerLink *link;
909     int i = 0;
910
911     /* prepare cage mesh */
912     if (ob_cage) {
913       me_cage = bake_mesh_new_from_object(ob_cage_eval);
914       if ((me_low->totpoly != me_cage->totpoly) || (me_low->totloop != me_cage->totloop)) {
915         BKE_report(reports,
916                    RPT_ERROR,
917                    "Invalid cage object, the cage mesh must have the same number "
918                    "of faces as the active object");
919         goto cleanup;
920       }
921     }
922     else if (is_cage) {
923       bool is_changed = false;
924
925       ModifierData *md = ob_low_eval->modifiers.first;
926       while (md) {
927         ModifierData *md_next = md->next;
928
929         /* Edge Split cannot be applied in the cage,
930          * the cage is supposed to have interpolated normals
931          * between the faces unless the geometry is physically
932          * split. So we create a copy of the low poly mesh without
933          * the eventual edge split.*/
934
935         if (md->type == eModifierType_EdgeSplit) {
936           BLI_remlink(&ob_low_eval->modifiers, md);
937           BKE_modifier_free(md);
938           is_changed = true;
939         }
940         md = md_next;
941       }
942
943       if (is_changed) {
944         /* Make sure object is evaluated with the new modifier settings.
945          *
946          * NOTE: Since the dependency graph was fully evaluated prior to bake, and we only made
947          * single modification to this object all the possible dependencies for evaluation are
948          * already up to date. This means we can do a cheap single object update
949          * (as an opposite of full depsgraph update). */
950         BKE_object_eval_reset(ob_low_eval);
951         BKE_object_handle_data_update(depsgraph, scene, ob_low_eval);
952       }
953
954       me_cage = BKE_mesh_new_from_object(NULL, ob_low_eval, false);
955       RE_bake_pixels_populate(me_cage, pixel_array_low, num_pixels, &bake_images, uv_layer);
956     }
957
958     highpoly = MEM_callocN(sizeof(BakeHighPolyData) * tot_highpoly, "bake high poly objects");
959
960     /* populate highpoly array */
961     for (link = selected_objects->first; link; link = link->next) {
962       Object *ob_iter = link->ptr.data;
963
964       if (ob_iter == ob_low) {
965         continue;
966       }
967
968       /* initialize highpoly_data */
969       highpoly[i].ob = ob_iter;
970       highpoly[i].ob_eval = DEG_get_evaluated_object(depsgraph, ob_iter);
971       highpoly[i].ob_eval->restrictflag &= ~OB_RESTRICT_RENDER;
972       highpoly[i].ob_eval->base_flag |= (BASE_VISIBLE_DEPSGRAPH | BASE_ENABLED_RENDER);
973       highpoly[i].me = BKE_mesh_new_from_object(NULL, highpoly[i].ob_eval, false);
974
975       /* lowpoly to highpoly transformation matrix */
976       copy_m4_m4(highpoly[i].obmat, highpoly[i].ob->obmat);
977       invert_m4_m4(highpoly[i].imat, highpoly[i].obmat);
978
979       highpoly[i].is_flip_object = is_negative_m4(highpoly[i].ob->obmat);
980
981       i++;
982     }
983
984     BLI_assert(i == tot_highpoly);
985
986     if (ob_cage != NULL) {
987       ob_cage_eval->restrictflag |= OB_RESTRICT_RENDER;
988       ob_cage_eval->base_flag &= ~(BASE_VISIBLE_DEPSGRAPH | BASE_ENABLED_RENDER);
989     }
990     ob_low_eval->restrictflag |= OB_RESTRICT_RENDER;
991     ob_low_eval->base_flag &= ~(BASE_VISIBLE_DEPSGRAPH | BASE_ENABLED_RENDER);
992
993     /* populate the pixel arrays with the corresponding face data for each high poly object */
994     if (!RE_bake_pixels_populate_from_objects(me_low,
995                                               pixel_array_low,
996                                               pixel_array_high,
997                                               highpoly,
998                                               tot_highpoly,
999                                               num_pixels,
1000                                               ob_cage != NULL,
1001                                               cage_extrusion,
1002                                               max_ray_distance,
1003                                               ob_low_eval->obmat,
1004                                               (ob_cage ? ob_cage->obmat : ob_low_eval->obmat),
1005                                               me_cage)) {
1006       BKE_report(reports, RPT_ERROR, "Error handling selected objects");
1007       goto cleanup;
1008     }
1009
1010     /* the baking itself */
1011     for (i = 0; i < tot_highpoly; i++) {
1012       ok = RE_bake_engine(re,
1013                           depsgraph,
1014                           highpoly[i].ob,
1015                           i,
1016                           pixel_array_high,
1017                           &bake_images,
1018                           depth,
1019                           pass_type,
1020                           pass_filter,
1021                           result);
1022       if (!ok) {
1023         BKE_reportf(
1024             reports, RPT_ERROR, "Error baking from object \"%s\"", highpoly[i].ob->id.name + 2);
1025         goto cleanup;
1026       }
1027     }
1028   }
1029   else {
1030     /* If low poly is not renderable it should have failed long ago. */
1031     BLI_assert((ob_low_eval->restrictflag & OB_RESTRICT_RENDER) == 0);
1032
1033     if (RE_bake_has_engine(re)) {
1034       ok = RE_bake_engine(re,
1035                           depsgraph,
1036                           ob_low_eval,
1037                           0,
1038                           pixel_array_low,
1039                           &bake_images,
1040                           depth,
1041                           pass_type,
1042                           pass_filter,
1043                           result);
1044     }
1045     else {
1046       BKE_report(reports, RPT_ERROR, "Current render engine does not support baking");
1047       goto cleanup;
1048     }
1049   }
1050
1051   /* normal space conversion
1052    * the normals are expected to be in world space, +X +Y +Z */
1053   if (ok && pass_type == SCE_PASS_NORMAL) {
1054     switch (normal_space) {
1055       case R_BAKE_SPACE_WORLD: {
1056         /* Cycles internal format */
1057         if ((normal_swizzle[0] == R_BAKE_POSX) && (normal_swizzle[1] == R_BAKE_POSY) &&
1058             (normal_swizzle[2] == R_BAKE_POSZ)) {
1059           break;
1060         }
1061         RE_bake_normal_world_to_world(pixel_array_low, num_pixels, depth, result, normal_swizzle);
1062         break;
1063       }
1064       case R_BAKE_SPACE_OBJECT: {
1065         RE_bake_normal_world_to_object(
1066             pixel_array_low, num_pixels, depth, result, ob_low_eval, normal_swizzle);
1067         break;
1068       }
1069       case R_BAKE_SPACE_TANGENT: {
1070         if (is_selected_to_active) {
1071           RE_bake_normal_world_to_tangent(pixel_array_low,
1072                                           num_pixels,
1073                                           depth,
1074                                           result,
1075                                           me_low,
1076                                           normal_swizzle,
1077                                           ob_low_eval->obmat);
1078         }
1079         else {
1080           /* from multiresolution */
1081           Mesh *me_nores = NULL;
1082           ModifierData *md = NULL;
1083           int mode;
1084
1085           BKE_object_eval_reset(ob_low_eval);
1086           md = BKE_modifiers_findby_type(ob_low_eval, eModifierType_Multires);
1087
1088           if (md) {
1089             mode = md->mode;
1090             md->mode &= ~eModifierMode_Render;
1091           }
1092
1093           /* Evaluate modifiers again. */
1094           me_nores = BKE_mesh_new_from_object(NULL, ob_low_eval, false);
1095           RE_bake_pixels_populate(me_nores, pixel_array_low, num_pixels, &bake_images, uv_layer);
1096
1097           RE_bake_normal_world_to_tangent(pixel_array_low,
1098                                           num_pixels,
1099                                           depth,
1100                                           result,
1101                                           me_nores,
1102                                           normal_swizzle,
1103                                           ob_low_eval->obmat);
1104           BKE_id_free(NULL, &me_nores->id);
1105
1106           if (md) {
1107             md->mode = mode;
1108           }
1109         }
1110         break;
1111       }
1112       default:
1113         break;
1114     }
1115   }
1116
1117   if (!ok) {
1118     BKE_reportf(reports, RPT_ERROR, "Problem baking object \"%s\"", ob_low->id.name + 2);
1119     op_result = OPERATOR_CANCELLED;
1120   }
1121   else {
1122     /* save the results */
1123     for (int i = 0; i < bake_images.size; i++) {
1124       BakeImage *bk_image = &bake_images.data[i];
1125
1126       if (is_save_internal) {
1127         ok = write_internal_bake_pixels(bk_image->image,
1128                                         pixel_array_low + bk_image->offset,
1129                                         result + bk_image->offset * depth,
1130                                         bk_image->width,
1131                                         bk_image->height,
1132                                         margin,
1133                                         is_clear,
1134                                         is_noncolor);
1135
1136         /* might be read by UI to set active image for display */
1137         bake_update_image(area, bk_image->image);
1138
1139         if (!ok) {
1140           BKE_reportf(reports,
1141                       RPT_ERROR,
1142                       "Problem saving the bake map internally for object \"%s\"",
1143                       ob_low->id.name + 2);
1144           op_result = OPERATOR_CANCELLED;
1145         }
1146         else {
1147           BKE_report(reports,
1148                      RPT_INFO,
1149                      "Baking map saved to internal image, save it externally or pack it");
1150           op_result = OPERATOR_FINISHED;
1151         }
1152       }
1153       /* save externally */
1154       else {
1155         BakeData *bake = &scene->r.bake;
1156         char name[FILE_MAX];
1157
1158         BKE_image_path_from_imtype(name,
1159                                    filepath,
1160                                    BKE_main_blendfile_path(bmain),
1161                                    0,
1162                                    bake->im_format.imtype,
1163                                    true,
1164                                    false,
1165                                    NULL);
1166
1167         if (is_automatic_name) {
1168           BLI_path_suffix(name, FILE_MAX, ob_low->id.name + 2, "_");
1169           BLI_path_suffix(name, FILE_MAX, identifier, "_");
1170         }
1171
1172         if (is_split_materials) {
1173           if (bk_image->image) {
1174             BLI_path_suffix(name, FILE_MAX, bk_image->image->id.name + 2, "_");
1175           }
1176           else {
1177             if (ob_low_eval->mat[i]) {
1178               BLI_path_suffix(name, FILE_MAX, ob_low_eval->mat[i]->id.name + 2, "_");
1179             }
1180             else if (me_low->mat[i]) {
1181               BLI_path_suffix(name, FILE_MAX, me_low->mat[i]->id.name + 2, "_");
1182             }
1183             else {
1184               /* if everything else fails, use the material index */
1185               char tmp[5];
1186               sprintf(tmp, "%d", i % 1000);
1187               BLI_path_suffix(name, FILE_MAX, tmp, "_");
1188             }
1189           }
1190         }
1191
1192         /* save it externally */
1193         ok = write_external_bake_pixels(name,
1194                                         pixel_array_low + bk_image->offset,
1195                                         result + bk_image->offset * depth,
1196                                         bk_image->width,
1197                                         bk_image->height,
1198                                         margin,
1199                                         &bake->im_format,
1200                                         is_noncolor);
1201
1202         if (!ok) {
1203           BKE_reportf(reports, RPT_ERROR, "Problem saving baked map in \"%s\"", name);
1204           op_result = OPERATOR_CANCELLED;
1205         }
1206         else {
1207           BKE_reportf(reports, RPT_INFO, "Baking map written to \"%s\"", name);
1208           op_result = OPERATOR_FINISHED;
1209         }
1210
1211         if (!is_split_materials) {
1212           break;
1213         }
1214       }
1215     }
1216   }
1217
1218   if (is_save_internal) {
1219     refresh_images(&bake_images);
1220   }
1221
1222 cleanup:
1223
1224   if (highpoly) {
1225     for (int i = 0; i < tot_highpoly; i++) {
1226       if (highpoly[i].me != NULL) {
1227         BKE_id_free(NULL, &highpoly[i].me->id);
1228       }
1229     }
1230     MEM_freeN(highpoly);
1231   }
1232
1233   if (mmd_low) {
1234     mmd_low->flags = mmd_flags_low;
1235   }
1236
1237   if (pixel_array_low) {
1238     MEM_freeN(pixel_array_low);
1239   }
1240
1241   if (pixel_array_high) {
1242     MEM_freeN(pixel_array_high);
1243   }
1244
1245   if (bake_images.data) {
1246     MEM_freeN(bake_images.data);
1247   }
1248
1249   if (bake_images.lookup) {
1250     MEM_freeN(bake_images.lookup);
1251   }
1252
1253   if (result) {
1254     MEM_freeN(result);
1255   }
1256
1257   if (me_low != NULL) {
1258     BKE_id_free(NULL, &me_low->id);
1259   }
1260
1261   if (me_cage != NULL) {
1262     BKE_id_free(NULL, &me_cage->id);
1263   }
1264
1265   DEG_graph_free(depsgraph);
1266
1267   return op_result;
1268 }
1269
1270 static void bake_init_api_data(wmOperator *op, bContext *C, BakeAPIRender *bkr)
1271 {
1272   bool is_save_internal;
1273   bScreen *screen = CTX_wm_screen(C);
1274
1275   bkr->ob = CTX_data_active_object(C);
1276   bkr->main = CTX_data_main(C);
1277   bkr->view_layer = CTX_data_view_layer(C);
1278   bkr->scene = CTX_data_scene(C);
1279   bkr->area = screen ? BKE_screen_find_big_area(screen, SPACE_IMAGE, 10) : NULL;
1280
1281   bkr->pass_type = RNA_enum_get(op->ptr, "type");
1282   bkr->pass_filter = RNA_enum_get(op->ptr, "pass_filter");
1283   bkr->margin = RNA_int_get(op->ptr, "margin");
1284
1285   bkr->save_mode = RNA_enum_get(op->ptr, "save_mode");
1286   is_save_internal = (bkr->save_mode == R_BAKE_SAVE_INTERNAL);
1287
1288   bkr->is_clear = RNA_boolean_get(op->ptr, "use_clear");
1289   bkr->is_split_materials = (!is_save_internal) && RNA_boolean_get(op->ptr, "use_split_materials");
1290   bkr->is_automatic_name = RNA_boolean_get(op->ptr, "use_automatic_name");
1291   bkr->is_selected_to_active = RNA_boolean_get(op->ptr, "use_selected_to_active");
1292   bkr->is_cage = RNA_boolean_get(op->ptr, "use_cage");
1293   bkr->cage_extrusion = RNA_float_get(op->ptr, "cage_extrusion");
1294   bkr->max_ray_distance = RNA_float_get(op->ptr, "max_ray_distance");
1295
1296   bkr->normal_space = RNA_enum_get(op->ptr, "normal_space");
1297   bkr->normal_swizzle[0] = RNA_enum_get(op->ptr, "normal_r");
1298   bkr->normal_swizzle[1] = RNA_enum_get(op->ptr, "normal_g");
1299   bkr->normal_swizzle[2] = RNA_enum_get(op->ptr, "normal_b");
1300
1301   bkr->width = RNA_int_get(op->ptr, "width");
1302   bkr->height = RNA_int_get(op->ptr, "height");
1303   bkr->identifier = "";
1304
1305   RNA_string_get(op->ptr, "uv_layer", bkr->uv_layer);
1306
1307   RNA_string_get(op->ptr, "cage_object", bkr->custom_cage);
1308
1309   if ((!is_save_internal) && bkr->is_automatic_name) {
1310     PropertyRNA *prop = RNA_struct_find_property(op->ptr, "type");
1311     RNA_property_enum_identifier(C, op->ptr, prop, bkr->pass_type, &bkr->identifier);
1312   }
1313
1314   CTX_data_selected_objects(C, &bkr->selected_objects);
1315
1316   bkr->reports = op->reports;
1317
1318   bkr->result = OPERATOR_CANCELLED;
1319
1320   bkr->render = RE_NewSceneRender(bkr->scene);
1321
1322   /* XXX hack to force saving to always be internal. Whether (and how) to support
1323    * external saving will be addressed later */
1324   bkr->save_mode = R_BAKE_SAVE_INTERNAL;
1325 }
1326
1327 static int bake_exec(bContext *C, wmOperator *op)
1328 {
1329   Render *re;
1330   int result = OPERATOR_CANCELLED;
1331   BakeAPIRender bkr = {NULL};
1332   Scene *scene = CTX_data_scene(C);
1333
1334   G.is_break = false;
1335   G.is_rendering = true;
1336
1337   bake_set_props(op, scene);
1338
1339   bake_init_api_data(op, C, &bkr);
1340   re = bkr.render;
1341
1342   /* setup new render */
1343   RE_test_break_cb(re, NULL, bake_break);
1344
1345   if (!bake_pass_filter_check(bkr.pass_type, bkr.pass_filter, bkr.reports)) {
1346     goto finally;
1347   }
1348
1349   if (!bake_objects_check(bkr.main,
1350                           bkr.view_layer,
1351                           bkr.ob,
1352                           &bkr.selected_objects,
1353                           bkr.reports,
1354                           bkr.is_selected_to_active)) {
1355     goto finally;
1356   }
1357
1358   if (bkr.is_clear) {
1359     const bool is_tangent = ((bkr.pass_type == SCE_PASS_NORMAL) &&
1360                              (bkr.normal_space == R_BAKE_SPACE_TANGENT));
1361     bake_images_clear(bkr.main, is_tangent);
1362   }
1363
1364   RE_SetReports(re, bkr.reports);
1365
1366   if (bkr.is_selected_to_active) {
1367     result = bake(bkr.render,
1368                   bkr.main,
1369                   bkr.scene,
1370                   bkr.view_layer,
1371                   bkr.ob,
1372                   &bkr.selected_objects,
1373                   bkr.reports,
1374                   bkr.pass_type,
1375                   bkr.pass_filter,
1376                   bkr.margin,
1377                   bkr.save_mode,
1378                   bkr.is_clear,
1379                   bkr.is_split_materials,
1380                   bkr.is_automatic_name,
1381                   true,
1382                   bkr.is_cage,
1383                   bkr.cage_extrusion,
1384                   bkr.max_ray_distance,
1385                   bkr.normal_space,
1386                   bkr.normal_swizzle,
1387                   bkr.custom_cage,
1388                   bkr.filepath,
1389                   bkr.width,
1390                   bkr.height,
1391                   bkr.identifier,
1392                   bkr.area,
1393                   bkr.uv_layer);
1394   }
1395   else {
1396     CollectionPointerLink *link;
1397     const bool is_clear = bkr.is_clear && BLI_listbase_is_single(&bkr.selected_objects);
1398     for (link = bkr.selected_objects.first; link; link = link->next) {
1399       Object *ob_iter = link->ptr.data;
1400       result = bake(bkr.render,
1401                     bkr.main,
1402                     bkr.scene,
1403                     bkr.view_layer,
1404                     ob_iter,
1405                     NULL,
1406                     bkr.reports,
1407                     bkr.pass_type,
1408                     bkr.pass_filter,
1409                     bkr.margin,
1410                     bkr.save_mode,
1411                     is_clear,
1412                     bkr.is_split_materials,
1413                     bkr.is_automatic_name,
1414                     false,
1415                     bkr.is_cage,
1416                     bkr.cage_extrusion,
1417                     bkr.max_ray_distance,
1418                     bkr.normal_space,
1419                     bkr.normal_swizzle,
1420                     bkr.custom_cage,
1421                     bkr.filepath,
1422                     bkr.width,
1423                     bkr.height,
1424                     bkr.identifier,
1425                     bkr.area,
1426                     bkr.uv_layer);
1427     }
1428   }
1429
1430   RE_SetReports(re, NULL);
1431
1432 finally:
1433   G.is_rendering = false;
1434   BLI_freelistN(&bkr.selected_objects);
1435   return result;
1436 }
1437
1438 static void bake_startjob(void *bkv, short *UNUSED(stop), short *do_update, float *progress)
1439 {
1440   BakeAPIRender *bkr = (BakeAPIRender *)bkv;
1441
1442   /* setup new render */
1443   bkr->do_update = do_update;
1444   bkr->progress = progress;
1445
1446   RE_SetReports(bkr->render, bkr->reports);
1447
1448   if (!bake_pass_filter_check(bkr->pass_type, bkr->pass_filter, bkr->reports)) {
1449     bkr->result = OPERATOR_CANCELLED;
1450     return;
1451   }
1452
1453   if (!bake_objects_check(bkr->main,
1454                           bkr->view_layer,
1455                           bkr->ob,
1456                           &bkr->selected_objects,
1457                           bkr->reports,
1458                           bkr->is_selected_to_active)) {
1459     bkr->result = OPERATOR_CANCELLED;
1460     return;
1461   }
1462
1463   if (bkr->is_clear) {
1464     const bool is_tangent = ((bkr->pass_type == SCE_PASS_NORMAL) &&
1465                              (bkr->normal_space == R_BAKE_SPACE_TANGENT));
1466     bake_images_clear(bkr->main, is_tangent);
1467   }
1468
1469   if (bkr->is_selected_to_active) {
1470     bkr->result = bake(bkr->render,
1471                        bkr->main,
1472                        bkr->scene,
1473                        bkr->view_layer,
1474                        bkr->ob,
1475                        &bkr->selected_objects,
1476                        bkr->reports,
1477                        bkr->pass_type,
1478                        bkr->pass_filter,
1479                        bkr->margin,
1480                        bkr->save_mode,
1481                        bkr->is_clear,
1482                        bkr->is_split_materials,
1483                        bkr->is_automatic_name,
1484                        true,
1485                        bkr->is_cage,
1486                        bkr->cage_extrusion,
1487                        bkr->max_ray_distance,
1488                        bkr->normal_space,
1489                        bkr->normal_swizzle,
1490                        bkr->custom_cage,
1491                        bkr->filepath,
1492                        bkr->width,
1493                        bkr->height,
1494                        bkr->identifier,
1495                        bkr->area,
1496                        bkr->uv_layer);
1497   }
1498   else {
1499     CollectionPointerLink *link;
1500     const bool is_clear = bkr->is_clear && BLI_listbase_is_single(&bkr->selected_objects);
1501     for (link = bkr->selected_objects.first; link; link = link->next) {
1502       Object *ob_iter = link->ptr.data;
1503       bkr->result = bake(bkr->render,
1504                          bkr->main,
1505                          bkr->scene,
1506                          bkr->view_layer,
1507                          ob_iter,
1508                          NULL,
1509                          bkr->reports,
1510                          bkr->pass_type,
1511                          bkr->pass_filter,
1512                          bkr->margin,
1513                          bkr->save_mode,
1514                          is_clear,
1515                          bkr->is_split_materials,
1516                          bkr->is_automatic_name,
1517                          false,
1518                          bkr->is_cage,
1519                          bkr->cage_extrusion,
1520                          bkr->max_ray_distance,
1521                          bkr->normal_space,
1522                          bkr->normal_swizzle,
1523                          bkr->custom_cage,
1524                          bkr->filepath,
1525                          bkr->width,
1526                          bkr->height,
1527                          bkr->identifier,
1528                          bkr->area,
1529                          bkr->uv_layer);
1530
1531       if (bkr->result == OPERATOR_CANCELLED) {
1532         return;
1533       }
1534     }
1535   }
1536
1537   RE_SetReports(bkr->render, NULL);
1538 }
1539
1540 static void bake_freejob(void *bkv)
1541 {
1542   BakeAPIRender *bkr = (BakeAPIRender *)bkv;
1543
1544   BLI_freelistN(&bkr->selected_objects);
1545   MEM_freeN(bkr);
1546
1547   G.is_rendering = false;
1548 }
1549
1550 static void bake_set_props(wmOperator *op, Scene *scene)
1551 {
1552   PropertyRNA *prop;
1553   BakeData *bake = &scene->r.bake;
1554
1555   prop = RNA_struct_find_property(op->ptr, "filepath");
1556   if (!RNA_property_is_set(op->ptr, prop)) {
1557     RNA_property_string_set(op->ptr, prop, bake->filepath);
1558   }
1559
1560   prop = RNA_struct_find_property(op->ptr, "width");
1561   if (!RNA_property_is_set(op->ptr, prop)) {
1562     RNA_property_int_set(op->ptr, prop, bake->width);
1563   }
1564
1565   prop = RNA_struct_find_property(op->ptr, "height");
1566   if (!RNA_property_is_set(op->ptr, prop)) {
1567     RNA_property_int_set(op->ptr, prop, bake->width);
1568   }
1569
1570   prop = RNA_struct_find_property(op->ptr, "margin");
1571   if (!RNA_property_is_set(op->ptr, prop)) {
1572     RNA_property_int_set(op->ptr, prop, bake->margin);
1573   }
1574
1575   prop = RNA_struct_find_property(op->ptr, "use_selected_to_active");
1576   if (!RNA_property_is_set(op->ptr, prop)) {
1577     RNA_property_boolean_set(op->ptr, prop, (bake->flag & R_BAKE_TO_ACTIVE) != 0);
1578   }
1579
1580   prop = RNA_struct_find_property(op->ptr, "max_ray_distance");
1581   if (!RNA_property_is_set(op->ptr, prop)) {
1582     RNA_property_float_set(op->ptr, prop, bake->max_ray_distance);
1583   }
1584
1585   prop = RNA_struct_find_property(op->ptr, "cage_extrusion");
1586   if (!RNA_property_is_set(op->ptr, prop)) {
1587     RNA_property_float_set(op->ptr, prop, bake->cage_extrusion);
1588   }
1589
1590   prop = RNA_struct_find_property(op->ptr, "cage_object");
1591   if (!RNA_property_is_set(op->ptr, prop)) {
1592     RNA_property_string_set(
1593         op->ptr, prop, (bake->cage_object) ? bake->cage_object->id.name + 2 : "");
1594   }
1595
1596   prop = RNA_struct_find_property(op->ptr, "normal_space");
1597   if (!RNA_property_is_set(op->ptr, prop)) {
1598     RNA_property_enum_set(op->ptr, prop, bake->normal_space);
1599   }
1600
1601   prop = RNA_struct_find_property(op->ptr, "normal_r");
1602   if (!RNA_property_is_set(op->ptr, prop)) {
1603     RNA_property_enum_set(op->ptr, prop, bake->normal_swizzle[0]);
1604   }
1605
1606   prop = RNA_struct_find_property(op->ptr, "normal_g");
1607   if (!RNA_property_is_set(op->ptr, prop)) {
1608     RNA_property_enum_set(op->ptr, prop, bake->normal_swizzle[1]);
1609   }
1610
1611   prop = RNA_struct_find_property(op->ptr, "normal_b");
1612   if (!RNA_property_is_set(op->ptr, prop)) {
1613     RNA_property_enum_set(op->ptr, prop, bake->normal_swizzle[2]);
1614   }
1615
1616   prop = RNA_struct_find_property(op->ptr, "save_mode");
1617   if (!RNA_property_is_set(op->ptr, prop)) {
1618     RNA_property_enum_set(op->ptr, prop, bake->save_mode);
1619   }
1620
1621   prop = RNA_struct_find_property(op->ptr, "use_clear");
1622   if (!RNA_property_is_set(op->ptr, prop)) {
1623     RNA_property_boolean_set(op->ptr, prop, (bake->flag & R_BAKE_CLEAR) != 0);
1624   }
1625
1626   prop = RNA_struct_find_property(op->ptr, "use_cage");
1627   if (!RNA_property_is_set(op->ptr, prop)) {
1628     RNA_property_boolean_set(op->ptr, prop, (bake->flag & R_BAKE_CAGE) != 0);
1629   }
1630
1631   prop = RNA_struct_find_property(op->ptr, "use_split_materials");
1632   if (!RNA_property_is_set(op->ptr, prop)) {
1633     RNA_property_boolean_set(op->ptr, prop, (bake->flag & R_BAKE_SPLIT_MAT) != 0);
1634   }
1635
1636   prop = RNA_struct_find_property(op->ptr, "use_automatic_name");
1637   if (!RNA_property_is_set(op->ptr, prop)) {
1638     RNA_property_boolean_set(op->ptr, prop, (bake->flag & R_BAKE_AUTO_NAME) != 0);
1639   }
1640
1641   prop = RNA_struct_find_property(op->ptr, "pass_filter");
1642   if (!RNA_property_is_set(op->ptr, prop)) {
1643     RNA_property_enum_set(op->ptr, prop, bake->pass_filter);
1644   }
1645 }
1646
1647 static int bake_invoke(bContext *C, wmOperator *op, const wmEvent *UNUSED(event))
1648 {
1649   wmJob *wm_job;
1650   BakeAPIRender *bkr;
1651   Render *re;
1652   Scene *scene = CTX_data_scene(C);
1653
1654   bake_set_props(op, scene);
1655
1656   /* only one render job at a time */
1657   if (WM_jobs_test(CTX_wm_manager(C), scene, WM_JOB_TYPE_OBJECT_BAKE)) {
1658     return OPERATOR_CANCELLED;
1659   }
1660
1661   bkr = MEM_mallocN(sizeof(BakeAPIRender), "render bake");
1662
1663   /* init bake render */
1664   bake_init_api_data(op, C, bkr);
1665   re = bkr->render;
1666
1667   /* setup new render */
1668   RE_test_break_cb(re, NULL, bake_break);
1669   RE_progress_cb(re, bkr, bake_progress_update);
1670
1671   /* setup job */
1672   wm_job = WM_jobs_get(CTX_wm_manager(C),
1673                        CTX_wm_window(C),
1674                        scene,
1675                        "Texture Bake",
1676                        WM_JOB_EXCL_RENDER | WM_JOB_PRIORITY | WM_JOB_PROGRESS,
1677                        WM_JOB_TYPE_OBJECT_BAKE);
1678   WM_jobs_customdata_set(wm_job, bkr, bake_freejob);
1679   WM_jobs_timer(wm_job, 0.5, NC_IMAGE, 0); /* TODO - only draw bake image, can we enforce this */
1680   WM_jobs_callbacks(wm_job, bake_startjob, NULL, NULL, NULL);
1681
1682   G.is_break = false;
1683   G.is_rendering = true;
1684
1685   WM_jobs_start(CTX_wm_manager(C), wm_job);
1686
1687   WM_cursor_wait(0);
1688
1689   /* add modal handler for ESC */
1690   WM_event_add_modal_handler(C, op);
1691
1692   WM_event_add_notifier(C, NC_SCENE | ND_RENDER_RESULT, scene);
1693   return OPERATOR_RUNNING_MODAL;
1694 }
1695
1696 void OBJECT_OT_bake(wmOperatorType *ot)
1697 {
1698   PropertyRNA *prop;
1699
1700   /* identifiers */
1701   ot->name = "Bake";
1702   ot->description = "Bake image textures of selected objects";
1703   ot->idname = "OBJECT_OT_bake";
1704
1705   /* api callbacks */
1706   ot->exec = bake_exec;
1707   ot->modal = bake_modal;
1708   ot->invoke = bake_invoke;
1709   ot->poll = ED_operator_object_active_editable_mesh;
1710
1711   RNA_def_enum(
1712       ot->srna,
1713       "type",
1714       rna_enum_bake_pass_type_items,
1715       SCE_PASS_COMBINED,
1716       "Type",
1717       "Type of pass to bake, some of them may not be supported by the current render engine");
1718   prop = RNA_def_enum(ot->srna,
1719                       "pass_filter",
1720                       rna_enum_bake_pass_filter_type_items,
1721                       R_BAKE_PASS_FILTER_NONE,
1722                       "Pass Filter",
1723                       "Filter to combined, diffuse, glossy, transmission and subsurface passes");
1724   RNA_def_property_flag(prop, PROP_ENUM_FLAG);
1725   RNA_def_string_file_path(ot->srna,
1726                            "filepath",
1727                            NULL,
1728                            FILE_MAX,
1729                            "File Path",
1730                            "Image filepath to use when saving externally");
1731   RNA_def_int(ot->srna,
1732               "width",
1733               512,
1734               1,
1735               INT_MAX,
1736               "Width",
1737               "Horizontal dimension of the baking map (external only)",
1738               64,
1739               4096);
1740   RNA_def_int(ot->srna,
1741               "height",
1742               512,
1743               1,
1744               INT_MAX,
1745               "Height",
1746               "Vertical dimension of the baking map (external only)",
1747               64,
1748               4096);
1749   RNA_def_int(ot->srna,
1750               "margin",
1751               16,
1752               0,
1753               INT_MAX,
1754               "Margin",
1755               "Extends the baked result as a post process filter",
1756               0,
1757               64);
1758   RNA_def_boolean(ot->srna,
1759                   "use_selected_to_active",
1760                   false,
1761                   "Selected to Active",
1762                   "Bake shading on the surface of selected objects to the active object");
1763   RNA_def_float(ot->srna,
1764                 "max_ray_distance",
1765                 0.0f,
1766                 0.0f,
1767                 FLT_MAX,
1768                 "Max Ray Distance",
1769                 "The maximum ray distance for matching points between the active and selected "
1770                 "objects. If zero, there is no limit",
1771                 0.0f,
1772                 1.0f);
1773   RNA_def_float(ot->srna,
1774                 "cage_extrusion",
1775                 0.0f,
1776                 0.0f,
1777                 FLT_MAX,
1778                 "Cage Extrusion",
1779                 "Inflate the active object by the specified distance for baking. This helps "
1780                 "matching to points nearer to the outside of the selected object meshes",
1781                 0.0f,
1782                 1.0f);
1783   RNA_def_string(ot->srna,
1784                  "cage_object",
1785                  NULL,
1786                  MAX_NAME,
1787                  "Cage Object",
1788                  "Object to use as cage, instead of calculating the cage from the active object "
1789                  "with cage extrusion");
1790   RNA_def_enum(ot->srna,
1791                "normal_space",
1792                rna_enum_normal_space_items,
1793                R_BAKE_SPACE_TANGENT,
1794                "Normal Space",
1795                "Choose normal space for baking");
1796   RNA_def_enum(ot->srna,
1797                "normal_r",
1798                rna_enum_normal_swizzle_items,
1799                R_BAKE_POSX,
1800                "R",
1801                "Axis to bake in red channel");
1802   RNA_def_enum(ot->srna,
1803                "normal_g",
1804                rna_enum_normal_swizzle_items,
1805                R_BAKE_POSY,
1806                "G",
1807                "Axis to bake in green channel");
1808   RNA_def_enum(ot->srna,
1809                "normal_b",
1810                rna_enum_normal_swizzle_items,
1811                R_BAKE_POSZ,
1812                "B",
1813                "Axis to bake in blue channel");
1814   RNA_def_enum(ot->srna,
1815                "save_mode",
1816                rna_enum_bake_save_mode_items,
1817                R_BAKE_SAVE_INTERNAL,
1818                "Save Mode",
1819                "Choose how to save the baking map");
1820   RNA_def_boolean(ot->srna,
1821                   "use_clear",
1822                   false,
1823                   "Clear",
1824                   "Clear Images before baking (only for internal saving)");
1825   RNA_def_boolean(ot->srna, "use_cage", false, "Cage", "Cast rays to active object from a cage");
1826   RNA_def_boolean(
1827       ot->srna,
1828       "use_split_materials",
1829       false,
1830       "Split Materials",
1831       "Split baked maps per material, using material name in output file (external only)");
1832   RNA_def_boolean(ot->srna,
1833                   "use_automatic_name",
1834                   false,
1835                   "Automatic Name",
1836                   "Automatically name the output file with the pass type");
1837   RNA_def_string(ot->srna,
1838                  "uv_layer",
1839                  NULL,
1840                  MAX_CUSTOMDATA_LAYER_NAME,
1841                  "UV Layer",
1842                  "UV layer to override active");
1843 }