Fix #35561: freestyle + read full sample layers = crash.
[blender.git] / source / blender / freestyle / intern / blender_interface / BlenderStrokeRenderer.cpp
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * ***** END GPL LICENSE BLOCK *****
19  */
20
21 /** \file blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp
22  *  \ingroup freestyle
23  */
24
25 #include "BlenderStrokeRenderer.h"
26 #include "BlenderTextureManager.h"
27
28 #include "../application/AppConfig.h"
29 #include "../stroke/Canvas.h"
30
31 #include "BKE_global.h"
32
33 extern "C" {
34 #include "MEM_guardedalloc.h"
35
36 #include "DNA_camera_types.h"
37 #include "DNA_customdata_types.h"
38 #include "DNA_listBase.h"
39 #include "DNA_meshdata_types.h"
40 #include "DNA_mesh_types.h"
41 #include "DNA_object_types.h"
42 #include "DNA_screen_types.h"
43
44 #include "BKE_customdata.h"
45 #include "BKE_global.h"
46 #include "BKE_library.h" /* free_libblock */
47 #include "BKE_main.h" /* struct Main */
48 #include "BKE_material.h"
49 #include "BKE_mesh.h"
50 #include "BKE_object.h"
51 #include "BKE_scene.h"
52
53 #include "RE_pipeline.h"
54 }
55
56 namespace Freestyle {
57
58 BlenderStrokeRenderer::BlenderStrokeRenderer(Render *re, int render_count) : StrokeRenderer()
59 {
60         freestyle_bmain = &re->freestyle_bmain;
61
62         // TEMPORARY - need a  texture manager
63         _textureManager = new BlenderTextureManager;
64         _textureManager->load();
65
66         // for stroke mesh generation
67         _width = re->winx;
68         _height = re->winy;
69
70         old_scene = re->scene;
71
72         char name[MAX_ID_NAME - 2];
73         BLI_snprintf(name, sizeof(name), "FRS%d_%s", render_count, re->scene->id.name + 2);
74         freestyle_scene = BKE_scene_add(freestyle_bmain, name);
75         freestyle_scene->r.cfra = old_scene->r.cfra;
76         freestyle_scene->r.mode = old_scene->r.mode &
77                                   ~(R_EDGE_FRS | R_SHADOW | R_SSS | R_PANORAMA | R_ENVMAP | R_MBLUR | R_BORDER);
78         freestyle_scene->r.xsch = re->rectx; // old_scene->r.xsch
79         freestyle_scene->r.ysch = re->recty; // old_scene->r.ysch
80         freestyle_scene->r.xasp = 1.0f; // old_scene->r.xasp;
81         freestyle_scene->r.yasp = 1.0f; // old_scene->r.yasp;
82         freestyle_scene->r.tilex = old_scene->r.tilex;
83         freestyle_scene->r.tiley = old_scene->r.tiley;
84         freestyle_scene->r.size = 100; // old_scene->r.size
85         //freestyle_scene->r.maximsize = old_scene->r.maximsize; /* DEPRECATED */
86         freestyle_scene->r.ocres = old_scene->r.ocres;
87         freestyle_scene->r.color_mgt_flag = 0; // old_scene->r.color_mgt_flag;
88         freestyle_scene->r.scemode = old_scene->r.scemode & ~(R_SINGLE_LAYER);
89         freestyle_scene->r.flag = old_scene->r.flag;
90         freestyle_scene->r.threads = old_scene->r.threads;
91         freestyle_scene->r.border.xmin = old_scene->r.border.xmin;
92         freestyle_scene->r.border.ymin = old_scene->r.border.ymin;
93         freestyle_scene->r.border.xmax = old_scene->r.border.xmax;
94         freestyle_scene->r.border.ymax = old_scene->r.border.ymax;
95         strcpy(freestyle_scene->r.pic, old_scene->r.pic);
96         freestyle_scene->r.safety.xmin = old_scene->r.safety.xmin;
97         freestyle_scene->r.safety.ymin = old_scene->r.safety.ymin;
98         freestyle_scene->r.safety.xmax = old_scene->r.safety.xmax;
99         freestyle_scene->r.safety.ymax = old_scene->r.safety.ymax;
100         freestyle_scene->r.osa = old_scene->r.osa;
101         freestyle_scene->r.filtertype = old_scene->r.filtertype;
102         freestyle_scene->r.gauss = old_scene->r.gauss;
103         freestyle_scene->r.dither_intensity = old_scene->r.dither_intensity;
104         BLI_strncpy(freestyle_scene->r.engine, old_scene->r.engine, sizeof(freestyle_scene->r.engine));
105         freestyle_scene->r.im_format.planes = R_IMF_PLANES_RGBA; 
106         freestyle_scene->r.im_format.imtype = R_IMF_IMTYPE_PNG;
107         BKE_scene_disable_color_management(freestyle_scene);
108
109         if (G.debug & G_DEBUG_FREESTYLE) {
110                 printf("%s: %d threads\n", __func__, freestyle_scene->r.threads);
111         }
112
113         // Render layer
114         SceneRenderLayer *srl = (SceneRenderLayer *)freestyle_scene->r.layers.first;
115         srl->layflag = SCE_LAY_SOLID | SCE_LAY_ZTRA;
116
117         BKE_scene_set_background(freestyle_bmain, freestyle_scene);
118
119         // Camera
120         Object *object_camera = BKE_object_add(freestyle_bmain, freestyle_scene, OB_CAMERA);
121
122         Camera *camera = (Camera *)object_camera->data;
123         camera->type = CAM_ORTHO;
124         camera->ortho_scale = max(re->rectx, re->recty);
125         camera->clipsta = 0.1f;
126         camera->clipend = 100.0f;
127
128         _z_delta = 0.00001f;
129         _z = camera->clipsta + _z_delta;
130
131         object_camera->loc[0] = re->disprect.xmin + 0.5f * re->rectx;
132         object_camera->loc[1] = re->disprect.ymin + 0.5f * re->recty;
133         object_camera->loc[2] = 1.0f;
134
135         freestyle_scene->camera = object_camera;
136
137         // Material
138         material = BKE_material_add(freestyle_bmain, "stroke_material");
139         material->mode |= MA_VERTEXCOLP;
140         material->mode |= MA_TRANSP;
141         material->mode |= MA_SHLESS;
142         material->vcol_alpha = 1;
143
144         // Reset serial mesh ID (used for BlenderStrokeRenderer::NewMesh())
145         _mesh_id = 0xffffffff;
146 }
147
148 BlenderStrokeRenderer::~BlenderStrokeRenderer()
149 {
150         if (0 != _textureManager) {
151                 delete _textureManager;
152                 _textureManager = NULL;
153         }
154
155         // The freestyle_scene object is not released here.  Instead,
156         // the scene is released in free_all_freestyle_renders() in
157         // source/blender/render/intern/source/pipeline.c, after the
158         // compositor has finished.
159
160         // release objects and data blocks
161         for (Base *b = (Base *)freestyle_scene->base.first; b; b = b->next) {
162                 Object *ob = b->object;
163                 void *data = ob->data;
164                 char *name = ob->id.name;
165 #if 0
166                 if (G.debug & G_DEBUG_FREESTYLE) {
167                         cout << "removing " << name[0] << name[1] << ":" << (name+2) << endl;
168                 }
169 #endif
170                 switch (ob->type) {
171                 case OB_MESH:
172                         BKE_libblock_free(&freestyle_bmain->object, ob);
173                         BKE_libblock_free(&freestyle_bmain->mesh, data);
174                         break;
175                 case OB_CAMERA:
176                         BKE_libblock_free(&freestyle_bmain->object, ob);
177                         BKE_libblock_free(&freestyle_bmain->camera, data);
178                         freestyle_scene->camera = NULL;
179                         break;
180                 default:
181                         cerr << "Warning: unexpected object in the scene: " << name[0] << name[1] << ":" << (name + 2) << endl;
182                 }
183         }
184         BLI_freelistN(&freestyle_scene->base);
185
186         // release material
187         BKE_libblock_free(&freestyle_bmain->mat, material);
188 }
189
190 float BlenderStrokeRenderer::get_stroke_vertex_z(void) const
191 {
192         float z = _z;
193         BlenderStrokeRenderer *self = const_cast<BlenderStrokeRenderer *>(this);
194         if (!(_z < _z_delta * 100000.0f))
195                 self->_z_delta *= 10.0f;
196         self->_z += _z_delta;
197         return -z;
198 }
199
200 unsigned int BlenderStrokeRenderer::get_stroke_mesh_id(void) const
201 {
202         unsigned mesh_id = _mesh_id;
203         BlenderStrokeRenderer *self = const_cast<BlenderStrokeRenderer *>(this);
204         self->_mesh_id--;
205         return mesh_id;
206 }
207
208 void BlenderStrokeRenderer::RenderStrokeRep(StrokeRep *iStrokeRep) const
209 {
210         RenderStrokeRepBasic(iStrokeRep);
211 }
212
213 void BlenderStrokeRenderer::RenderStrokeRepBasic(StrokeRep *iStrokeRep) const
214 {
215         ////////////////////
216         //  Build up scene
217         ////////////////////
218
219         vector<Strip*>& strips = iStrokeRep->getStrips();
220         Strip::vertex_container::iterator v[3];
221         StrokeVertexRep *svRep[3];
222         /* Vec3r color[3]; */ /* UNUSED */
223         unsigned int vertex_index, edge_index, loop_index;
224         Vec2r p;
225
226         for (vector<Strip*>::iterator s = strips.begin(), send = strips.end(); s != send; ++s) {
227                 Strip::vertex_container& strip_vertices = (*s)->vertices();
228                 int strip_vertex_count = (*s)->sizeStrip();
229                 int xl, xu, yl, yu, n, visible_faces, visible_segments;
230                 bool visible;
231
232                 // iterate over all vertices and count visible faces and strip segments
233                 // (note: a strip segment is a series of visible faces, while two strip
234                 // segments are separated by one or more invisible faces)
235                 v[0] = strip_vertices.begin();
236                 v[1] = v[0] + 1;
237                 v[2] = v[0] + 2;
238                 visible_faces = visible_segments = 0;
239                 visible = false;
240                 for (n = 2; n < strip_vertex_count; n++, v[0]++, v[1]++, v[2]++) {
241                         svRep[0] = *(v[0]);
242                         svRep[1] = *(v[1]);
243                         svRep[2] = *(v[2]);
244                         xl = xu = yl = yu = 0;
245                         for (int j = 0; j < 3; j++) {
246                                 p = svRep[j]->point2d();
247                                 if (p[0] < 0.0)
248                                         xl++;
249                                 else if (p[0] > _width)
250                                         xu++;
251                                 if (p[1] < 0.0)
252                                         yl++;
253                                 else if (p[1] > _height)
254                                         yu++;
255                         }
256                         if (xl == 3 || xu == 3 || yl == 3 || yu == 3) {
257                                 visible = false;
258                         }
259                         else {
260                                 visible_faces++;
261                                 if (!visible)
262                                         visible_segments++;
263                                 visible = true;
264                         }
265                 }
266                 if (visible_faces == 0)
267                         continue;
268
269                 //me = Mesh.New()
270 #if 0
271                 Object *object_mesh = BKE_object_add(freestyle_bmain, freestyle_scene, OB_MESH);
272 #else
273                 Object *object_mesh = NewMesh();
274 #endif
275                 Mesh *mesh = (Mesh *)object_mesh->data;
276                 mesh->mat = (Material **)MEM_mallocN(1 * sizeof(Material *), "MaterialList");
277                 mesh->mat[0] = material;
278                 mesh->totcol = 1;
279                 test_object_materials(freestyle_bmain, (ID *)mesh);
280
281                 // vertices allocation
282                 mesh->totvert = visible_faces + visible_segments * 2;
283                 mesh->mvert = (MVert *)CustomData_add_layer(&mesh->vdata, CD_MVERT, CD_CALLOC, NULL, mesh->totvert);
284
285                 // edges allocation
286                 mesh->totedge = visible_faces * 2 + visible_segments;
287                 mesh->medge = (MEdge *)CustomData_add_layer(&mesh->edata, CD_MEDGE, CD_CALLOC, NULL, mesh->totedge);
288
289                 // faces allocation
290                 mesh->totpoly = visible_faces;
291                 mesh->mpoly = (MPoly *)CustomData_add_layer(&mesh->pdata, CD_MPOLY, CD_CALLOC, NULL, mesh->totpoly);
292
293                 // loops allocation
294                 mesh->totloop = visible_faces * 3;
295                 mesh->mloop = (MLoop *)CustomData_add_layer(&mesh->ldata, CD_MLOOP, CD_CALLOC, NULL, mesh->totloop);
296
297                 // colors allocation
298                 mesh->mloopcol = (MLoopCol *)CustomData_add_layer(&mesh->ldata, CD_MLOOPCOL, CD_CALLOC, NULL, mesh->totloop);
299
300                 ////////////////////
301                 //  Data copy
302                 ////////////////////
303
304                 MVert *vertices = mesh->mvert;
305                 MEdge *edges = mesh->medge;
306                 MPoly *polys = mesh->mpoly;
307                 MLoop *loops = mesh->mloop;
308                 MLoopCol *colors = mesh->mloopcol;
309
310                 v[0] = strip_vertices.begin();
311                 v[1] = v[0] + 1;
312                 v[2] = v[0] + 2;
313
314                 vertex_index = edge_index = loop_index = 0;
315                 visible = false;
316
317                 // Note: Mesh generation in the following loop assumes stroke strips
318                 // to be triangle strips.
319                 for (n = 2; n < strip_vertex_count; n++, v[0]++, v[1]++, v[2]++) {
320                         svRep[0] = *(v[0]);
321                         svRep[1] = *(v[1]);
322                         svRep[2] = *(v[2]);
323                         xl = xu = yl = yu = 0;
324                         for (int j = 0; j < 3; j++) {
325                                 p = svRep[j]->point2d();
326                                 if (p[0] < 0.0)
327                                         xl++;
328                                 else if (p[0] > _width)
329                                         xu++;
330                                 if (p[1] < 0.0)
331                                         yl++;
332                                 else if (p[1] > _height)
333                                         yu++;
334                         }
335                         if (xl == 3 || xu == 3 || yl == 3 || yu == 3) {
336                                 visible = false;
337                         }
338                         else {
339                                 if (!visible) {
340                                         // first vertex
341                                         vertices->co[0] = svRep[0]->point2d()[0];
342                                         vertices->co[1] = svRep[0]->point2d()[1];
343                                         vertices->co[2] = get_stroke_vertex_z();
344                                         ++vertices;
345                                         ++vertex_index;
346
347                                         // second vertex
348                                         vertices->co[0] = svRep[1]->point2d()[0];
349                                         vertices->co[1] = svRep[1]->point2d()[1];
350                                         vertices->co[2] = get_stroke_vertex_z();
351                                         ++vertices;
352                                         ++vertex_index;
353
354                                         // first edge
355                                         edges->v1 = vertex_index - 2;
356                                         edges->v2 = vertex_index - 1;
357                                         ++edges;
358                                         ++edge_index;
359                                 }
360                                 visible = true;
361
362                                 // vertex
363                                 vertices->co[0] = svRep[2]->point2d()[0];
364                                 vertices->co[1] = svRep[2]->point2d()[1];
365                                 vertices->co[2] = get_stroke_vertex_z();
366                                 ++vertices;
367                                 ++vertex_index;
368
369                                 // edges
370                                 edges->v1 = vertex_index - 1;
371                                 edges->v2 = vertex_index - 3;
372                                 ++edges;
373                                 ++edge_index;
374
375                                 edges->v1 = vertex_index - 1;
376                                 edges->v2 = vertex_index - 2;
377                                 ++edges;
378                                 ++edge_index;
379
380                                 // poly
381                                 polys->loopstart = loop_index;
382                                 polys->totloop = 3;
383                                 ++polys;
384
385                                 // loops
386                                 if (n % 2 == 0) {
387                                         loops[0].v = vertex_index - 1;
388                                         loops[0].e = edge_index - 1;
389
390                                         loops[1].v = vertex_index - 2;
391                                         loops[1].e = edge_index - 3;
392
393                                         loops[2].v = vertex_index - 3;
394                                         loops[2].e = edge_index - 2;
395                                 }
396                                 else {
397                                         loops[0].v = vertex_index - 1;
398                                         loops[0].e = edge_index - 2;
399
400                                         loops[1].v = vertex_index - 3;
401                                         loops[1].e = edge_index - 3;
402
403                                         loops[2].v = vertex_index - 2;
404                                         loops[2].e = edge_index - 1;
405                                 }
406                                 loops += 3;
407                                 loop_index += 3;
408
409                                 // colors
410                                 if (n % 2 == 0) {
411                                         colors[0].r = (short)(255.0f * svRep[2]->color()[0]);
412                                         colors[0].g = (short)(255.0f * svRep[2]->color()[1]);
413                                         colors[0].b = (short)(255.0f * svRep[2]->color()[2]);
414                                         colors[0].a = (short)(255.0f * svRep[2]->alpha());
415
416                                         colors[1].r = (short)(255.0f * svRep[1]->color()[0]);
417                                         colors[1].g = (short)(255.0f * svRep[1]->color()[1]);
418                                         colors[1].b = (short)(255.0f * svRep[1]->color()[2]);
419                                         colors[1].a = (short)(255.0f * svRep[1]->alpha());
420
421                                         colors[2].r = (short)(255.0f * svRep[0]->color()[0]);
422                                         colors[2].g = (short)(255.0f * svRep[0]->color()[1]);
423                                         colors[2].b = (short)(255.0f * svRep[0]->color()[2]);
424                                         colors[2].a = (short)(255.0f * svRep[0]->alpha());
425                                 }
426                                 else {
427                                         colors[0].r = (short)(255.0f * svRep[2]->color()[0]);
428                                         colors[0].g = (short)(255.0f * svRep[2]->color()[1]);
429                                         colors[0].b = (short)(255.0f * svRep[2]->color()[2]);
430                                         colors[0].a = (short)(255.0f * svRep[2]->alpha());
431
432                                         colors[1].r = (short)(255.0f * svRep[0]->color()[0]);
433                                         colors[1].g = (short)(255.0f * svRep[0]->color()[1]);
434                                         colors[1].b = (short)(255.0f * svRep[0]->color()[2]);
435                                         colors[1].a = (short)(255.0f * svRep[0]->alpha());
436
437                                         colors[2].r = (short)(255.0f * svRep[1]->color()[0]);
438                                         colors[2].g = (short)(255.0f * svRep[1]->color()[1]);
439                                         colors[2].b = (short)(255.0f * svRep[1]->color()[2]);
440                                         colors[2].a = (short)(255.0f * svRep[1]->alpha());
441                                 }
442                                 colors += 3;
443                         }
444                 } // loop over strip vertices
445 #if 0
446                 BKE_mesh_validate(mesh, TRUE);
447 #endif
448         } // loop over strips
449 }
450
451 // A replacement of BKE_object_add() for better performance.
452 Object *BlenderStrokeRenderer::NewMesh() const
453 {
454         Object *ob;
455         Base *base;
456         char name[MAX_ID_NAME];
457         unsigned int mesh_id = get_stroke_mesh_id();
458
459         /* XXX this is for later review, for now we start names with 27 (DEL) 
460            to allow ignoring them in DAG_ids_check_recalc() */
461         BLI_snprintf(name, MAX_ID_NAME, "%c0%08xOB", 27, mesh_id);
462         ob = BKE_object_add_only_object(freestyle_bmain, OB_MESH, name);
463         BLI_snprintf(name, MAX_ID_NAME, "%c0%08xME", 27, mesh_id);
464         ob->data = BKE_mesh_add(freestyle_bmain, name);
465         ob->lay = 1;
466
467         base = BKE_scene_base_add(freestyle_scene, ob);
468 #if 0
469         BKE_scene_base_deselect_all(scene);
470         BKE_scene_base_select(scene, base);
471 #else
472         (void)base;
473 #endif
474         ob->recalc = OB_RECALC_OB | OB_RECALC_DATA | OB_RECALC_TIME;
475
476         return ob;
477 }
478
479 Render *BlenderStrokeRenderer::RenderScene(Render *re, bool render)
480 {
481         Camera *camera = (Camera *)freestyle_scene->camera->data;
482         if (camera->clipend < _z)
483                 camera->clipend = _z + _z_delta * 100.0f;
484 #if 0
485         if (G.debug & G_DEBUG_FREESTYLE) {
486                 cout << "clipsta " << camera->clipsta << ", clipend " << camera->clipend << endl;
487         }
488 #endif
489
490         Render *freestyle_render = RE_NewRender(freestyle_scene->id.name);
491
492         RE_RenderFreestyleStrokes(freestyle_render, freestyle_bmain, freestyle_scene, render);
493
494         return freestyle_render;
495 }
496
497 } /* namespace Freestyle */