1914c40f817286891eb6697da35c43ee2d1d92a8
[blender-staging.git] / source / blender / editors / sculpt_paint / sculpt.c
1 /*
2  * $Id$
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software  Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2006 by Nicholas Bishop
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  *
29  * Implements the Sculpt Mode tools
30  *
31  */
32
33 #include "MEM_guardedalloc.h"
34
35 #include "BLI_math.h"
36 #include "BLI_blenlib.h"
37 #include "BLI_dynstr.h"
38 #include "BLI_ghash.h"
39 #include "BLI_pbvh.h"
40 #include "BLI_threads.h"
41
42 #include "DNA_armature_types.h"
43 #include "DNA_brush_types.h"
44 #include "DNA_image_types.h"
45 #include "DNA_key_types.h"
46 #include "DNA_mesh_types.h"
47 #include "DNA_meshdata_types.h"
48 #include "DNA_modifier_types.h"
49 #include "DNA_object_types.h"
50 #include "DNA_screen_types.h"
51 #include "DNA_scene_types.h"
52 #include "DNA_texture_types.h"
53 #include "DNA_view3d_types.h"
54 #include "DNA_userdef_types.h"
55 #include "DNA_color_types.h"
56
57 #include "BKE_brush.h"
58 #include "BKE_cdderivedmesh.h"
59 #include "BKE_context.h"
60 #include "BKE_customdata.h"
61 #include "BKE_DerivedMesh.h"
62 #include "BKE_depsgraph.h"
63 #include "BKE_global.h"
64 #include "BKE_image.h"
65 #include "BKE_key.h"
66 #include "BKE_library.h"
67 #include "BKE_main.h"
68 #include "BKE_mesh.h"
69 #include "BKE_modifier.h"
70 #include "BKE_multires.h"
71 #include "BKE_paint.h"
72 #include "BKE_report.h"
73 #include "BKE_texture.h"
74 #include "BKE_utildefines.h"
75 #include "BKE_colortools.h"
76
77 #include "BIF_gl.h"
78 #include "BIF_glutil.h"
79
80 #include "WM_api.h"
81 #include "WM_types.h"
82 #include "ED_object.h"
83 #include "ED_screen.h"
84 #include "ED_sculpt.h"
85 #include "ED_space_api.h"
86 #include "ED_util.h"
87 #include "ED_view3d.h"
88 #include "paint_intern.h"
89 #include "sculpt_intern.h"
90
91 #include "RNA_access.h"
92 #include "RNA_define.h"
93
94 #include "IMB_imbuf_types.h"
95
96 #include "RE_render_ext.h"
97 #include "RE_shader_ext.h" /*for multitex_ext*/
98
99 #include "GPU_draw.h"
100 #include "gpu_buffers.h"
101
102 #include <math.h>
103 #include <stdlib.h>
104 #include <string.h>
105
106 /* Number of vertices to average in order to determine the flatten distance */
107 #define FLATTEN_SAMPLE_SIZE 10
108
109 /* ===== STRUCTS =====
110  *
111  */
112
113 typedef enum StrokeFlags {
114         CLIP_X = 1,
115         CLIP_Y = 2,
116         CLIP_Z = 4
117 } StrokeFlags;
118
119 /* Cache stroke properties. Used because
120    RNA property lookup isn't particularly fast.
121
122    For descriptions of these settings, check the operator properties.
123 */
124 typedef struct StrokeCache {
125         /* Invariants */
126         float initial_radius;
127         float scale[3];
128         int flag;
129         float clip_tolerance[3];
130         float initial_mouse[2];
131
132         /* Variants */
133         float radius;
134         float true_location[3];
135         float location[3];
136         float flip;
137         float pressure;
138         float mouse[2];
139         float bstrength;
140         float tex_mouse[2];
141
142         /* The rest is temporary storage that isn't saved as a property */
143
144         int first_time; /* Beginning of stroke may do some things special */
145
146         bglMats *mats;
147
148         /* Clean this up! */
149         ViewContext *vc;
150         Brush *brush;
151
152         float (*face_norms)[3]; /* Copy of the mesh faces' normals */
153         float rotation; /* Texture rotation (radians) for anchored and rake modes */
154         int pixel_radius, previous_pixel_radius;
155         float grab_active_location[8][3];
156         float grab_delta[3], grab_delta_symmetry[3];
157         float old_grab_location[3], orig_grab_location[3];
158         int symmetry; /* Symmetry index between 0 and 7 */
159         float view_normal[3], view_normal_symmetry[3];
160         int last_rake[2]; /* Last location of updating rake rotation */
161         int original;
162 } StrokeCache;
163
164 /* ===== OPENGL =====
165  *
166  * Simple functions to get data from the GL
167  */
168
169 /* Convert a point in model coordinates to 2D screen coordinates. */
170 static void projectf(bglMats *mats, const float v[3], float p[2])
171 {
172         double ux, uy, uz;
173
174         gluProject(v[0],v[1],v[2], mats->modelview, mats->projection,
175                    (GLint *)mats->viewport, &ux, &uy, &uz);
176         p[0]= ux;
177         p[1]= uy;
178 }
179
180 /*XXX: static void project(bglMats *mats, const float v[3], short p[2])
181 {
182         float f[2];
183         projectf(mats, v, f);
184
185         p[0]= f[0];
186         p[1]= f[1];
187 }
188 */
189
190 /*** BVH Tree ***/
191
192 /* Get a screen-space rectangle of the modified area */
193 int sculpt_get_redraw_rect(ARegion *ar, RegionView3D *rv3d,
194                             Object *ob, rcti *rect)
195 {
196         float bb_min[3], bb_max[3], pmat[4][4];
197         int i, j, k;
198
199         view3d_get_object_project_mat(rv3d, ob, pmat);
200
201         BLI_pbvh_redraw_BB(ob->sculpt->tree, bb_min, bb_max);
202
203         rect->xmin = rect->ymin = INT_MAX;
204         rect->xmax = rect->ymax = INT_MIN;
205
206         if(bb_min[0] > bb_max[0] || bb_min[1] > bb_max[1] || bb_min[2] > bb_max[2])
207                 return 0;
208
209         for(i = 0; i < 2; ++i) {
210                 for(j = 0; j < 2; ++j) {
211                         for(k = 0; k < 2; ++k) {
212                                 float vec[3], proj[2];
213                                 vec[0] = i ? bb_min[0] : bb_max[0];
214                                 vec[1] = j ? bb_min[1] : bb_max[1];
215                                 vec[2] = k ? bb_min[2] : bb_max[2];
216                                 view3d_project_float(ar, vec, proj, pmat);
217                                 rect->xmin = MIN2(rect->xmin, proj[0]);
218                                 rect->xmax = MAX2(rect->xmax, proj[0]);
219                                 rect->ymin = MIN2(rect->ymin, proj[1]);
220                                 rect->ymax = MAX2(rect->ymax, proj[1]);
221                         }
222                 }
223         }
224         
225         return rect->xmin < rect->xmax && rect->ymin < rect->ymax;
226 }
227
228 void sculpt_get_redraw_planes(float planes[4][4], ARegion *ar,
229                               RegionView3D *rv3d, Object *ob)
230 {
231         BoundBox *bb = MEM_callocN(sizeof(BoundBox), "sculpt boundbox");
232         bglMats mats;
233         int i;
234         rcti rect;
235
236         view3d_get_transformation(ar, rv3d, ob, &mats);
237         sculpt_get_redraw_rect(ar, rv3d,ob, &rect);
238
239 #if 1
240         /* use some extra space just in case */
241         rect.xmin -= 2;
242         rect.xmax += 2;
243         rect.ymin -= 2;
244         rect.ymax += 2;
245 #else
246         /* it was doing this before, allows to redraw a smaller
247            part of the screen but also gives artifaces .. */
248         rect.xmin += 2;
249         rect.xmax -= 2;
250         rect.ymin += 2;
251         rect.ymax -= 2;
252 #endif
253
254         view3d_calculate_clipping(bb, planes, &mats, &rect);
255
256         for(i = 0; i < 16; ++i)
257                 ((float*)planes)[i] = -((float*)planes)[i];
258
259         MEM_freeN(bb);
260
261         /* clear redraw flag from nodes */
262         BLI_pbvh_update(ob->sculpt->tree, PBVH_UpdateRedraw, NULL);
263 }
264
265 /************************** Undo *************************/
266
267 typedef struct SculptUndoNode {
268         struct SculptUndoNode *next, *prev;
269
270         char idname[MAX_ID_NAME];       /* name instead of pointer*/
271         void *node;                                     /* only during push, not valid afterwards! */
272
273         float (*co)[3];
274         short (*no)[3];
275         int totvert;
276
277         /* non-multires */
278         int maxvert;                            /* to verify if totvert it still the same */
279         int *index;                                     /* to restore into right location */
280
281         /* multires */
282         int maxgrid;                            /* same for grid */
283         int gridsize;                           /* same for grid */
284         int totgrid;                            /* to restore into right location */
285         int *grids;                                     /* to restore into right location */
286 } SculptUndoNode;
287
288 static void update_cb(PBVHNode *node, void *data)
289 {
290         BLI_pbvh_node_mark_update(node);
291 }
292
293 static void sculpt_undo_restore(bContext *C, ListBase *lb)
294 {
295         Scene *scene = CTX_data_scene(C);
296         Object *ob = CTX_data_active_object(C);
297         DerivedMesh *dm = mesh_get_derived_final(scene, ob, 0);
298         SculptSession *ss = ob->sculpt;
299         SculptUndoNode *unode;
300         MVert *mvert;
301         MultiresModifierData *mmd;
302         int *index;
303         int i, j, update= 0;
304
305         sculpt_update_mesh_elements(scene, ob, 0);
306
307         for(unode=lb->first; unode; unode=unode->next) {
308                 if(!(strcmp(unode->idname, ob->id.name)==0))
309                         continue;
310
311                 if(unode->maxvert) {
312                         /* regular mesh restore */
313                         if(ss->totvert != unode->maxvert)
314                                 continue;
315
316                         index= unode->index;
317                         mvert= ss->mvert;
318
319                         for(i=0; i<unode->totvert; i++) {
320                                 swap_v3_v3(mvert[index[i]].co, unode->co[i]);
321                                 mvert[index[i]].flag |= ME_VERT_PBVH_UPDATE;
322                         }
323                 }
324                 else if(unode->maxgrid) {
325                         /* multires restore */
326                         DMGridData **grids, *grid;
327                         float (*co)[3];
328                         int gridsize;
329
330                         if(dm->getNumGrids(dm) != unode->maxgrid)
331                                 continue;
332                         if(dm->getGridSize(dm) != unode->gridsize)
333                                 continue;
334
335                         grids= dm->getGridData(dm);
336                         gridsize= dm->getGridSize(dm);
337
338                         co = unode->co;
339                         for(j=0; j<unode->totgrid; j++) {
340                                 grid= grids[unode->grids[j]];
341
342                                 for(i=0; i<gridsize*gridsize; i++, co++)
343                                         swap_v3_v3(grid[i].co, co[0]);
344                         }
345                 }
346
347                 update= 1;
348         }
349
350         if(update) {
351                 if(ss->kb) sculpt_mesh_to_key(ss->ob, ss->kb);
352                 if(ss->refkb) sculpt_key_to_mesh(ss->refkb, ob);
353
354                 /* we update all nodes still, should be more clever, but also
355                    needs to work correct when exiting/entering sculpt mode and
356                    the nodes get recreated, though in that case it could do all */
357                 BLI_pbvh_search_callback(ss->tree, NULL, NULL, update_cb, NULL);
358                 BLI_pbvh_update(ss->tree, PBVH_UpdateBB|PBVH_UpdateOriginalBB|PBVH_UpdateRedraw, NULL);
359
360                 if((mmd=sculpt_multires_active(ob)))
361                         multires_mark_as_modified(ob);
362         }
363 }
364
365 static void sculpt_undo_free(ListBase *lb)
366 {
367         SculptUndoNode *unode;
368
369         for(unode=lb->first; unode; unode=unode->next) {
370                 if(unode->co)
371                         MEM_freeN(unode->co);
372                 if(unode->no)
373                         MEM_freeN(unode->no);
374                 if(unode->index)
375                         MEM_freeN(unode->index);
376                 if(unode->grids)
377                         MEM_freeN(unode->grids);
378         }
379 }
380
381 static SculptUndoNode *sculpt_undo_get_node(SculptSession *ss, PBVHNode *node)
382 {
383         ListBase *lb= undo_paint_push_get_list(UNDO_PAINT_MESH);
384         SculptUndoNode *unode;
385
386         if(!lb)
387                 return NULL;
388
389         for(unode=lb->first; unode; unode=unode->next)
390                 if(unode->node == node)
391                         return unode;
392
393         return NULL;
394 }
395
396 static SculptUndoNode *sculpt_undo_push_node(SculptSession *ss, PBVHNode *node)
397 {
398         ListBase *lb= undo_paint_push_get_list(UNDO_PAINT_MESH);
399         Object *ob= ss->ob;
400         SculptUndoNode *unode;
401         int totvert, allvert, totgrid, maxgrid, gridsize, *grids;
402
403         /* list is manipulated by multiple threads, so we lock */
404         BLI_lock_thread(LOCK_CUSTOM1);
405
406         if((unode= sculpt_undo_get_node(ss, node))) {
407                 BLI_unlock_thread(LOCK_CUSTOM1);
408                 return unode;
409         }
410
411         unode= MEM_callocN(sizeof(SculptUndoNode), "SculptUndoNode");
412         strcpy(unode->idname, ob->id.name);
413         unode->node= node;
414
415         BLI_pbvh_node_num_verts(ss->tree, node, &totvert, &allvert);
416         BLI_pbvh_node_get_grids(ss->tree, node, &grids, &totgrid,
417                 &maxgrid, &gridsize, NULL, NULL);
418
419         unode->totvert= totvert;
420         /* we will use this while sculpting, is mapalloc slow to access then? */
421         unode->co= MEM_mapallocN(sizeof(float)*3*allvert, "SculptUndoNode.co");
422         unode->no= MEM_mapallocN(sizeof(short)*3*allvert, "SculptUndoNode.no");
423         undo_paint_push_count_alloc(UNDO_PAINT_MESH, (sizeof(float)*3 + sizeof(short)*3 + sizeof(int))*allvert);
424         BLI_addtail(lb, unode);
425
426         if(maxgrid) {
427                 /* multires */
428                 unode->maxgrid= maxgrid;
429                 unode->totgrid= totgrid;
430                 unode->gridsize= gridsize;
431                 unode->grids= MEM_mapallocN(sizeof(int)*totgrid, "SculptUndoNode.grids");
432         }
433         else {
434                 /* regular mesh */
435                 unode->maxvert= ss->totvert;
436                 unode->index= MEM_mapallocN(sizeof(int)*allvert, "SculptUndoNode.index");
437         }
438
439         BLI_unlock_thread(LOCK_CUSTOM1);
440
441         /* copy threaded, hopefully this is the performance critical part */
442         {
443                 PBVHVertexIter vd;
444
445                 BLI_pbvh_vertex_iter_begin(ss->tree, node, vd, PBVH_ITER_ALL) {
446                         copy_v3_v3(unode->co[vd.i], vd.co);
447                         if(vd.no) VECCOPY(unode->no[vd.i], vd.no)
448                         else normal_float_to_short_v3(unode->no[vd.i], vd.fno);
449                         if(vd.vert_indices) unode->index[vd.i]= vd.vert_indices[vd.i];
450                 }
451                 BLI_pbvh_vertex_iter_end;
452         }
453
454         if(unode->grids)
455                 memcpy(unode->grids, grids, sizeof(int)*totgrid);
456         
457         return unode;
458 }
459
460 static void sculpt_undo_push_begin(SculptSession *ss, char *name)
461 {
462         undo_paint_push_begin(UNDO_PAINT_MESH, name,
463                 sculpt_undo_restore, sculpt_undo_free);
464 }
465
466 static void sculpt_undo_push_end(SculptSession *ss)
467 {
468         ListBase *lb= undo_paint_push_get_list(UNDO_PAINT_MESH);
469         SculptUndoNode *unode;
470
471         /* we don't need normals in the undo stack */
472         for(unode=lb->first; unode; unode=unode->next) {
473                 if(unode->no) {
474                         MEM_freeN(unode->no);
475                         unode->no= NULL;
476                 }
477         }
478
479         undo_paint_push_end(UNDO_PAINT_MESH);
480 }
481
482 void ED_sculpt_force_update(bContext *C)
483 {
484         Object *ob= CTX_data_active_object(C);
485
486         if(ob && (ob->mode & OB_MODE_SCULPT))
487                 multires_force_update(ob);
488 }
489
490 /************************ Brush Testing *******************/
491
492 typedef struct SculptBrushTest {
493         float radius_squared;
494         float location[3];
495
496         float dist;
497 } SculptBrushTest;
498
499 static void sculpt_brush_test_init(SculptSession *ss, SculptBrushTest *test)
500 {
501         test->radius_squared= ss->cache->radius*ss->cache->radius;
502         copy_v3_v3(test->location, ss->cache->location);
503 }
504
505 static int sculpt_brush_test(SculptBrushTest *test, float co[3])
506 {
507         float distsq, delta[3];
508
509         sub_v3_v3v3(delta, co, test->location);
510         distsq = INPR(delta, delta);
511
512         if(distsq < test->radius_squared) {
513                 test->dist = sqrt(distsq);
514                 return 1;
515         }
516
517         return 0;
518 }
519
520 /* ===== Sculpting =====
521  *
522  */
523
524 /* Return modified brush strength. Includes the direction of the brush, positive
525    values pull vertices, negative values push. Uses tablet pressure and a
526    special multiplier found experimentally to scale the strength factor. */
527 static float brush_strength(Sculpt *sd, StrokeCache *cache)
528 {
529         Brush *brush = paint_brush(&sd->paint);
530         /* Primary strength input; square it to make lower values more sensitive */
531         float alpha = brush->alpha * brush->alpha;
532
533         float dir= brush->flag & BRUSH_DIR_IN ? -1 : 1;
534         float pressure= 1;
535         float flip= cache->flip ? -1:1;
536
537         if(brush->flag & BRUSH_ALPHA_PRESSURE)
538                 pressure *= cache->pressure;
539         
540         switch(brush->sculpt_tool){
541         case SCULPT_TOOL_DRAW:
542         case SCULPT_TOOL_INFLATE:
543         case SCULPT_TOOL_CLAY:
544         case SCULPT_TOOL_FLATTEN:
545         case SCULPT_TOOL_LAYER:
546                 return alpha * dir * pressure * flip; /*XXX: not sure why? was multiplied by G.vd->grid */;
547         case SCULPT_TOOL_SMOOTH:
548                 return alpha * 4 * pressure;
549         case SCULPT_TOOL_PINCH:
550                 return alpha / 2 * dir * pressure * flip;
551         case SCULPT_TOOL_GRAB:
552                 return 1;
553         default:
554                 return 0;
555         }
556 }
557
558 /* Uses symm to selectively flip any axis of a coordinate. */
559 static void flip_coord(float out[3], float in[3], const char symm)
560 {
561         if(symm & SCULPT_SYMM_X)
562                 out[0]= -in[0];
563         else
564                 out[0]= in[0];
565         if(symm & SCULPT_SYMM_Y)
566                 out[1]= -in[1];
567         else
568                 out[1]= in[1];
569         if(symm & SCULPT_SYMM_Z)
570                 out[2]= -in[2];
571         else
572                 out[2]= in[2];
573 }
574
575 /* Get a pixel from the texcache at (px, py) */
576 static unsigned char get_texcache_pixel(const SculptSession *ss, int px, int py)
577 {
578         unsigned *p;
579         p = ss->texcache + py * ss->texcache_side + px;
580         return ((unsigned char*)(p))[0];
581 }
582
583 static float get_texcache_pixel_bilinear(const SculptSession *ss, float u, float v)
584 {
585         int x, y, x2, y2;
586         const int tc_max = ss->texcache_side - 1;
587         float urat, vrat, uopp;
588
589         if(u < 0) u = 0;
590         else if(u >= ss->texcache_side) u = tc_max;
591         if(v < 0) v = 0;
592         else if(v >= ss->texcache_side) v = tc_max;
593
594         x = floor(u);
595         y = floor(v);
596         x2 = x + 1;
597         y2 = y + 1;
598
599         if(x2 > ss->texcache_side) x2 = tc_max;
600         if(y2 > ss->texcache_side) y2 = tc_max;
601         
602         urat = u - x;
603         vrat = v - y;
604         uopp = 1 - urat;
605                 
606         return ((get_texcache_pixel(ss, x, y) * uopp +
607                  get_texcache_pixel(ss, x2, y) * urat) * (1 - vrat) + 
608                 (get_texcache_pixel(ss, x, y2) * uopp +
609                  get_texcache_pixel(ss, x2, y2) * urat) * vrat) / 255.0;
610 }
611
612 /* Return a multiplier for brush strength on a particular vertex. */
613 static float tex_strength(SculptSession *ss, Brush *br, float *point, const float len)
614 {
615         MTex *tex = &br->mtex;
616         float avg= 1;
617
618         if(!tex) {
619                 avg= 1;
620         }
621         else if(tex->brush_map_mode == MTEX_MAP_MODE_3D) {
622                 float jnk;
623
624                 /* Get strength by feeding the vertex 
625                    location directly into a texture */
626                 externtex(tex, point, &avg,
627                           &jnk, &jnk, &jnk, &jnk);
628         }
629         else if(ss->texcache) {
630                 const float bsize= ss->cache->pixel_radius * 2;
631                 const float rot= tex->rot + ss->cache->rotation;
632                 int px, py;
633                 float flip[3], point_2d[2];
634
635                 /* If the active area is being applied for symmetry, flip it
636                    across the symmetry axis in order to project it. This insures
637                    that the brush texture will be oriented correctly. */
638                 copy_v3_v3(flip, point);
639                 flip_coord(flip, flip, ss->cache->symmetry);
640                 projectf(ss->cache->mats, flip, point_2d);
641
642                 /* For Tile and Drag modes, get the 2D screen coordinates of the
643                    and scale them up or down to the texture size. */
644                 if(tex->brush_map_mode == MTEX_MAP_MODE_TILED) {
645                         const int sx= (const int)tex->size[0];
646                         const int sy= (const int)tex->size[1];
647                         
648                         float fx= point_2d[0];
649                         float fy= point_2d[1];
650                         
651                         float angle= atan2(fy, fx) - rot;
652                         float flen= sqrtf(fx*fx + fy*fy);
653                         
654                         if(rot<0.001 && rot>-0.001) {
655                                 px= point_2d[0];
656                                 py= point_2d[1];
657                         } else {
658                                 px= flen * cos(angle) + 2000;
659                                 py= flen * sin(angle) + 2000;
660                         }
661                         if(sx != 1)
662                                 px %= sx-1;
663                         if(sy != 1)
664                                 py %= sy-1;
665                         avg= get_texcache_pixel_bilinear(ss, ss->texcache_side*px/sx, ss->texcache_side*py/sy);
666                 }
667                 else if(tex->brush_map_mode == MTEX_MAP_MODE_FIXED) {
668                         float fx= (point_2d[0] - ss->cache->tex_mouse[0]) / bsize;
669                         float fy= (point_2d[1] - ss->cache->tex_mouse[1]) / bsize;
670
671                         float angle= atan2(fy, fx) - rot;
672                         float flen= sqrtf(fx*fx + fy*fy);
673                         
674                         fx = flen * cos(angle) + 0.5;
675                         fy = flen * sin(angle) + 0.5;
676
677                         avg= get_texcache_pixel_bilinear(ss, fx * ss->texcache_side, fy * ss->texcache_side);
678                 }
679         }
680
681         avg*= brush_curve_strength(br, len, ss->cache->radius); /* Falloff curve */
682
683         return avg;
684 }
685
686 typedef struct {
687         Sculpt *sd;
688         SculptSession *ss;
689         float radius_squared;
690         int original;
691 } SculptSearchSphereData;
692
693 /* Test AABB against sphere */
694 static int sculpt_search_sphere_cb(PBVHNode *node, void *data_v)
695 {
696         SculptSearchSphereData *data = data_v;
697         float *center = data->ss->cache->location, nearest[3];
698         float t[3], bb_min[3], bb_max[3];
699         int i;
700
701         if(data->original)
702                 BLI_pbvh_node_get_original_BB(node, bb_min, bb_max);
703         else
704                 BLI_pbvh_node_get_BB(node, bb_min, bb_max);
705
706         for(i = 0; i < 3; ++i) {
707                 if(bb_min[i] > center[i])
708                         nearest[i] = bb_min[i];
709                 else if(bb_max[i] < center[i])
710                         nearest[i] = bb_max[i];
711                 else
712                         nearest[i] = center[i]; 
713         }
714         
715         sub_v3_v3v3(t, center, nearest);
716
717         return t[0] * t[0] + t[1] * t[1] + t[2] * t[2] < data->radius_squared;
718 }
719
720 /* Handles clipping against a mirror modifier and SCULPT_LOCK axis flags */
721 static void sculpt_clip(Sculpt *sd, SculptSession *ss, float *co, const float val[3])
722 {
723         int i;
724
725         for(i=0; i<3; ++i) {
726                 if(sd->flags & (SCULPT_LOCK_X << i))
727                         continue;
728
729                 if((ss->cache->flag & (CLIP_X << i)) && (fabs(co[i]) <= ss->cache->clip_tolerance[i]))
730                         co[i]= 0.0f;
731                 else
732                         co[i]= val[i];
733         }               
734 }
735
736 static void add_norm_if(float view_vec[3], float out[3], float out_flip[3], float fno[3])
737 {
738         if((dot_v3v3(view_vec, fno)) > 0) {
739                 add_v3_v3v3(out, out, fno);
740         } else {
741                 add_v3_v3v3(out_flip, out_flip, fno); /* out_flip is used when out is {0,0,0} */
742         }
743 }
744
745 /* For draw/layer/flatten; finds average normal for all active vertices */
746 static void calc_area_normal(Sculpt *sd, SculptSession *ss, float area_normal[3], PBVHNode **nodes, int totnode)
747 {
748         PBVH *bvh= ss->tree;
749         StrokeCache *cache = ss->cache;
750         const int view = 0; /* XXX: should probably be a flag, not number: brush_type==SCULPT_TOOL_DRAW ? sculptmode_brush()->view : 0; */
751         float out[3] = {0.0f, 0.0f, 0.0f};
752         float out_flip[3] = {0.0f, 0.0f, 0.0f};
753         float out_dir[3];
754         int n;
755
756         copy_v3_v3(out_dir, cache->view_normal_symmetry);
757
758         /* threaded loop over nodes */
759         #pragma omp parallel for private(n) schedule(static)
760         for(n=0; n<totnode; n++) {
761                 PBVHVertexIter vd;
762                 SculptBrushTest test;
763                 SculptUndoNode *unode;
764                 float fno[3];
765                 float nout[3] = {0.0f, 0.0f, 0.0f};
766                 float nout_flip[3] = {0.0f, 0.0f, 0.0f};
767                 
768                 // XXX push instead of get for thread safety in draw
769                 // brush .. lame, but also not harmful really
770                 unode= sculpt_undo_push_node(ss, nodes[n]);
771                 sculpt_brush_test_init(ss, &test);
772
773                 if(ss->cache->original) {
774                         BLI_pbvh_vertex_iter_begin(bvh, nodes[n], vd, PBVH_ITER_UNIQUE) {
775                                 if(sculpt_brush_test(&test, unode->co[vd.i])) {
776                                         normal_short_to_float_v3(fno, unode->no[vd.i]);
777                                         add_norm_if(out_dir, nout, nout_flip, fno);
778                                 }
779                         }
780                         BLI_pbvh_vertex_iter_end;
781                 }
782                 else {
783                         BLI_pbvh_vertex_iter_begin(bvh, nodes[n], vd, PBVH_ITER_UNIQUE) {
784                                 if(sculpt_brush_test(&test, vd.co)) {
785                                         if(vd.no) {
786                                                 normal_short_to_float_v3(fno, vd.no);
787                                                 add_norm_if(out_dir, nout, nout_flip, fno);
788                                         }
789                                         else
790                                                 add_norm_if(out_dir, nout, nout_flip, vd.fno);
791                                 }
792                         }
793                         BLI_pbvh_vertex_iter_end;
794                 }
795
796                 #pragma omp critical
797                 {
798                         /* we sum per node and add together later for threads */
799                         add_v3_v3v3(out, out, nout);
800                         add_v3_v3v3(out_flip, out_flip, nout_flip);
801                 }
802         }
803
804         if (out[0]==0.0 && out[1]==0.0 && out[2]==0.0) {
805                 copy_v3_v3(out, out_flip);
806         }
807         
808         normalize_v3(out);
809
810         out[0] = out_dir[0] * view + out[0] * (10-view);
811         out[1] = out_dir[1] * view + out[1] * (10-view);
812         out[2] = out_dir[2] * view + out[2] * (10-view);
813         
814         normalize_v3(out);
815         copy_v3_v3(area_normal, out);
816 }
817
818 static void do_draw_brush(Sculpt *sd, SculptSession *ss, PBVHNode **nodes, int totnode)
819 {
820         Brush *brush = paint_brush(&sd->paint);
821         float offset[3], area_normal[3];
822         float bstrength= ss->cache->bstrength;
823         int n;
824
825         /* area normal */
826         calc_area_normal(sd, ss, area_normal, nodes, totnode);
827
828         /* offset with as much as possible factored in already */
829         offset[0]= area_normal[0]*ss->cache->radius*ss->cache->scale[0]*bstrength;
830         offset[1]= area_normal[1]*ss->cache->radius*ss->cache->scale[1]*bstrength;
831         offset[2]= area_normal[2]*ss->cache->radius*ss->cache->scale[2]*bstrength;
832
833         /* threaded loop over nodes */
834         #pragma omp parallel for private(n) schedule(static)
835         for(n=0; n<totnode; n++) {
836                 PBVHVertexIter vd;
837                 SculptBrushTest test;
838                 
839                 sculpt_undo_push_node(ss, nodes[n]);
840                 sculpt_brush_test_init(ss, &test);
841
842                 BLI_pbvh_vertex_iter_begin(ss->tree, nodes[n], vd, PBVH_ITER_UNIQUE) {
843                         if(sculpt_brush_test(&test, vd.co)) {
844                                 /* offset vertex */
845                                 float fade = tex_strength(ss, brush, vd.co, test.dist);
846                                 float val[3]= {vd.co[0] + offset[0]*fade,
847                                                            vd.co[1] + offset[1]*fade,
848                                                            vd.co[2] + offset[2]*fade};
849
850                                 sculpt_clip(sd, ss, vd.co, val);
851                                 if(vd.mvert) vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
852                         }
853                 }
854                 BLI_pbvh_vertex_iter_end;
855
856                 BLI_pbvh_node_mark_update(nodes[n]);
857         }
858 }
859
860 /* For the smooth brush, uses the neighboring vertices around vert to calculate
861    a smoothed location for vert. Skips corner vertices (used by only one
862    polygon.) */
863 static void neighbor_average(SculptSession *ss, float avg[3], const int vert)
864 {
865         int i, skip= -1, total=0;
866         IndexNode *node= ss->fmap[vert].first;
867         char ncount= BLI_countlist(&ss->fmap[vert]);
868         MFace *f;
869
870         avg[0] = avg[1] = avg[2] = 0;
871                 
872         /* Don't modify corner vertices */
873         if(ncount==1) {
874                 copy_v3_v3(avg, ss->mvert[vert].co);
875                 return;
876         }
877
878         while(node){
879                 f= &ss->mface[node->index];
880                 
881                 if(f->v4) {
882                         skip= (f->v1==vert?2:
883                                f->v2==vert?3:
884                                f->v3==vert?0:
885                                f->v4==vert?1:-1);
886                 }
887
888                 for(i=0; i<(f->v4?4:3); ++i) {
889                         if(i != skip && (ncount!=2 || BLI_countlist(&ss->fmap[(&f->v1)[i]]) <= 2)) {
890                                 add_v3_v3v3(avg, avg, ss->mvert[(&f->v1)[i]].co);
891                                 ++total;
892                         }
893                 }
894
895                 node= node->next;
896         }
897
898         if(total>0)
899                 mul_v3_fl(avg, 1.0f / total);
900         else
901                 copy_v3_v3(avg, ss->mvert[vert].co);
902 }
903
904 static void do_mesh_smooth_brush(Sculpt *sd, SculptSession *ss, PBVHNode *node)
905 {
906         Brush *brush = paint_brush(&sd->paint);
907         float bstrength= ss->cache->bstrength;
908         PBVHVertexIter vd;
909         SculptBrushTest test;
910         
911         sculpt_brush_test_init(ss, &test);
912
913         BLI_pbvh_vertex_iter_begin(ss->tree, node, vd, PBVH_ITER_UNIQUE) {
914                 if(sculpt_brush_test(&test, vd.co)) {
915                         float fade = tex_strength(ss, brush, vd.co, test.dist)*bstrength;
916                         float avg[3], val[3];
917
918                         CLAMP(fade, 0.0f, 1.0f);
919                         
920                         neighbor_average(ss, avg, vd.vert_indices[vd.i]);
921                         val[0] = vd.co[0]+(avg[0]-vd.co[0])*fade;
922                         val[1] = vd.co[1]+(avg[1]-vd.co[1])*fade;
923                         val[2] = vd.co[2]+(avg[2]-vd.co[2])*fade;
924                         
925                         sculpt_clip(sd, ss, vd.co, val);                        
926                         if(vd.mvert) vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
927                 }
928         }
929         BLI_pbvh_vertex_iter_end;
930 }
931
932 static void do_multires_smooth_brush(Sculpt *sd, SculptSession *ss, PBVHNode *node)
933 {
934         Brush *brush = paint_brush(&sd->paint);
935         SculptBrushTest test;
936         DMGridData **griddata, *data;
937         DMGridAdjacency *gridadj, *adj;
938         float bstrength= ss->cache->bstrength;
939         float co[3], (*tmpgrid)[3];
940         int v1, v2, v3, v4;
941         int *grid_indices, totgrid, gridsize, i, x, y;
942                         
943         sculpt_brush_test_init(ss, &test);
944
945         BLI_pbvh_node_get_grids(ss->tree, node, &grid_indices, &totgrid,
946                 NULL, &gridsize, &griddata, &gridadj);
947
948         #pragma omp critical
949         tmpgrid= MEM_mallocN(sizeof(float)*3*gridsize*gridsize, "tmpgrid");
950
951         for(i = 0; i < totgrid; ++i) {
952                 data = griddata[grid_indices[i]];
953                 adj = &gridadj[grid_indices[i]];
954
955                 memset(tmpgrid, 0, sizeof(float)*3*gridsize*gridsize);
956
957                 /* average grid values */
958                 for(y = 0; y < gridsize-1; ++y)  {
959                         for(x = 0; x < gridsize-1; ++x)  {
960                                 v1 = x + y*gridsize;
961                                 v2 = (x + 1) + y*gridsize;
962                                 v3 = (x + 1) + (y + 1)*gridsize;
963                                 v4 = x + (y + 1)*gridsize;
964
965                                 cent_quad_v3(co, data[v1].co, data[v2].co, data[v3].co, data[v4].co);
966                                 mul_v3_fl(co, 0.25f);
967
968                                 add_v3_v3(tmpgrid[v1], co);
969                                 add_v3_v3(tmpgrid[v2], co);
970                                 add_v3_v3(tmpgrid[v3], co);
971                                 add_v3_v3(tmpgrid[v4], co);
972                         }
973                 }
974
975                 /* blend with existing coordinates */
976                 for(y = 0; y < gridsize; ++y)  {
977                         for(x = 0; x < gridsize; ++x)  {
978                                 if(x == 0 && adj->index[0] == -1) continue;
979                                 if(x == gridsize - 1 && adj->index[2] == -1) continue;
980                                 if(y == 0 && adj->index[3] == -1) continue;
981                                 if(y == gridsize - 1 && adj->index[1] == -1) continue;
982
983                                 copy_v3_v3(co, data[x + y*gridsize].co);
984
985                                 if(sculpt_brush_test(&test, co)) {
986                                         float fade = tex_strength(ss, brush, co, test.dist)*bstrength;
987                                         float avg[3], val[3];
988
989                                         copy_v3_v3(avg, tmpgrid[x + y*gridsize]);
990                                         if(x == 0 || x == gridsize - 1)
991                                                 mul_v3_fl(avg, 2.0f);
992                                         if(y == 0 || y == gridsize - 1)
993                                                 mul_v3_fl(avg, 2.0f);
994
995                                         CLAMP(fade, 0.0f, 1.0f);
996
997                                         val[0] = co[0]+(avg[0]-co[0])*fade;
998                                         val[1] = co[1]+(avg[1]-co[1])*fade;
999                                         val[2] = co[2]+(avg[2]-co[2])*fade;
1000                                         
1001                                         sculpt_clip(sd, ss, data[x + y*gridsize].co, val);
1002                                 }
1003                         }
1004                 }
1005         }
1006
1007         #pragma omp critical
1008         MEM_freeN(tmpgrid);
1009 }
1010
1011 static void do_smooth_brush(Sculpt *sd, SculptSession *ss, PBVHNode **nodes, int totnode)
1012 {
1013         int iteration, n;
1014
1015         for(iteration = 0; iteration < 2; ++iteration) {
1016                 #pragma omp parallel for private(n) schedule(static)
1017                 for(n=0; n<totnode; n++) {
1018                         sculpt_undo_push_node(ss, nodes[n]);
1019
1020                         if(ss->multires)
1021                                 do_multires_smooth_brush(sd, ss, nodes[n]);
1022                         else if(ss->fmap)
1023                                 do_mesh_smooth_brush(sd, ss, nodes[n]);
1024
1025                         BLI_pbvh_node_mark_update(nodes[n]);
1026                 }
1027
1028                 if(ss->multires)
1029                         multires_stitch_grids(ss->ob);
1030         }
1031 }
1032
1033 static void do_pinch_brush(Sculpt *sd, SculptSession *ss, PBVHNode **nodes, int totnode)
1034 {
1035         Brush *brush = paint_brush(&sd->paint);
1036         float bstrength= ss->cache->bstrength;
1037         int n;
1038
1039         #pragma omp parallel for private(n) schedule(static)
1040         for(n=0; n<totnode; n++) {
1041                 PBVHVertexIter vd;
1042                 SculptBrushTest test;
1043                 
1044                 sculpt_undo_push_node(ss, nodes[n]);
1045                 sculpt_brush_test_init(ss, &test);
1046
1047                 BLI_pbvh_vertex_iter_begin(ss->tree, nodes[n], vd, PBVH_ITER_UNIQUE) {
1048                         if(sculpt_brush_test(&test, vd.co)) {
1049                                 float fade = tex_strength(ss, brush, vd.co, test.dist)*bstrength;
1050                                 float val[3]= {vd.co[0]+(test.location[0]-vd.co[0])*fade,
1051                                                            vd.co[1]+(test.location[1]-vd.co[1])*fade,
1052                                                            vd.co[2]+(test.location[2]-vd.co[2])*fade};
1053                                 
1054                                 sculpt_clip(sd, ss, vd.co, val);                        
1055                                 if(vd.mvert) vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
1056                         }
1057                 }
1058                 BLI_pbvh_vertex_iter_end;
1059
1060                 BLI_pbvh_node_mark_update(nodes[n]);
1061         }
1062 }
1063
1064 static void do_grab_brush(Sculpt *sd, SculptSession *ss, PBVHNode **nodes, int totnode)
1065 {
1066         Brush *brush = paint_brush(&sd->paint);
1067         float bstrength= ss->cache->bstrength;
1068         float grab_delta[3];
1069         int n;
1070         
1071         copy_v3_v3(grab_delta, ss->cache->grab_delta_symmetry);
1072
1073         #pragma omp parallel for private(n) schedule(static)
1074         for(n=0; n<totnode; n++) {
1075                 PBVHVertexIter vd;
1076                 SculptBrushTest test;
1077                 float (*origco)[3];
1078                 
1079                 origco= sculpt_undo_push_node(ss, nodes[n])->co;
1080                 sculpt_brush_test_init(ss, &test);
1081
1082                 BLI_pbvh_vertex_iter_begin(ss->tree, nodes[n], vd, PBVH_ITER_UNIQUE) {
1083                         if(sculpt_brush_test(&test, origco[vd.i])) {
1084                                 float fade = tex_strength(ss, brush, origco[vd.i], test.dist)*bstrength;
1085                                 float add[3]= {vd.co[0]+fade*grab_delta[0],
1086                                                            vd.co[1]+fade*grab_delta[1],
1087                                                            vd.co[2]+fade*grab_delta[2]};
1088
1089                                 sculpt_clip(sd, ss, vd.co, add);                        
1090                                 if(vd.mvert) vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
1091                         }
1092                 }
1093                 BLI_pbvh_vertex_iter_end;
1094
1095                 BLI_pbvh_node_mark_update(nodes[n]);
1096         }
1097 }
1098
1099 static void do_layer_brush(Sculpt *sd, SculptSession *ss, PBVHNode **nodes, int totnode)
1100 {
1101         Brush *brush = paint_brush(&sd->paint);
1102         float bstrength= ss->cache->bstrength;
1103         float area_normal[3], offset[3];
1104         float lim= ss->cache->radius / 4;
1105         int n;
1106
1107         /* XXX not working yet for multires */
1108         if(ss->multires)
1109                 return;
1110
1111         if(ss->cache->flip)
1112                 lim = -lim;
1113
1114         calc_area_normal(sd, ss, area_normal, nodes, totnode);
1115
1116         offset[0]= ss->cache->scale[0]*area_normal[0];
1117         offset[1]= ss->cache->scale[1]*area_normal[1];
1118         offset[2]= ss->cache->scale[2]*area_normal[2];
1119
1120         #pragma omp parallel for private(n) schedule(static)
1121         for(n=0; n<totnode; n++) {
1122                 PBVHVertexIter vd;
1123                 SculptBrushTest test;
1124                 float (*origco)[3];
1125                 
1126                 origco= sculpt_undo_push_node(ss, nodes[n])->co;
1127                 sculpt_brush_test_init(ss, &test);
1128
1129                 BLI_pbvh_vertex_iter_begin(ss->tree, nodes[n], vd, PBVH_ITER_UNIQUE) {
1130                         if(sculpt_brush_test(&test, vd.co)) {
1131                                 float fade = tex_strength(ss, brush, vd.co, test.dist)*bstrength;
1132                                 int index= vd.vert_indices[vd.i];
1133                                 float *disp= &ss->layer_disps[index];
1134                                 float val[3];
1135                                 
1136                                 *disp+= fade;
1137                                 
1138                                 /* Don't let the displacement go past the limit */
1139                                 if((lim < 0 && *disp < lim) || (lim > 0 && *disp > lim))
1140                                         *disp = lim;
1141                                 
1142                                 if(ss->layer_co && (brush->flag & BRUSH_PERSISTENT)) {
1143                                         /* persistent base */
1144                                         val[0] = ss->layer_co[index][0] + (*disp)*offset[0];
1145                                         val[1] = ss->layer_co[index][1] + (*disp)*offset[1];
1146                                         val[2] = ss->layer_co[index][2] + (*disp)*offset[2];
1147                                 }
1148                                 else {
1149                                         val[0] = origco[vd.i][0] + (*disp)*offset[0];
1150                                         val[1] = origco[vd.i][1] + (*disp)*offset[1];
1151                                         val[2] = origco[vd.i][2] + (*disp)*offset[2];
1152                                 }
1153
1154                                 sculpt_clip(sd, ss, vd.co, val);
1155                                 if(vd.mvert) vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
1156                         }
1157                 }
1158                 BLI_pbvh_vertex_iter_end;
1159
1160                 BLI_pbvh_node_mark_update(nodes[n]);
1161         }
1162 }
1163
1164 static void do_inflate_brush(Sculpt *sd, SculptSession *ss, PBVHNode **nodes, int totnode)
1165 {
1166         Brush *brush = paint_brush(&sd->paint);
1167         float bstrength= ss->cache->bstrength;
1168         int n;
1169
1170         #pragma omp parallel for private(n) schedule(static)
1171         for(n=0; n<totnode; n++) {
1172                 PBVHVertexIter vd;
1173                 SculptBrushTest test;
1174                 
1175                 sculpt_undo_push_node(ss, nodes[n]);
1176                 sculpt_brush_test_init(ss, &test);
1177
1178                 BLI_pbvh_vertex_iter_begin(ss->tree, nodes[n], vd, PBVH_ITER_UNIQUE) {
1179                         if(sculpt_brush_test(&test, vd.co)) {
1180                                 float fade = tex_strength(ss, brush, vd.co, test.dist)*bstrength;
1181                                 float add[3];
1182
1183                                 if(vd.fno) copy_v3_v3(add, vd.fno);
1184                                 else normal_short_to_float_v3(add, vd.no);
1185                                 
1186                                 mul_v3_fl(add, fade * ss->cache->radius);
1187                                 add[0]*= ss->cache->scale[0];
1188                                 add[1]*= ss->cache->scale[1];
1189                                 add[2]*= ss->cache->scale[2];
1190                                 add_v3_v3v3(add, add, vd.co);
1191                                 
1192                                 sculpt_clip(sd, ss, vd.co, add);
1193                                 if(vd.mvert) vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
1194                         }
1195                 }
1196                 BLI_pbvh_vertex_iter_end;
1197
1198                 BLI_pbvh_node_mark_update(nodes[n]);
1199         }
1200 }
1201
1202 static void calc_flatten_center(Sculpt *sd, SculptSession *ss, PBVHNode **nodes, int totnode, float co[3])
1203 {
1204         float outer_dist[FLATTEN_SAMPLE_SIZE];
1205         float outer_co[FLATTEN_SAMPLE_SIZE][3];
1206         int i, n;
1207         
1208         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i) {
1209                 zero_v3(outer_co[i]);
1210                 outer_dist[i]= -1.0f;
1211         }
1212                 
1213         #pragma omp parallel for private(n) schedule(static)
1214         for(n=0; n<totnode; n++) {
1215                 PBVHVertexIter vd;
1216                 SculptBrushTest test;
1217                 int j;
1218                 
1219                 sculpt_undo_push_node(ss, nodes[n]);
1220                 sculpt_brush_test_init(ss, &test);
1221
1222                 BLI_pbvh_vertex_iter_begin(ss->tree, nodes[n], vd, PBVH_ITER_UNIQUE) {
1223                         if(sculpt_brush_test(&test, vd.co)) {
1224                                 for(j = 0; j < FLATTEN_SAMPLE_SIZE; ++j) {
1225                                         if(test.dist > outer_dist[j]) {
1226                                                 copy_v3_v3(outer_co[j], vd.co);
1227                                                 outer_dist[j] = test.dist;
1228                                                 break;
1229                                         }
1230                                 }
1231                         }
1232                 }
1233                 BLI_pbvh_vertex_iter_end;
1234
1235                 BLI_pbvh_node_mark_update(nodes[n]);
1236         }
1237         
1238         co[0] = co[1] = co[2] = 0.0f;
1239         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
1240                 if(outer_dist[i] >= 0.0f)
1241                         add_v3_v3v3(co, co, outer_co[i]);
1242         mul_v3_fl(co, 1.0f / FLATTEN_SAMPLE_SIZE);
1243 }
1244
1245 /* Projects a point onto a plane along the plane's normal */
1246 static void point_plane_project(float intr[3], float co[3], float plane_normal[3], float plane_center[3])
1247 {
1248         float p1[3], sub1[3], sub2[3];
1249
1250         /* Find the intersection between squash-plane and vertex (along the area normal) */
1251         sub_v3_v3v3(p1, co, plane_normal);
1252         sub_v3_v3v3(sub1, plane_center, p1);
1253         sub_v3_v3v3(sub2, co, p1);
1254         sub_v3_v3v3(intr, co, p1);
1255         mul_v3_fl(intr, dot_v3v3(plane_normal, sub1) / dot_v3v3(plane_normal, sub2));
1256         add_v3_v3v3(intr, intr, p1);
1257 }
1258
1259 static int plane_point_side(float co[3], float plane_normal[3], float plane_center[3], int flip)
1260 {
1261         float delta[3];
1262         float d;
1263
1264         sub_v3_v3v3(delta, co, plane_center);
1265         d = dot_v3v3(plane_normal, delta);
1266
1267         if(flip)
1268                 d = -d;
1269
1270         return d <= 0.0f;
1271 }
1272
1273 static void do_flatten_clay_brush(Sculpt *sd, SculptSession *ss, PBVHNode **nodes, int totnode, int clay)
1274 {
1275         /* area_normal and cntr define the plane towards which vertices are squashed */
1276         Brush *brush = paint_brush(&sd->paint);
1277         float bstrength= ss->cache->bstrength;
1278         float area_normal[3];
1279         float cntr[3], cntr2[3], bstr = 0;
1280         int n, flip = 0;
1281
1282         calc_area_normal(sd, ss, area_normal, nodes, totnode);
1283         calc_flatten_center(sd, ss, nodes, totnode, cntr);
1284
1285         if(clay) {
1286                 bstr= brush_strength(sd, ss->cache);
1287                 /* Limit clay application to here */
1288                 cntr2[0]=cntr[0]+area_normal[0]*bstr*ss->cache->scale[0];
1289                 cntr2[1]=cntr[1]+area_normal[1]*bstr*ss->cache->scale[1];
1290                 cntr2[2]=cntr[2]+area_normal[2]*bstr*ss->cache->scale[2];
1291                 flip = bstr < 0;
1292         }
1293
1294         #pragma omp parallel for private(n) schedule(static)
1295         for(n=0; n<totnode; n++) {
1296                 PBVHVertexIter vd;
1297                 SculptBrushTest test;
1298                 
1299                 sculpt_undo_push_node(ss, nodes[n]);
1300                 sculpt_brush_test_init(ss, &test);
1301
1302                 BLI_pbvh_vertex_iter_begin(ss->tree, nodes[n], vd, PBVH_ITER_UNIQUE) {
1303                         if(sculpt_brush_test(&test, vd.co)) {
1304                                 float intr[3], val[3];
1305                                 
1306                                 if(!clay || plane_point_side(vd.co, area_normal, cntr2, flip)) {
1307                                         const float fade = tex_strength(ss, brush, vd.co, test.dist)*bstrength;
1308
1309                                         /* Find the intersection between squash-plane and vertex (along the area normal) */             
1310                                         point_plane_project(intr, vd.co, area_normal, cntr);
1311
1312                                         sub_v3_v3v3(val, intr, vd.co);
1313
1314                                         if(clay) {
1315                                                 if(bstr > FLT_EPSILON)
1316                                                         mul_v3_fl(val, fade / bstr);
1317                                                 else
1318                                                         mul_v3_fl(val, fade);
1319                                                 /* Clay displacement */
1320                                                 val[0]+=area_normal[0] * ss->cache->scale[0]*fade;
1321                                                 val[1]+=area_normal[1] * ss->cache->scale[1]*fade;
1322                                                 val[2]+=area_normal[2] * ss->cache->scale[2]*fade;
1323                                         }
1324                                         else
1325                                                 mul_v3_fl(val, fabs(fade));
1326
1327                                         add_v3_v3v3(val, val, vd.co);
1328
1329                                         sculpt_clip(sd, ss, vd.co, val);
1330                                         if(vd.mvert) vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
1331                                 }
1332                         }
1333                 }
1334                 BLI_pbvh_vertex_iter_end;
1335
1336                 BLI_pbvh_node_mark_update(nodes[n]);
1337         }
1338 }
1339
1340 static void do_brush_action(Sculpt *sd, SculptSession *ss, StrokeCache *cache)
1341 {
1342         SculptSearchSphereData data;
1343         Brush *brush = paint_brush(&sd->paint);
1344         PBVHNode **nodes= NULL;
1345         int totnode;
1346
1347         data.ss = ss;
1348         data.sd = sd;
1349         data.radius_squared = ss->cache->radius * ss->cache->radius;
1350
1351         /* Build a list of all nodes that are potentially within the brush's
1352            area of influence */
1353         if(brush->sculpt_tool == SCULPT_TOOL_GRAB) {
1354                 data.original= 1;
1355                 BLI_pbvh_search_gather(ss->tree, sculpt_search_sphere_cb, &data,
1356                                 &nodes, &totnode);
1357
1358                 if(cache->first_time)
1359                         copy_v3_v3(ss->cache->grab_active_location[ss->cache->symmetry], ss->cache->location);
1360                 else
1361                         copy_v3_v3(ss->cache->location, ss->cache->grab_active_location[ss->cache->symmetry]);
1362         }
1363         else {
1364                 BLI_pbvh_search_gather(ss->tree, sculpt_search_sphere_cb, &data,
1365                         &nodes, &totnode);
1366         }
1367
1368         /* Only act if some verts are inside the brush area */
1369         if(totnode) {
1370                 /* Apply one type of brush action */
1371                 switch(brush->sculpt_tool){
1372                 case SCULPT_TOOL_DRAW:
1373                         do_draw_brush(sd, ss, nodes, totnode);
1374                         break;
1375                 case SCULPT_TOOL_SMOOTH:
1376                         do_smooth_brush(sd, ss, nodes, totnode);
1377                         break;
1378                 case SCULPT_TOOL_PINCH:
1379                         do_pinch_brush(sd, ss, nodes, totnode);
1380                         break;
1381                 case SCULPT_TOOL_INFLATE:
1382                         do_inflate_brush(sd, ss, nodes, totnode);
1383                         break;
1384                 case SCULPT_TOOL_GRAB:
1385                         do_grab_brush(sd, ss, nodes, totnode);
1386                         break;
1387                 case SCULPT_TOOL_LAYER:
1388                         do_layer_brush(sd, ss, nodes, totnode);
1389                         break;
1390                 case SCULPT_TOOL_FLATTEN:
1391                         do_flatten_clay_brush(sd, ss, nodes, totnode, 0);
1392                         break;
1393                 case SCULPT_TOOL_CLAY:
1394                         do_flatten_clay_brush(sd, ss, nodes, totnode, 1);
1395                         break;
1396                 }
1397         
1398                 /* copy the modified vertices from mesh to the active key */
1399                 if(ss->kb) mesh_to_key(ss->ob->data, ss->kb);
1400                 
1401                 if(nodes)
1402                         MEM_freeN(nodes);
1403         }       
1404 }
1405
1406 /* Flip all the editdata across the axis/axes specified by symm. Used to
1407    calculate multiple modifications to the mesh when symmetry is enabled. */
1408 static void calc_brushdata_symm(StrokeCache *cache, const char symm)
1409 {
1410         flip_coord(cache->location, cache->true_location, symm);
1411         flip_coord(cache->view_normal_symmetry, cache->view_normal, symm);
1412         flip_coord(cache->grab_delta_symmetry, cache->grab_delta, symm);
1413         cache->symmetry= symm;
1414 }
1415
1416 static void do_symmetrical_brush_actions(Sculpt *sd, SculptSession *ss)
1417 {
1418         StrokeCache *cache = ss->cache;
1419         const char symm = sd->flags & 7;
1420         int i;
1421
1422         copy_v3_v3(cache->location, cache->true_location);
1423         copy_v3_v3(cache->grab_delta_symmetry, cache->grab_delta);
1424         cache->symmetry = 0;
1425         cache->bstrength = brush_strength(sd, cache);
1426         do_brush_action(sd, ss, cache);
1427
1428         for(i = 1; i <= symm; ++i) {
1429                 if(symm & i && (symm != 5 || i != 3) && (symm != 6 || (i != 3 && i != 5))) {
1430                         calc_brushdata_symm(cache, i);
1431                         do_brush_action(sd, ss, cache);
1432                 }
1433         }
1434
1435         cache->first_time = 0;
1436 }
1437
1438 static void sculpt_update_tex(Sculpt *sd, SculptSession *ss)
1439 {
1440         Brush *brush = paint_brush(&sd->paint);
1441
1442         if(ss->texcache) {
1443                 MEM_freeN(ss->texcache);
1444                 ss->texcache= NULL;
1445         }
1446
1447         /* Need to allocate a bigger buffer for bigger brush size */
1448         ss->texcache_side = brush->size * 2;
1449         if(!ss->texcache || ss->texcache_side > ss->texcache_actual) {
1450                 ss->texcache = brush_gen_texture_cache(brush, brush->size);
1451                 ss->texcache_actual = ss->texcache_side;
1452         }
1453 }
1454
1455 /* Checks whether full update mode (slower) needs to be used to work with modifiers */
1456 static int sculpt_modifiers_active(Scene *scene, Object *ob)
1457 {
1458         ModifierData *md;
1459         
1460         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1461                 if(modifier_isEnabled(scene, md, eModifierMode_Realtime))
1462                         if(!ELEM(md->type, eModifierType_Multires, eModifierType_ShapeKey))
1463                                 return 1;
1464         }
1465         
1466         return 0;
1467 }
1468
1469 /* Sculpt mode handles multires differently from regular meshes, but only if
1470    it's the last modifier on the stack and it is not on the first level */
1471 struct MultiresModifierData *sculpt_multires_active(Object *ob)
1472 {
1473         ModifierData *md, *nmd;
1474         
1475         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1476                 if(md->type == eModifierType_Multires) {
1477                         MultiresModifierData *mmd= (MultiresModifierData*)md;
1478
1479                         /* Check if any of the modifiers after multires are active
1480                          * if not it can use the multires struct */
1481                         for (nmd= md->next; nmd; nmd= nmd->next)
1482                                 if(nmd->mode & eModifierMode_Realtime)
1483                                         break;
1484
1485                         if(!nmd && mmd->sculptlvl > 0)
1486                                 return mmd;
1487                 }
1488         }
1489
1490         return NULL;
1491 }
1492
1493 void sculpt_key_to_mesh(KeyBlock *kb, Object *ob)
1494 {
1495         Mesh *me= ob->data;
1496
1497         key_to_mesh(kb, me);
1498         mesh_calc_normals(me->mvert, me->totvert, me->mface, me->totface, NULL);
1499 }
1500
1501 void sculpt_mesh_to_key(Object *ob, KeyBlock *kb)
1502 {
1503         Mesh *me= ob->data;
1504
1505         mesh_to_key(me, kb);
1506 }
1507
1508 void sculpt_update_mesh_elements(Scene *scene, Object *ob, int need_fmap)
1509 {
1510         DerivedMesh *dm = mesh_get_derived_final(scene, ob, 0);
1511         SculptSession *ss = ob->sculpt;
1512         
1513         ss->ob= ob;
1514
1515         if((ob->shapeflag & OB_SHAPE_LOCK) && !sculpt_multires_active(ob)) {
1516                 ss->kb= ob_get_keyblock(ob);
1517                 ss->refkb= ob_get_reference_keyblock(ob);
1518         }
1519         else {
1520                 ss->kb= NULL;
1521                 ss->refkb= NULL;
1522         }
1523
1524         /* need to make PBVH with shape key coordinates */
1525         if(ss->kb) sculpt_key_to_mesh(ss->kb, ss->ob);
1526
1527         if((ss->multires = sculpt_multires_active(ob))) {
1528                 ss->totvert = dm->getNumVerts(dm);
1529                 ss->totface = dm->getNumFaces(dm);
1530                 ss->mvert= NULL;
1531                 ss->mface= NULL;
1532                 ss->face_normals= NULL;
1533         }
1534         else {
1535                 Mesh *me = get_mesh(ob);
1536                 ss->totvert = me->totvert;
1537                 ss->totface = me->totface;
1538                 ss->mvert = me->mvert;
1539                 ss->mface = me->mface;
1540                 ss->face_normals = NULL;
1541         }
1542
1543         ss->tree = dm->getPBVH(ob, dm);
1544         ss->fmap = (need_fmap && dm->getFaceMap)? dm->getFaceMap(ob, dm): NULL;
1545 }
1546
1547 static int sculpt_mode_poll(bContext *C)
1548 {
1549         Object *ob = CTX_data_active_object(C);
1550         return ob && ob->mode & OB_MODE_SCULPT;
1551 }
1552
1553 int sculpt_poll(bContext *C)
1554 {
1555         return sculpt_mode_poll(C) && paint_poll(C);
1556 }
1557
1558 static char *sculpt_tool_name(Sculpt *sd)
1559 {
1560         Brush *brush = paint_brush(&sd->paint);
1561
1562         switch(brush->sculpt_tool) {
1563         case SCULPT_TOOL_DRAW:
1564                 return "Draw Brush"; break;
1565         case SCULPT_TOOL_SMOOTH:
1566                 return "Smooth Brush"; break;
1567         case SCULPT_TOOL_PINCH:
1568                 return "Pinch Brush"; break;
1569         case SCULPT_TOOL_INFLATE:
1570                 return "Inflate Brush"; break;
1571         case SCULPT_TOOL_GRAB:
1572                 return "Grab Brush"; break;
1573         case SCULPT_TOOL_LAYER:
1574                 return "Layer Brush"; break;
1575         case SCULPT_TOOL_FLATTEN:
1576                 return "Flatten Brush"; break;
1577         default:
1578                 return "Sculpting"; break;
1579         }
1580 }
1581
1582 /**** Radial control ****/
1583 static int sculpt_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1584 {
1585         Paint *p = paint_get_active(CTX_data_scene(C));
1586         Brush *brush = paint_brush(p);
1587
1588         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1589         p->paint_cursor = NULL;
1590         brush_radial_control_invoke(op, brush, 1);
1591         return WM_radial_control_invoke(C, op, event);
1592 }
1593
1594 static int sculpt_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1595 {
1596         int ret = WM_radial_control_modal(C, op, event);
1597         if(ret != OPERATOR_RUNNING_MODAL)
1598                 paint_cursor_start(C, sculpt_poll);
1599         return ret;
1600 }
1601
1602 static int sculpt_radial_control_exec(bContext *C, wmOperator *op)
1603 {
1604         Brush *brush = paint_brush(&CTX_data_tool_settings(C)->sculpt->paint);
1605         int ret = brush_radial_control_exec(op, brush, 1);
1606
1607         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1608
1609         return ret;
1610 }
1611
1612 static void SCULPT_OT_radial_control(wmOperatorType *ot)
1613 {
1614         WM_OT_radial_control_partial(ot);
1615
1616         ot->name= "Sculpt Radial Control";
1617         ot->idname= "SCULPT_OT_radial_control";
1618
1619         ot->invoke= sculpt_radial_control_invoke;
1620         ot->modal= sculpt_radial_control_modal;
1621         ot->exec= sculpt_radial_control_exec;
1622         ot->poll= sculpt_poll;
1623
1624         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1625 }
1626
1627 /**** Operator for applying a stroke (various attributes including mouse path)
1628       using the current brush. ****/
1629
1630 static float unproject_brush_radius(Object *ob, ViewContext *vc, float center[3], float offset)
1631 {
1632         float delta[3], scale, loc[3];
1633
1634         mul_v3_m4v3(loc, ob->obmat, center);
1635
1636         initgrabz(vc->rv3d, loc[0], loc[1], loc[2]);
1637         window_to_3d_delta(vc->ar, delta, offset, 0);
1638
1639         scale= fabsf(mat4_to_scale(ob->obmat));
1640         scale= (scale == 0.0f)? 1.0f: scale;
1641
1642         return len_v3(delta)/scale;
1643 }
1644
1645 static void sculpt_cache_free(StrokeCache *cache)
1646 {
1647         if(cache->face_norms)
1648                 MEM_freeN(cache->face_norms);
1649         if(cache->mats)
1650                 MEM_freeN(cache->mats);
1651         MEM_freeN(cache);
1652 }
1653
1654 /* Initialize the stroke cache invariants from operator properties */
1655 static void sculpt_update_cache_invariants(Sculpt *sd, SculptSession *ss, bContext *C, wmOperator *op)
1656 {
1657         StrokeCache *cache = MEM_callocN(sizeof(StrokeCache), "stroke cache");
1658         Brush *brush = paint_brush(&sd->paint);
1659         ViewContext *vc = paint_stroke_view_context(op->customdata);
1660         int i;
1661
1662         ss->cache = cache;
1663
1664         RNA_float_get_array(op->ptr, "scale", cache->scale);
1665         cache->flag = RNA_int_get(op->ptr, "flag");
1666         RNA_float_get_array(op->ptr, "clip_tolerance", cache->clip_tolerance);
1667         RNA_float_get_array(op->ptr, "initial_mouse", cache->initial_mouse);
1668
1669         copy_v2_v2(cache->mouse, cache->initial_mouse);
1670         copy_v2_v2(cache->tex_mouse, cache->initial_mouse);
1671
1672         /* Truly temporary data that isn't stored in properties */
1673
1674         cache->vc = vc;
1675         cache->brush = brush;
1676
1677         cache->mats = MEM_callocN(sizeof(bglMats), "sculpt bglMats");
1678         view3d_get_transformation(vc->ar, vc->rv3d, vc->obact, cache->mats);
1679
1680         /* Initialize layer brush displacements and persistent coords */
1681         if(brush->sculpt_tool == SCULPT_TOOL_LAYER && !ss->multires) {
1682                 if(!ss->layer_disps || !(brush->flag & BRUSH_PERSISTENT)) {
1683                         if(ss->layer_disps)
1684                                 MEM_freeN(ss->layer_disps);
1685                         ss->layer_disps = MEM_callocN(sizeof(float) * ss->totvert, "layer brush displacements");
1686                 }
1687                 if(!ss->layer_co && (brush->flag & BRUSH_PERSISTENT)) {
1688                         if(!ss->layer_co)
1689                                 ss->layer_co= MEM_mallocN(sizeof(float) * 3 * ss->totvert,
1690                                                                        "sculpt mesh vertices copy");
1691                         for(i = 0; i < ss->totvert; ++i)
1692                                 copy_v3_v3(ss->layer_co[i], ss->mvert[i].co);
1693                 }
1694         }
1695
1696         /* Make copies of the mesh vertex locations and normals for some tools */
1697         if(brush->flag & BRUSH_ANCHORED) {
1698                 if(ss->face_normals) {
1699                         float *fn = ss->face_normals;
1700                         cache->face_norms= MEM_mallocN(sizeof(float) * 3 * ss->totface, "Sculpt face norms");
1701                         for(i = 0; i < ss->totface; ++i, fn += 3)
1702                                 copy_v3_v3(cache->face_norms[i], fn);
1703                 }
1704
1705                 cache->original = 1;
1706         }
1707
1708         if(ELEM3(brush->sculpt_tool, SCULPT_TOOL_DRAW, SCULPT_TOOL_LAYER, SCULPT_TOOL_INFLATE))
1709                 if(!(brush->flag & BRUSH_ACCUMULATE))
1710                         cache->original = 1;
1711
1712         cache->rotation = 0;
1713         cache->first_time = 1;
1714 }
1715
1716 /* Initialize the stroke cache variants from operator properties */
1717 static void sculpt_update_cache_variants(Sculpt *sd, SculptSession *ss, struct PaintStroke *stroke, PointerRNA *ptr)
1718 {
1719         StrokeCache *cache = ss->cache;
1720         Brush *brush = paint_brush(&sd->paint);
1721         
1722         int dx, dy;
1723
1724         if(!(brush->flag & BRUSH_ANCHORED) || cache->first_time)
1725                 RNA_float_get_array(ptr, "location", cache->true_location);
1726         cache->flip = RNA_boolean_get(ptr, "flip");
1727         RNA_float_get_array(ptr, "mouse", cache->mouse);
1728         cache->pressure = RNA_float_get(ptr, "pressure");
1729         
1730         /* Truly temporary data that isn't stored in properties */
1731
1732         cache->previous_pixel_radius = cache->pixel_radius;
1733         cache->pixel_radius = brush->size;
1734
1735         if(cache->first_time)
1736                 cache->initial_radius = unproject_brush_radius(ss->ob, cache->vc, cache->true_location, brush->size);
1737
1738         if(brush->flag & BRUSH_SIZE_PRESSURE && brush->sculpt_tool != SCULPT_TOOL_GRAB) {
1739                 cache->pixel_radius *= cache->pressure;
1740                 cache->radius = cache->initial_radius * cache->pressure;
1741         }
1742         else
1743                 cache->radius = cache->initial_radius;
1744
1745         if(!(brush->flag & BRUSH_ANCHORED))
1746                 copy_v2_v2(cache->tex_mouse, cache->mouse);
1747
1748         if(brush->flag & BRUSH_ANCHORED) {
1749                 dx = cache->mouse[0] - cache->initial_mouse[0];
1750                 dy = cache->mouse[1] - cache->initial_mouse[1];
1751                 cache->pixel_radius = sqrt(dx*dx + dy*dy);
1752                 cache->radius = unproject_brush_radius(ss->ob, paint_stroke_view_context(stroke),
1753                                                        cache->true_location, cache->pixel_radius);
1754                 cache->rotation = atan2(dy, dx);
1755         }
1756         else if(brush->flag & BRUSH_RAKE) {
1757                 int update;
1758
1759                 dx = cache->last_rake[0] - cache->mouse[0];
1760                 dy = cache->last_rake[1] - cache->mouse[1];
1761
1762                 update = dx*dx + dy*dy > 100;
1763
1764                 /* To prevent jitter, only update the angle if the mouse has moved over 10 pixels */
1765                 if(update && !cache->first_time)
1766                         cache->rotation = M_PI_2 + atan2(dy, dx);
1767
1768                 if(update || cache->first_time) {
1769                         cache->last_rake[0] = cache->mouse[0];
1770                         cache->last_rake[1] = cache->mouse[1];
1771                 }
1772         }
1773
1774         /* Find the grab delta */
1775         if(brush->sculpt_tool == SCULPT_TOOL_GRAB) {
1776                 float grab_location[3];
1777
1778                 if(cache->first_time)
1779                         copy_v3_v3(cache->orig_grab_location, cache->true_location);
1780
1781                 initgrabz(cache->vc->rv3d, cache->orig_grab_location[0], cache->orig_grab_location[1], cache->orig_grab_location[2]);
1782                 window_to_3d_delta(cache->vc->ar, grab_location, cache->mouse[0], cache->mouse[1]);
1783
1784                 if(!cache->first_time)
1785                         sub_v3_v3v3(cache->grab_delta, grab_location, cache->old_grab_location);
1786                 copy_v3_v3(cache->old_grab_location, grab_location);
1787         }
1788 }
1789
1790 static void sculpt_stroke_modifiers_check(bContext *C, SculptSession *ss)
1791 {
1792         Scene *scene= CTX_data_scene(C);
1793
1794         if(sculpt_modifiers_active(scene, ss->ob)) {
1795                 Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
1796                 Brush *brush = paint_brush(&sd->paint);
1797
1798                 sculpt_update_mesh_elements(CTX_data_scene(C), ss->ob, brush->sculpt_tool == SCULPT_TOOL_SMOOTH);
1799         }
1800 }
1801
1802 typedef struct {
1803         SculptSession *ss;
1804         float *ray_start, *ray_normal;
1805         int hit;
1806         float dist;
1807         int original;
1808 } SculptRaycastData;
1809
1810 void sculpt_raycast_cb(PBVHNode *node, void *data_v)
1811 {
1812         SculptRaycastData *srd = data_v;
1813         float (*origco)[3]= NULL;
1814
1815         if(srd->original && srd->ss->cache) {
1816                 /* intersect with coordinates from before we started stroke */
1817                 SculptUndoNode *unode= sculpt_undo_get_node(srd->ss, node);
1818                 origco= (unode)? unode->co: NULL;
1819         }
1820
1821         srd->hit |= BLI_pbvh_node_raycast(srd->ss->tree, node, origco,
1822                 srd->ray_start, srd->ray_normal, &srd->dist);
1823 }
1824
1825 /* Do a raycast in the tree to find the 3d brush location
1826    (This allows us to ignore the GL depth buffer)
1827    Returns 0 if the ray doesn't hit the mesh, non-zero otherwise
1828  */
1829 int sculpt_stroke_get_location(bContext *C, struct PaintStroke *stroke, float out[3], float mouse[2])
1830 {
1831         ViewContext *vc = paint_stroke_view_context(stroke);
1832         SculptSession *ss= vc->obact->sculpt;
1833         StrokeCache *cache= ss->cache;
1834         float ray_start[3], ray_end[3], ray_normal[3], dist;
1835         float obimat[4][4];
1836         float mval[2] = {mouse[0] - vc->ar->winrct.xmin,
1837                          mouse[1] - vc->ar->winrct.ymin};
1838         SculptRaycastData srd;
1839
1840         sculpt_stroke_modifiers_check(C, ss);
1841
1842         viewline(vc->ar, vc->v3d, mval, ray_start, ray_end);
1843         sub_v3_v3v3(ray_normal, ray_end, ray_start);
1844         dist= normalize_v3(ray_normal);
1845
1846         invert_m4_m4(obimat, ss->ob->obmat);
1847         mul_m4_v3(obimat, ray_start);
1848         mul_mat3_m4_v3(obimat, ray_normal);
1849         normalize_v3(ray_normal);
1850
1851         srd.ss = vc->obact->sculpt;
1852         srd.ray_start = ray_start;
1853         srd.ray_normal = ray_normal;
1854         srd.dist = dist;
1855         srd.hit = 0;
1856         srd.original = (cache)? cache->original: 0;
1857         BLI_pbvh_raycast(ss->tree, sculpt_raycast_cb, &srd,
1858                      ray_start, ray_normal, srd.original);
1859         
1860         copy_v3_v3(out, ray_normal);
1861         mul_v3_fl(out, srd.dist);
1862         add_v3_v3v3(out, out, ray_start);
1863
1864         return srd.hit;
1865 }
1866
1867 /* Initialize stroke operator properties */
1868 static void sculpt_brush_stroke_init_properties(bContext *C, wmOperator *op, wmEvent *event, SculptSession *ss)
1869 {
1870         Object *ob= CTX_data_active_object(C);
1871         ModifierData *md;
1872         float scale[3], clip_tolerance[3] = {0,0,0};
1873         float mouse[2];
1874         int flag = 0;
1875
1876         /* Set scaling adjustment */
1877         scale[0] = 1.0f / ob->size[0];
1878         scale[1] = 1.0f / ob->size[1];
1879         scale[2] = 1.0f / ob->size[2];
1880         RNA_float_set_array(op->ptr, "scale", scale);
1881
1882         /* Initialize mirror modifier clipping */
1883         for(md= ob->modifiers.first; md; md= md->next) {
1884                 if(md->type==eModifierType_Mirror && (md->mode & eModifierMode_Realtime)) {
1885                         const MirrorModifierData *mmd = (MirrorModifierData*) md;
1886                         
1887                         /* Mark each axis that needs clipping along with its tolerance */
1888                         if(mmd->flag & MOD_MIR_CLIPPING) {
1889                                 flag |= CLIP_X << mmd->axis;
1890                                 if(mmd->tolerance > clip_tolerance[mmd->axis])
1891                                         clip_tolerance[mmd->axis] = mmd->tolerance;
1892                         }
1893                 }
1894         }
1895         RNA_int_set(op->ptr, "flag", flag);
1896         RNA_float_set_array(op->ptr, "clip_tolerance", clip_tolerance);
1897
1898         /* Initial mouse location */
1899         mouse[0] = event->x;
1900         mouse[1] = event->y;
1901         RNA_float_set_array(op->ptr, "initial_mouse", mouse);
1902 }
1903
1904 static int sculpt_brush_stroke_init(bContext *C, ReportList *reports)
1905 {
1906         Scene *scene= CTX_data_scene(C);
1907         Object *ob= CTX_data_active_object(C);
1908         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
1909         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1910         Brush *brush = paint_brush(&sd->paint);
1911
1912         if(ob_get_key(ob) && !(ob->shapeflag & OB_SHAPE_LOCK)) {
1913                 BKE_report(reports, RPT_ERROR, "Shape key sculpting requires a locked shape.");
1914                 return 0;
1915         }
1916
1917         view3d_operator_needs_opengl(C);
1918
1919         /* TODO: Shouldn't really have to do this at the start of every
1920            stroke, but sculpt would need some sort of notification when
1921            changes are made to the texture. */
1922         sculpt_update_tex(sd, ss);
1923
1924         sculpt_update_mesh_elements(scene, ob, brush->sculpt_tool == SCULPT_TOOL_SMOOTH);
1925
1926         return 1;
1927 }
1928
1929 static void sculpt_restore_mesh(Sculpt *sd, SculptSession *ss)
1930 {
1931         StrokeCache *cache = ss->cache;
1932         Brush *brush = paint_brush(&sd->paint);
1933         int i;
1934
1935         /* Restore the mesh before continuing with anchored stroke */
1936         if(brush->flag & BRUSH_ANCHORED) {
1937                 PBVHNode **nodes;
1938                 int n, totnode;
1939
1940                 BLI_pbvh_search_gather(ss->tree, NULL, NULL, &nodes, &totnode);
1941
1942                 #pragma omp parallel for private(n) schedule(static)
1943                 for(n=0; n<totnode; n++) {
1944                         SculptUndoNode *unode;
1945                         
1946                         unode= sculpt_undo_get_node(ss, nodes[n]);
1947                         if(unode) {
1948                                 PBVHVertexIter vd;
1949
1950                                 BLI_pbvh_vertex_iter_begin(ss->tree, nodes[n], vd, PBVH_ITER_UNIQUE) {
1951                                         copy_v3_v3(vd.co, unode->co[vd.i]);
1952                                         if(vd.no) VECCOPY(vd.no, unode->no[vd.i])
1953                                         else normal_short_to_float_v3(vd.fno, unode->no[vd.i]);
1954                                 }
1955                                 BLI_pbvh_vertex_iter_end;
1956                         }
1957                 }
1958
1959                 if(ss->face_normals) {
1960                         float *fn = ss->face_normals;
1961                         for(i = 0; i < ss->totface; ++i, fn += 3)
1962                                 copy_v3_v3(fn, cache->face_norms[i]);
1963                 }
1964
1965                 if(brush->sculpt_tool == SCULPT_TOOL_LAYER && !ss->multires)
1966                         memset(ss->layer_disps, 0, sizeof(float) * ss->totvert);
1967         }
1968 }
1969
1970 static void sculpt_flush_update(bContext *C)
1971 {
1972         Scene *scene = CTX_data_scene(C);
1973         Object *ob = CTX_data_active_object(C);
1974         SculptSession *ss = ob->sculpt;
1975         ARegion *ar = CTX_wm_region(C);
1976         MultiresModifierData *mmd = ss->multires;
1977         int redraw = 0;
1978
1979         if(mmd)
1980                 multires_mark_as_modified(ob);
1981
1982         if(sculpt_modifiers_active(scene, ob)) {
1983                 DAG_id_flush_update(&ob->id, OB_RECALC_DATA);
1984                 ED_region_tag_redraw(ar);
1985         }
1986         else {
1987                 rcti r;
1988
1989                 BLI_pbvh_update(ss->tree, PBVH_UpdateBB, NULL);
1990                 redraw = sculpt_get_redraw_rect(ar, CTX_wm_region_view3d(C), ob, &r);
1991
1992                 if(redraw) {
1993                         r.xmin += ar->winrct.xmin + 1;
1994                         r.xmax += ar->winrct.xmin - 1;
1995                         r.ymin += ar->winrct.ymin + 1;
1996                         r.ymax += ar->winrct.ymin - 1;
1997                         
1998                         ss->partial_redraw = 1;
1999                         ED_region_tag_redraw_partial(ar, &r);
2000                 }
2001         }
2002 }
2003
2004 /* Returns whether the mouse/stylus is over the mesh (1)
2005    or over the background (0) */
2006 static int over_mesh(bContext *C, struct wmOperator *op, float x, float y)
2007 {
2008         float mouse[2] = {x, y}, co[3];
2009         
2010         return (int)sculpt_stroke_get_location(C, op->customdata, co, mouse);
2011 }
2012
2013 static int sculpt_stroke_test_start(bContext *C, struct wmOperator *op,
2014                                     wmEvent *event)
2015 {
2016         /* Don't start the stroke until mouse goes over the mesh */
2017         if(over_mesh(C, op, event->x, event->y)) {
2018                 Object *ob = CTX_data_active_object(C);
2019                 SculptSession *ss = ob->sculpt;
2020                 Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
2021
2022                 ED_view3d_init_mats_rv3d(ob, CTX_wm_region_view3d(C));
2023
2024                 sculpt_brush_stroke_init_properties(C, op, event, ss);
2025
2026                 sculpt_update_cache_invariants(sd, ss, C, op);
2027
2028                 sculpt_undo_push_begin(ss, sculpt_tool_name(sd));
2029
2030                 return 1;
2031         }
2032         else
2033                 return 0;
2034 }
2035
2036 static void sculpt_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
2037 {
2038         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
2039         SculptSession *ss = CTX_data_active_object(C)->sculpt;
2040
2041         sculpt_stroke_modifiers_check(C, ss);
2042         sculpt_update_cache_variants(sd, ss, stroke, itemptr);
2043         sculpt_restore_mesh(sd, ss);
2044         do_symmetrical_brush_actions(sd, ss);
2045
2046         /* Cleanup */
2047         sculpt_flush_update(C);
2048 }
2049
2050 static void sculpt_stroke_done(bContext *C, struct PaintStroke *stroke)
2051 {
2052         Object *ob= CTX_data_active_object(C);
2053         SculptSession *ss = ob->sculpt;
2054
2055         /* Finished */
2056         if(ss->cache) {
2057                 sculpt_stroke_modifiers_check(C, ss);
2058
2059                 sculpt_cache_free(ss->cache);
2060                 ss->cache = NULL;
2061
2062                 sculpt_undo_push_end(ss);
2063
2064                 BLI_pbvh_update(ss->tree, PBVH_UpdateOriginalBB, NULL);
2065
2066                 if(ss->refkb) sculpt_key_to_mesh(ss->refkb, ob);
2067
2068                 ss->partial_redraw = 0;
2069                 
2070                 /* try to avoid calling this, only for e.g. linked duplicates now */
2071                 if(((Mesh*)ob->data)->id.us > 1)
2072                         DAG_id_flush_update(&ob->id, OB_RECALC_DATA);
2073
2074                 WM_event_add_notifier(C, NC_OBJECT|ND_DRAW, ob);
2075         }
2076 }
2077
2078 static int sculpt_brush_stroke_invoke(bContext *C, wmOperator *op, wmEvent *event)
2079 {
2080         struct PaintStroke *stroke;
2081         int ignore_background_click;
2082
2083         if(!sculpt_brush_stroke_init(C, op->reports))
2084                 return OPERATOR_CANCELLED;
2085
2086         stroke = paint_stroke_new(C, sculpt_stroke_get_location,
2087                                   sculpt_stroke_test_start,
2088                                   sculpt_stroke_update_step,
2089                                   sculpt_stroke_done);
2090
2091         op->customdata = stroke;
2092
2093         /* For tablet rotation */
2094         ignore_background_click = RNA_boolean_get(op->ptr,
2095                                                   "ignore_background_click"); 
2096         if(ignore_background_click && !over_mesh(C, op, event->x, event->y)) {
2097                 paint_stroke_free(stroke);
2098                 return OPERATOR_PASS_THROUGH;
2099         }
2100         
2101         /* add modal handler */
2102         WM_event_add_modal_handler(C, op);
2103
2104         op->type->modal(C, op, event);
2105         
2106         return OPERATOR_RUNNING_MODAL;
2107 }
2108
2109 static int sculpt_brush_stroke_exec(bContext *C, wmOperator *op)
2110 {
2111         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
2112         SculptSession *ss = CTX_data_active_object(C)->sculpt;
2113
2114         if(!sculpt_brush_stroke_init(C, op->reports))
2115                 return OPERATOR_CANCELLED;
2116
2117         op->customdata = paint_stroke_new(C, sculpt_stroke_get_location, sculpt_stroke_test_start,
2118                                           sculpt_stroke_update_step, sculpt_stroke_done);
2119
2120         sculpt_update_cache_invariants(sd, ss, C, op);
2121
2122         paint_stroke_exec(C, op);
2123
2124         sculpt_flush_update(C);
2125         sculpt_cache_free(ss->cache);
2126
2127         return OPERATOR_FINISHED;
2128 }
2129
2130 static void SCULPT_OT_brush_stroke(wmOperatorType *ot)
2131 {
2132         ot->flag |= OPTYPE_REGISTER;
2133
2134         /* identifiers */
2135         ot->name= "Sculpt Mode";
2136         ot->idname= "SCULPT_OT_brush_stroke";
2137         
2138         /* api callbacks */
2139         ot->invoke= sculpt_brush_stroke_invoke;
2140         ot->modal= paint_stroke_modal;
2141         ot->exec= sculpt_brush_stroke_exec;
2142         ot->poll= sculpt_poll;
2143         
2144         /* flags (sculpt does own undo? (ton) */
2145         ot->flag= OPTYPE_REGISTER|OPTYPE_BLOCKING;
2146
2147         /* properties */
2148         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
2149
2150         /* If the object has a scaling factor, brushes also need to be scaled
2151            to work as expected. */
2152         RNA_def_float_vector(ot->srna, "scale", 3, NULL, 0.0f, FLT_MAX, "Scale", "", 0.0f, 1000.0f);
2153
2154         RNA_def_int(ot->srna, "flag", 0, 0, INT_MAX, "flag", "", 0, INT_MAX);
2155
2156         /* For mirror modifiers */
2157         RNA_def_float_vector(ot->srna, "clip_tolerance", 3, NULL, 0.0f, FLT_MAX, "clip_tolerance", "", 0.0f, 1000.0f);
2158
2159         /* The initial 2D location of the mouse */
2160         RNA_def_float_vector(ot->srna, "initial_mouse", 2, NULL, INT_MIN, INT_MAX, "initial_mouse", "", INT_MIN, INT_MAX);
2161
2162         RNA_def_boolean(ot->srna, "ignore_background_click", 0,
2163                         "Ignore Background Click",
2164                         "Clicks on the background don't start the stroke");
2165 }
2166
2167 /**** Reset the copy of the mesh that is being sculpted on (currently just for the layer brush) ****/
2168
2169 static int sculpt_set_persistent_base(bContext *C, wmOperator *op)
2170 {
2171         SculptSession *ss = CTX_data_active_object(C)->sculpt;
2172
2173         if(ss) {
2174                 if(ss->layer_disps)
2175                         MEM_freeN(ss->layer_disps);
2176                 ss->layer_disps = NULL;
2177
2178                 if(ss->layer_co)
2179                         MEM_freeN(ss->layer_co);
2180                 ss->layer_co = NULL;
2181         }
2182
2183         return OPERATOR_FINISHED;
2184 }
2185
2186 static void SCULPT_OT_set_persistent_base(wmOperatorType *ot)
2187 {
2188         /* identifiers */
2189         ot->name= "Set Persistent Base";
2190         ot->idname= "SCULPT_OT_set_persistent_base";
2191         
2192         /* api callbacks */
2193         ot->exec= sculpt_set_persistent_base;
2194         ot->poll= sculpt_mode_poll;
2195         
2196         ot->flag= OPTYPE_REGISTER;
2197 }
2198
2199 /**** Toggle operator for turning sculpt mode on or off ****/
2200
2201 static void sculpt_init_session(Scene *scene, Object *ob)
2202 {
2203         ob->sculpt = MEM_callocN(sizeof(SculptSession), "sculpt session");
2204         
2205         sculpt_update_mesh_elements(scene, ob, 0);
2206
2207         if(ob->sculpt->refkb)
2208                 sculpt_key_to_mesh(ob->sculpt->refkb, ob);
2209 }
2210
2211 static int sculpt_toggle_mode(bContext *C, wmOperator *op)
2212 {
2213         Scene *scene = CTX_data_scene(C);
2214         ToolSettings *ts = CTX_data_tool_settings(C);
2215         Object *ob = CTX_data_active_object(C);
2216         MultiresModifierData *mmd = sculpt_multires_active(ob);
2217
2218         if(ob->mode & OB_MODE_SCULPT) {
2219                 if(sculpt_multires_active(ob))
2220                         multires_force_update(ob);
2221
2222                 if(mmd && mmd->sculptlvl != mmd->lvl)
2223                         DAG_id_flush_update(&ob->id, OB_RECALC_DATA);
2224
2225                 /* Leave sculptmode */
2226                 ob->mode &= ~OB_MODE_SCULPT;
2227
2228                 free_sculptsession(&ob->sculpt);
2229         }
2230         else {
2231                 /* Enter sculptmode */
2232                 ob->mode |= OB_MODE_SCULPT;
2233
2234                 if(mmd && mmd->sculptlvl != mmd->lvl)
2235                         DAG_id_flush_update(&ob->id, OB_RECALC_DATA);
2236                 
2237                 /* Create persistent sculpt mode data */
2238                 if(!ts->sculpt)
2239                         ts->sculpt = MEM_callocN(sizeof(Sculpt), "sculpt mode data");
2240
2241                 /* Create sculpt mode session data */
2242                 if(ob->sculpt)
2243                         free_sculptsession(&ob->sculpt);
2244
2245                 sculpt_init_session(scene, ob);
2246
2247                 paint_init(&ts->sculpt->paint, PAINT_CURSOR_SCULPT);
2248                 
2249                 paint_cursor_start(C, sculpt_poll);
2250         }
2251
2252         WM_event_add_notifier(C, NC_SCENE|ND_MODE, CTX_data_scene(C));
2253
2254         return OPERATOR_FINISHED;
2255 }
2256
2257 static void SCULPT_OT_sculptmode_toggle(wmOperatorType *ot)
2258 {
2259         /* identifiers */
2260         ot->name= "Sculpt Mode";
2261         ot->idname= "SCULPT_OT_sculptmode_toggle";
2262         
2263         /* api callbacks */
2264         ot->exec= sculpt_toggle_mode;
2265         ot->poll= ED_operator_object_active;
2266         
2267         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
2268 }
2269
2270 void ED_operatortypes_sculpt()
2271 {
2272         WM_operatortype_append(SCULPT_OT_radial_control);
2273         WM_operatortype_append(SCULPT_OT_brush_stroke);
2274         WM_operatortype_append(SCULPT_OT_sculptmode_toggle);
2275         WM_operatortype_append(SCULPT_OT_set_persistent_base);
2276 }
2277