Bugfix: sculpting with a multires modifier at level 1 would crash.
[blender-staging.git] / source / blender / editors / sculpt_paint / sculpt.c
1 /*
2  * $Id$
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software  Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2006 by Nicholas Bishop
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  *
29  * Implements the Sculpt Mode tools
30  *
31  */
32
33 #include "MEM_guardedalloc.h"
34
35 #include "BLI_arithb.h"
36 #include "BLI_blenlib.h"
37 #include "BLI_dynstr.h"
38
39 #include "DNA_armature_types.h"
40 #include "DNA_brush_types.h"
41 #include "DNA_image_types.h"
42 #include "DNA_key_types.h"
43 #include "DNA_mesh_types.h"
44 #include "DNA_meshdata_types.h"
45 #include "DNA_modifier_types.h"
46 #include "DNA_object_types.h"
47 #include "DNA_screen_types.h"
48 #include "DNA_scene_types.h"
49 #include "DNA_texture_types.h"
50 #include "DNA_view3d_types.h"
51 #include "DNA_userdef_types.h"
52 #include "DNA_color_types.h"
53
54 #include "BKE_brush.h"
55 #include "BKE_context.h"
56 #include "BKE_customdata.h"
57 #include "BKE_DerivedMesh.h"
58 #include "BKE_depsgraph.h"
59 #include "BKE_global.h"
60 #include "BKE_image.h"
61 #include "BKE_key.h"
62 #include "BKE_library.h"
63 #include "BKE_main.h"
64 #include "BKE_mesh.h"
65 #include "BKE_modifier.h"
66 #include "BKE_multires.h"
67 #include "BKE_paint.h"
68 #include "BKE_texture.h"
69 #include "BKE_utildefines.h"
70 #include "BKE_colortools.h"
71
72 #include "BIF_gl.h"
73 #include "BIF_glutil.h"
74
75 #include "WM_api.h"
76 #include "WM_types.h"
77 #include "ED_screen.h"
78 #include "ED_sculpt.h"
79 #include "ED_space_api.h"
80 #include "ED_util.h"
81 #include "ED_view3d.h"
82 #include "paint_intern.h"
83 #include "sculpt_intern.h"
84
85 #include "RNA_access.h"
86 #include "RNA_define.h"
87
88 #include "IMB_imbuf_types.h"
89
90 #include "RE_render_ext.h"
91 #include "RE_shader_ext.h" /*for multitex_ext*/
92
93 #include "GPU_draw.h"
94 #include "gpu_buffers.h"
95
96 #include <math.h>
97 #include <stdlib.h>
98 #include <string.h>
99
100 /* Number of vertices to average in order to determine the flatten distance */
101 #define FLATTEN_SAMPLE_SIZE 10
102
103 /* ===== STRUCTS =====
104  *
105  */
106
107 /* ActiveData stores an Index into the mvert array of Mesh, plus Fade, which
108    stores how far the vertex is from the brush center, scaled to the range [0,1]. */
109 typedef struct ActiveData {
110         struct ActiveData *next, *prev;
111         unsigned int Index;
112         float Fade;
113         float dist;
114 } ActiveData;
115
116 typedef enum StrokeFlags {
117         CLIP_X = 1,
118         CLIP_Y = 2,
119         CLIP_Z = 4
120 } StrokeFlags;
121
122 /* Cache stroke properties. Used because
123    RNA property lookup isn't particularly fast.
124
125    For descriptions of these settings, check the operator properties.
126 */
127 typedef struct StrokeCache {
128         /* Invariants */
129         float initial_radius;
130         float scale[3];
131         int flag;
132         float clip_tolerance[3];
133         float initial_mouse[2];
134         float depth;
135
136         /* Variants */
137         float radius;
138         float true_location[3];
139         float location[3];
140         float flip;
141         float pressure;
142         float mouse[2];
143
144         /* The rest is temporary storage that isn't saved as a property */
145
146         int first_time; /* Beginning of stroke may do some things special */
147
148         bglMats *mats;
149
150         short (*orig_norms)[3]; /* Copy of the mesh vertices' normals */
151         float (*face_norms)[3]; /* Copy of the mesh faces' normals */
152         float rotation; /* Texture rotation (radians) for anchored and rake modes */
153         int pixel_radius, previous_pixel_radius;
154         ListBase grab_active_verts[8]; /* The same list of verts is used throught grab stroke */
155         float grab_delta[3], grab_delta_symmetry[3];
156         float old_grab_location[3];
157         int symmetry; /* Symmetry index between 0 and 7 */
158         float view_normal[3], view_normal_symmetry[3];
159         int last_rake[2]; /* Last location of updating rake rotation */
160 } StrokeCache;
161
162 typedef struct RectNode {
163         struct RectNode *next, *prev;
164         rcti r;
165 } RectNode;
166
167 /* Used to store to 2D screen coordinates of each vertex in the mesh. */
168 typedef struct ProjVert {
169         short co[2];
170         
171         /* Used to mark whether a vertex is inside a rough bounding box
172            containing the brush. */
173         char inside;
174 } ProjVert;
175
176 /* ===== OPENGL =====
177  *
178  * Simple functions to get data from the GL
179  */
180
181 /* Convert a point in model coordinates to 2D screen coordinates. */
182 static void projectf(bglMats *mats, const float v[3], float p[2])
183 {
184         double ux, uy, uz;
185
186         gluProject(v[0],v[1],v[2], mats->modelview, mats->projection,
187                    (GLint *)mats->viewport, &ux, &uy, &uz);
188         p[0]= ux;
189         p[1]= uy;
190 }
191
192 static void project(bglMats *mats, const float v[3], short p[2])
193 {
194         float f[2];
195         projectf(mats, v, f);
196
197         p[0]= f[0];
198         p[1]= f[1];
199 }
200
201 /* ===== Sculpting =====
202  *
203  */
204
205 /* Return modified brush strength. Includes the direction of the brush, positive
206    values pull vertices, negative values push. Uses tablet pressure and a
207    special multiplier found experimentally to scale the strength factor. */
208 static float brush_strength(Sculpt *sd, StrokeCache *cache)
209 {
210         Brush *brush = paint_brush(&sd->paint);
211         /* Primary strength input; square it to make lower values more sensitive */
212         float alpha = brush->alpha * brush->alpha;
213
214         float dir= brush->flag & BRUSH_DIR_IN ? -1 : 1;
215         float pressure= 1;
216         float flip= cache->flip ? -1:1;
217
218         if(brush->flag & BRUSH_ALPHA_PRESSURE)
219                 pressure *= cache->pressure;
220         
221         switch(brush->sculpt_tool){
222         case SCULPT_TOOL_DRAW:
223         case SCULPT_TOOL_INFLATE:
224         case SCULPT_TOOL_CLAY:
225         case SCULPT_TOOL_FLATTEN:
226         case SCULPT_TOOL_LAYER:
227                 return alpha * dir * pressure * flip; /*XXX: not sure why? was multiplied by G.vd->grid */;
228         case SCULPT_TOOL_SMOOTH:
229                 return alpha * 4 * pressure;
230         case SCULPT_TOOL_PINCH:
231                 return alpha / 2 * dir * pressure * flip;
232         case SCULPT_TOOL_GRAB:
233                 return 1;
234         default:
235                 return 0;
236         }
237 }
238
239 /* Handles clipping against a mirror modifier and SCULPT_LOCK axis flags */
240 static void sculpt_clip(Sculpt *sd, SculptSession *ss, float *co, const float val[3])
241 {
242         int i;
243
244         for(i=0; i<3; ++i) {
245                 if(sd->flags & (SCULPT_LOCK_X << i))
246                         continue;
247
248                 if((ss->cache->flag & (CLIP_X << i)) && (fabs(co[i]) <= ss->cache->clip_tolerance[i]))
249                         co[i]= 0.0f;
250                 else
251                         co[i]= val[i];
252         }               
253 }
254
255 static void add_norm_if(float view_vec[3], float out[3], float out_flip[3], const short no[3])
256 {
257         float fno[3] = {no[0], no[1], no[2]};
258
259         Normalize(fno);
260
261         if((Inpf(view_vec, fno)) > 0) {
262                 VecAddf(out, out, fno);
263         } else {
264                 VecAddf(out_flip, out_flip, fno); /* out_flip is used when out is {0,0,0} */
265         }
266 }
267
268 /* Currently only for the draw brush; finds average normal for all active
269    vertices */
270 static void calc_area_normal(Sculpt *sd, SculptSession *ss, float out[3], const ListBase* active_verts)
271 {
272         Brush *brush = paint_brush(&sd->paint);
273         StrokeCache *cache = ss->cache;
274         ActiveData *node = active_verts->first;
275         const int view = 0; /* XXX: should probably be a flag, not number: brush_type==SCULPT_TOOL_DRAW ? sculptmode_brush()->view : 0; */
276         float out_flip[3];
277         float *out_dir = cache->view_normal_symmetry;
278         
279         out[0]=out[1]=out[2] = out_flip[0]=out_flip[1]=out_flip[2] = 0;
280
281         if(brush->flag & BRUSH_ANCHORED) {
282                 for(; node; node = node->next)
283                         add_norm_if(out_dir, out, out_flip, cache->orig_norms[node->Index]);
284         }
285         else {
286                 for(; node; node = node->next)
287                         add_norm_if(out_dir, out, out_flip, ss->mvert[node->Index].no);
288         }
289
290         if (out[0]==0.0 && out[1]==0.0 && out[2]==0.0) {
291                 VECCOPY(out, out_flip);
292         }
293         
294         Normalize(out);
295
296         if(out_dir) {
297                 out[0] = out_dir[0] * view + out[0] * (10-view);
298                 out[1] = out_dir[1] * view + out[1] * (10-view);
299                 out[2] = out_dir[2] * view + out[2] * (10-view);
300         }
301         
302         Normalize(out);
303 }
304
305 static void do_draw_brush(Sculpt *sd, SculptSession *ss, const ListBase* active_verts)
306 {
307         float area_normal[3];
308         ActiveData *node= active_verts->first;
309         float* buffer;
310
311         calc_area_normal(sd, ss, area_normal, active_verts);
312         
313         buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
314
315         while(node){
316                 float *co= ss->mvert[node->Index].co;
317
318                 const float val[3]= {co[0]+area_normal[0]*ss->cache->radius*node->Fade*ss->cache->scale[0],
319                                      co[1]+area_normal[1]*ss->cache->radius*node->Fade*ss->cache->scale[1],
320                                      co[2]+area_normal[2]*ss->cache->radius*node->Fade*ss->cache->scale[2]};
321
322                 if( buffer != 0 ) {
323                         IndexLink *cur = &ss->drawobject->indices[node->Index];
324                         while( cur != 0 && cur->element != -1 ) {
325                                 sculpt_clip(sd, ss, &buffer[cur->element*3], val);
326                                 cur = cur->next;
327                         }
328                 }
329
330                 sculpt_clip(sd, ss, co, val);
331
332                 node= node->next;
333         }
334         if( buffer != 0 )
335                 GPU_buffer_unlock( ss->drawobject->vertices );
336 }
337
338 /* For the smooth brush, uses the neighboring vertices around vert to calculate
339    a smoothed location for vert. Skips corner vertices (used by only one
340    polygon.) */
341 static void neighbor_average(SculptSession *ss, float avg[3], const int vert)
342 {
343         int i, skip= -1, total=0;
344         IndexNode *node= ss->fmap[vert].first;
345         char ncount= BLI_countlist(&ss->fmap[vert]);
346         MFace *f;
347
348         avg[0] = avg[1] = avg[2] = 0;
349                 
350         /* Don't modify corner vertices */
351         if(ncount==1) {
352                 VecCopyf(avg, ss->mvert[vert].co);
353                 return;
354         }
355
356         while(node){
357                 f= &ss->mface[node->index];
358                 
359                 if(f->v4) {
360                         skip= (f->v1==vert?2:
361                                f->v2==vert?3:
362                                f->v3==vert?0:
363                                f->v4==vert?1:-1);
364                 }
365
366                 for(i=0; i<(f->v4?4:3); ++i) {
367                         if(i != skip && (ncount!=2 || BLI_countlist(&ss->fmap[(&f->v1)[i]]) <= 2)) {
368                                 VecAddf(avg, avg, ss->mvert[(&f->v1)[i]].co);
369                                 ++total;
370                         }
371                 }
372
373                 node= node->next;
374         }
375
376         if(total>0)
377                 VecMulf(avg, 1.0f / total);
378         else
379                 VecCopyf(avg, ss->mvert[vert].co);
380 }
381
382 static void do_smooth_brush(Sculpt *s, SculptSession *ss, const ListBase* active_verts)
383 {
384         ActiveData *node= active_verts->first;
385         float *buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
386         int i;
387         
388         for(i = 0; i < 2; ++i) {
389                 while(node){
390                         float *co= ss->mvert[node->Index].co;
391                         float avg[3], val[3];
392                         
393                         neighbor_average(ss, avg, node->Index);
394                         val[0] = co[0]+(avg[0]-co[0])*node->Fade;
395                         val[1] = co[1]+(avg[1]-co[1])*node->Fade;
396                         val[2] = co[2]+(avg[2]-co[2])*node->Fade;
397                         
398                         sculpt_clip(s, ss, co, val);                    
399                         if( buffer != 0 ) {                             
400                                 IndexLink *cur = &ss->drawobject->indices[node->Index]; 
401                                 while( cur != 0 && cur->element != -1 ) {
402                                         sculpt_clip(s, ss, &buffer[cur->element*3], val);
403                                         cur = cur->next;
404                                 }
405                         }
406                         node= node->next;
407                 }
408         }
409         if( buffer != 0 )
410                 GPU_buffer_unlock( ss->drawobject->vertices );
411 }
412
413 static void do_pinch_brush(Sculpt *s, SculptSession *ss, const ListBase* active_verts)
414 {
415         ActiveData *node= active_verts->first;
416         float *buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
417
418         while(node) {
419                 float *co= ss->mvert[node->Index].co;
420                 const float val[3]= {co[0]+(ss->cache->location[0]-co[0])*node->Fade,
421                                      co[1]+(ss->cache->location[1]-co[1])*node->Fade,
422                                      co[2]+(ss->cache->location[2]-co[2])*node->Fade};
423
424                 if( buffer != 0 ) {
425                         IndexLink *cur = &ss->drawobject->indices[node->Index];
426                         while( cur != 0 && cur->element != -1 ) {
427                                 sculpt_clip(s, ss, &buffer[cur->element*3], val);
428                                 cur = cur->next;
429                         }
430                 }
431
432                 sculpt_clip(s, ss, co, val);
433                 node= node->next;
434         }
435         if( buffer != 0 )
436                 GPU_buffer_unlock( ss->drawobject->vertices );
437 }
438
439 static void do_grab_brush(Sculpt *sd, SculptSession *ss)
440 {
441         ActiveData *node= ss->cache->grab_active_verts[ss->cache->symmetry].first;
442         float add[3];
443         float grab_delta[3];
444         float *buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
445         
446         VecCopyf(grab_delta, ss->cache->grab_delta_symmetry);
447         
448         while(node) {
449                 float *co= ss->mvert[node->Index].co;
450                 
451                 VecCopyf(add, grab_delta);
452                 VecMulf(add, node->Fade);
453                 VecAddf(add, add, co);
454
455                 if( buffer != 0 ) {
456                         IndexLink *cur = &ss->drawobject->indices[node->Index];
457                         while( cur != 0 && cur->element != -1 ) {
458                                 sculpt_clip(sd, ss, &buffer[cur->element*3], add);
459                                 cur = cur->next;
460                         }
461                 }
462
463                 sculpt_clip(sd, ss, co, add);
464
465                 node= node->next;
466         }
467         if( buffer != 0 )
468                 GPU_buffer_unlock( ss->drawobject->vertices );
469         
470 }
471
472 static void do_layer_brush(Sculpt *sd, SculptSession *ss, const ListBase *active_verts)
473 {
474         float area_normal[3];
475         ActiveData *node= active_verts->first;
476         float *buffer;
477         float lim= ss->cache->radius / 4;
478
479         if(ss->cache->flip)
480                 lim = -lim;
481
482         calc_area_normal(sd, ss, area_normal, active_verts);
483
484         buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
485         while(node){
486                 float *disp= &ss->layer_disps[node->Index];
487                 float *co= ss->mvert[node->Index].co;
488                 float val[3];
489                 
490                 *disp+= node->Fade;
491                 
492                 /* Don't let the displacement go past the limit */
493                 if((lim < 0 && *disp < lim) || (lim > 0 && *disp > lim))
494                         *disp = lim;
495                 
496                 val[0] = ss->mesh_co_orig[node->Index][0]+area_normal[0] * *disp*ss->cache->scale[0];
497                 val[1] = ss->mesh_co_orig[node->Index][1]+area_normal[1] * *disp*ss->cache->scale[1];
498                 val[2] = ss->mesh_co_orig[node->Index][2]+area_normal[2] * *disp*ss->cache->scale[2];
499
500                 if( buffer != 0 ) {
501                         IndexLink *cur = &ss->drawobject->indices[node->Index];
502                         while( cur != 0 && cur->element != -1 ) {
503                                 sculpt_clip(sd, ss, &buffer[cur->element*3], val);
504                                 cur = cur->next;
505                         }
506                 }
507
508                 sculpt_clip(sd, ss, co, val);
509
510                 node= node->next;
511         }
512         if( buffer != 0 )
513                 GPU_buffer_unlock( ss->drawobject->vertices );
514 }
515
516 static void do_inflate_brush(Sculpt *s, SculptSession *ss, const ListBase *active_verts)
517 {
518         ActiveData *node= active_verts->first;
519         float add[3];
520         float *buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
521
522         while(node) {
523                 float *co= ss->mvert[node->Index].co;
524                 short *no= ss->mvert[node->Index].no;
525
526                 add[0]= no[0]/ 32767.0f;
527                 add[1]= no[1]/ 32767.0f;
528                 add[2]= no[2]/ 32767.0f;
529                 VecMulf(add, node->Fade * ss->cache->radius);
530                 add[0]*= ss->cache->scale[0];
531                 add[1]*= ss->cache->scale[1];
532                 add[2]*= ss->cache->scale[2];
533                 VecAddf(add, add, co);
534                 
535                 if( buffer != 0 ) {
536                         IndexLink *cur = &ss->drawobject->indices[node->Index];
537                         while( cur != 0 && cur->element != -1 ) {
538                                 sculpt_clip(s, ss, &buffer[cur->element*3], add);
539                                 cur = cur->next;
540                         }
541                 }
542
543                 sculpt_clip(s, ss, co, add);
544
545                 node= node->next;
546         }
547         if( buffer != 0 )
548                 GPU_buffer_unlock( ss->drawobject->vertices );
549 }
550
551 static void calc_flatten_center(SculptSession *ss, ActiveData *node, float co[3])
552 {
553         ActiveData *outer[FLATTEN_SAMPLE_SIZE];
554         int i;
555         
556         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
557                 outer[i] = node;
558                 
559         for(; node; node = node->next) {
560                 for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i) {
561                         if(node->dist > outer[i]->dist) {
562                                 outer[i] = node;
563                                 break;
564                         }
565                 }
566         }
567         
568         co[0] = co[1] = co[2] = 0.0f;
569         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
570                 VecAddf(co, co, ss->mvert[outer[i]->Index].co);
571         VecMulf(co, 1.0f / FLATTEN_SAMPLE_SIZE);
572 }
573
574 /* Projects a point onto a plane along the plane's normal */
575 static void point_plane_project(float intr[3], float co[3], float plane_normal[3], float plane_center[3])
576 {
577         float p1[3], sub1[3], sub2[3];
578
579         /* Find the intersection between squash-plane and vertex (along the area normal) */
580         VecSubf(p1, co, plane_normal);
581         VecSubf(sub1, plane_center, p1);
582         VecSubf(sub2, co, p1);
583         VecSubf(intr, co, p1);
584         VecMulf(intr, Inpf(plane_normal, sub1) / Inpf(plane_normal, sub2));
585         VecAddf(intr, intr, p1);
586 }
587
588 static int plane_point_side(float co[3], float plane_normal[3], float plane_center[3], int flip)
589 {
590         float delta[3];
591         float d;
592
593         VecSubf(delta, co, plane_center);
594         d = Inpf(plane_normal, delta);
595
596         if(flip)
597                 d = -d;
598
599         return d <= 0.0f;
600 }
601
602 static void do_flatten_clay_brush(Sculpt *sd, SculptSession *ss, const ListBase *active_verts, int clay)
603 {
604         ActiveData *node= active_verts->first;
605         /* area_normal and cntr define the plane towards which vertices are squashed */
606         float area_normal[3];
607         float cntr[3], cntr2[3], bstr = 0;
608         int flip = 0;
609         float *buffer;
610         calc_area_normal(sd, ss, area_normal, active_verts);
611         calc_flatten_center(ss, node, cntr);
612
613         if(clay) {
614                 bstr= brush_strength(sd, ss->cache);
615                 /* Limit clay application to here */
616                 cntr2[0]=cntr[0]+area_normal[0]*bstr*ss->cache->scale[0];
617                 cntr2[1]=cntr[1]+area_normal[1]*bstr*ss->cache->scale[1];
618                 cntr2[2]=cntr[2]+area_normal[2]*bstr*ss->cache->scale[2];
619                 flip = bstr < 0;
620         }
621
622         buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
623
624         while(node){
625                 float *co= ss->mvert[node->Index].co;
626                 float intr[3], val[3];
627                 
628                 if(!clay || plane_point_side(co, area_normal, cntr2, flip)) {
629                         /* Find the intersection between squash-plane and vertex (along the area normal) */             
630                         point_plane_project(intr, co, area_normal, cntr);
631
632                         VecSubf(val, intr, co);
633
634                         if(clay) {
635                                 if(bstr > FLT_EPSILON)
636                                         VecMulf(val, node->Fade / bstr);
637                                 else
638                                         VecMulf(val, node->Fade);
639                                 /* Clay displacement */
640                                 val[0]+=area_normal[0] * ss->cache->scale[0]*node->Fade;
641                                 val[1]+=area_normal[1] * ss->cache->scale[1]*node->Fade;
642                                 val[2]+=area_normal[2] * ss->cache->scale[2]*node->Fade;
643                         }
644                         else
645                                 VecMulf(val, fabs(node->Fade));
646
647                         VecAddf(val, val, co);
648
649                         if( buffer != 0 ) {
650                                 IndexLink *cur = &ss->drawobject->indices[node->Index];
651                                 while( cur != 0 && cur->element != -1 ) {
652                                         sculpt_clip(sd, ss, &buffer[cur->element*3], val);
653                                         cur = cur->next;
654                                 }
655                         }                       
656                         sculpt_clip(sd, ss, co, val);
657
658                 }
659                 
660                 node= node->next;
661         }
662         if( buffer != 0 )
663                 GPU_buffer_unlock( ss->drawobject->vertices );
664 }
665
666 /* Uses symm to selectively flip any axis of a coordinate. */
667 static void flip_coord(float out[3], float in[3], const char symm)
668 {
669         if(symm & SCULPT_SYMM_X)
670                 out[0]= -in[0];
671         else
672                 out[0]= in[0];
673         if(symm & SCULPT_SYMM_Y)
674                 out[1]= -in[1];
675         else
676                 out[1]= in[1];
677         if(symm & SCULPT_SYMM_Z)
678                 out[2]= -in[2];
679         else
680                 out[2]= in[2];
681 }
682
683 /* Get a pixel from the texcache at (px, py) */
684 static unsigned char get_texcache_pixel(const SculptSession *ss, int px, int py)
685 {
686         unsigned *p;
687         p = ss->texcache + py * ss->texcache_side + px;
688         return ((unsigned char*)(p))[0];
689 }
690
691 static float get_texcache_pixel_bilinear(const SculptSession *ss, float u, float v)
692 {
693         int x, y, x2, y2;
694         const int tc_max = ss->texcache_side - 1;
695         float urat, vrat, uopp;
696
697         if(u < 0) u = 0;
698         else if(u >= ss->texcache_side) u = tc_max;
699         if(v < 0) v = 0;
700         else if(v >= ss->texcache_side) v = tc_max;
701
702         x = floor(u);
703         y = floor(v);
704         x2 = x + 1;
705         y2 = y + 1;
706
707         if(x2 > ss->texcache_side) x2 = tc_max;
708         if(y2 > ss->texcache_side) y2 = tc_max;
709         
710         urat = u - x;
711         vrat = v - y;
712         uopp = 1 - urat;
713                 
714         return ((get_texcache_pixel(ss, x, y) * uopp +
715                  get_texcache_pixel(ss, x2, y) * urat) * (1 - vrat) + 
716                 (get_texcache_pixel(ss, x, y2) * uopp +
717                  get_texcache_pixel(ss, x2, y2) * urat) * vrat) / 255.0;
718 }
719
720 /* Return a multiplier for brush strength on a particular vertex. */
721 static float tex_strength(Sculpt *sd, SculptSession *ss, float *point, const float len)
722 {
723         Brush *br = paint_brush(&sd->paint);
724         MTex *tex = NULL;
725         float avg= 1;
726
727         if(br->texact >= 0)
728                 tex = br->mtex[br->texact];
729
730         if(!tex) {
731                 avg= 1;
732         }
733         else if(tex->brush_map_mode == MTEX_MAP_MODE_3D) {
734                 float jnk;
735
736                 /* Get strength by feeding the vertex 
737                    location directly into a texture */
738                 externtex(tex, point, &avg,
739                           &jnk, &jnk, &jnk, &jnk);
740         }
741         else if(ss->texcache) {
742                 const float bsize= ss->cache->pixel_radius * 2;
743                 const float rot= tex->rot + ss->cache->rotation;
744                 int px, py;
745                 float flip[3], point_2d[2];
746
747                 /* If the active area is being applied for symmetry, flip it
748                    across the symmetry axis in order to project it. This insures
749                    that the brush texture will be oriented correctly. */
750                 VecCopyf(flip, point);
751                 flip_coord(flip, flip, ss->cache->symmetry);
752                 projectf(ss->cache->mats, flip, point_2d);
753
754                 /* For Tile and Drag modes, get the 2D screen coordinates of the
755                    and scale them up or down to the texture size. */
756                 if(tex->brush_map_mode == MTEX_MAP_MODE_TILED) {
757                         const int sx= (const int)tex->size[0];
758                         const int sy= (const int)tex->size[1];
759                         
760                         float fx= point_2d[0];
761                         float fy= point_2d[1];
762                         
763                         float angle= atan2(fy, fx) - rot;
764                         float flen= sqrtf(fx*fx + fy*fy);
765                         
766                         if(rot<0.001 && rot>-0.001) {
767                                 px= point_2d[0];
768                                 py= point_2d[1];
769                         } else {
770                                 px= flen * cos(angle) + 2000;
771                                 py= flen * sin(angle) + 2000;
772                         }
773                         if(sx != 1)
774                                 px %= sx-1;
775                         if(sy != 1)
776                                 py %= sy-1;
777                         avg= get_texcache_pixel_bilinear(ss, ss->texcache_side*px/sx, ss->texcache_side*py/sy);
778                 }
779                 else if(tex->brush_map_mode == MTEX_MAP_MODE_FIXED) {
780                         float fx= (point_2d[0] - ss->cache->mouse[0]) / bsize;
781                         float fy= (point_2d[1] - ss->cache->mouse[1]) / bsize;
782
783                         float angle= atan2(fy, fx) - rot;
784                         float flen= sqrtf(fx*fx + fy*fy);
785                         
786                         fx = flen * cos(angle) + 0.5;
787                         fy = flen * sin(angle) + 0.5;
788
789                         avg= get_texcache_pixel_bilinear(ss, fx * ss->texcache_side, fy * ss->texcache_side);
790                 }
791         }
792
793         avg*= brush_curve_strength(br, len, ss->cache->radius); /* Falloff curve */
794
795         return avg;
796 }
797
798 /* Mark area around the brush as damaged. projverts are marked if they are
799    inside the area and the damaged rectangle in 2D screen coordinates is 
800    added to damaged_rects. */
801 static void sculpt_add_damaged_rect(SculptSession *ss)
802 {
803         short p[2];
804         RectNode *rn= MEM_mallocN(sizeof(RectNode),"RectNode");
805         const float radius = MAX2(ss->cache->pixel_radius, ss->cache->previous_pixel_radius);
806         unsigned i;
807
808         /* Find center */
809         project(ss->cache->mats, ss->cache->location, p);
810         rn->r.xmin= p[0] - radius;
811         rn->r.ymin= p[1] - radius;
812         rn->r.xmax= p[0] + radius;
813         rn->r.ymax= p[1] + radius;
814
815         BLI_addtail(&ss->damaged_rects, rn);
816
817         /* Update insides */
818         for(i=0; i<ss->totvert; ++i) {
819                 if(!ss->projverts[i].inside) {
820                         if(ss->projverts[i].co[0] > rn->r.xmin && ss->projverts[i].co[1] > rn->r.ymin &&
821                            ss->projverts[i].co[0] < rn->r.xmax && ss->projverts[i].co[1] < rn->r.ymax) {
822                                 ss->projverts[i].inside= 1;
823                         }
824                 }
825                 // XXX: remember to fix this!
826                 // temporary pass
827                 ss->projverts[i].inside = 1;
828         }
829 }
830
831 static void do_brush_action(Sculpt *sd, SculptSession *ss, StrokeCache *cache)
832 {
833         Brush *brush = paint_brush(&sd->paint);
834         float av_dist;
835         ListBase active_verts={0,0};
836         ListBase *grab_active_verts = &ss->cache->grab_active_verts[ss->cache->symmetry];
837         ActiveData *adata= 0;
838         float *vert;
839         Mesh *me= NULL; /*XXX: get_mesh(OBACT); */
840         const float bstrength= brush_strength(sd, cache);
841         KeyBlock *keyblock= NULL; /*XXX: ob_get_keyblock(OBACT); */
842         Brush *b = brush;
843         int i;
844
845         sculpt_add_damaged_rect(ss);
846
847         /* Build a list of all vertices that are potentially within the brush's
848            area of influence. Only do this once for the grab brush. */
849         if((b->sculpt_tool != SCULPT_TOOL_GRAB) || cache->first_time) {
850                 for(i=0; i<ss->totvert; ++i) {
851                         /* Projverts.inside provides a rough bounding box */
852                         if(ss->multires || ss->projverts[i].inside) {
853                                 //vert= ss->vertexcosnos ? &ss->vertexcosnos[i*6] : a->verts[i].co;
854                                 vert= ss->mvert[i].co;
855                                 av_dist= VecLenf(ss->cache->location, vert);
856                                 if(av_dist < cache->radius) {
857                                         adata= (ActiveData*)MEM_mallocN(sizeof(ActiveData), "ActiveData");
858
859                                         adata->Index = i;
860                                         /* Fade is used to store the final strength at which the brush
861                                            should modify a particular vertex. */
862                                         adata->Fade= tex_strength(sd, ss, vert, av_dist) * bstrength;
863                                         adata->dist = av_dist;
864
865                                         if(b->sculpt_tool == SCULPT_TOOL_GRAB && cache->first_time)
866                                                 BLI_addtail(grab_active_verts, adata);
867                                         else
868                                                 BLI_addtail(&active_verts, adata);
869                                 }
870                         }
871                 }
872         }
873
874         /* Only act if some verts are inside the brush area */
875         if(active_verts.first || (b->sculpt_tool == SCULPT_TOOL_GRAB && grab_active_verts->first)) {
876                 /* Apply one type of brush action */
877                 switch(b->sculpt_tool){
878                 case SCULPT_TOOL_DRAW:
879                         do_draw_brush(sd, ss, &active_verts);
880                         break;
881                 case SCULPT_TOOL_SMOOTH:
882                         do_smooth_brush(sd, ss, &active_verts);
883                         break;
884                 case SCULPT_TOOL_PINCH:
885                         do_pinch_brush(sd, ss, &active_verts);
886                         break;
887                 case SCULPT_TOOL_INFLATE:
888                         do_inflate_brush(sd, ss, &active_verts);
889                         break;
890                 case SCULPT_TOOL_GRAB:
891                         do_grab_brush(sd, ss);
892                         break;
893                 case SCULPT_TOOL_LAYER:
894                         do_layer_brush(sd, ss, &active_verts);
895                         break;
896                 case SCULPT_TOOL_FLATTEN:
897                         do_flatten_clay_brush(sd, ss, &active_verts, 0);
898                         break;
899                 case SCULPT_TOOL_CLAY:
900                         do_flatten_clay_brush(sd, ss, &active_verts, 1);
901                 }
902         
903                 /* Copy the modified vertices from mesh to the active key */
904                 if(keyblock && !ss->multires) {
905                         float *co= keyblock->data;
906                         if(co) {
907                                 if(b->sculpt_tool == SCULPT_TOOL_GRAB)
908                                         adata = grab_active_verts->first;
909                                 else
910                                         adata = active_verts.first;
911
912                                 for(; adata; adata= adata->next)
913                                         if(adata->Index < keyblock->totelem)
914                                                 VecCopyf(&co[adata->Index*3], me->mvert[adata->Index].co);
915                         }
916                 }
917
918                 if(ss->vertexcosnos && !ss->multires)
919                         BLI_freelistN(&active_verts);
920                 else {
921                         if(b->sculpt_tool != SCULPT_TOOL_GRAB)
922                                 addlisttolist(&ss->damaged_verts, &active_verts);
923                 }
924         }
925 }
926
927 /* Flip all the editdata across the axis/axes specified by symm. Used to
928    calculate multiple modifications to the mesh when symmetry is enabled. */
929 static void calc_brushdata_symm(StrokeCache *cache, const char symm)
930 {
931         flip_coord(cache->location, cache->true_location, symm);
932         flip_coord(cache->view_normal_symmetry, cache->view_normal, symm);
933         flip_coord(cache->grab_delta_symmetry, cache->grab_delta, symm);
934         cache->symmetry= symm;
935 }
936
937 static void do_symmetrical_brush_actions(Sculpt *sd, SculptSession *ss)
938 {
939         StrokeCache *cache = ss->cache;
940         const char symm = sd->flags & 7;
941         int i;
942
943         VecCopyf(cache->location, cache->true_location);
944         VecCopyf(cache->grab_delta_symmetry, cache->grab_delta);
945         cache->symmetry = 0;
946         do_brush_action(sd, ss, cache);
947
948         for(i = 1; i <= symm; ++i) {
949                 if(symm & i && (symm != 5 || i != 3) && (symm != 6 || (i != 3 && i != 5))) {
950                         calc_brushdata_symm(cache, i);
951                         do_brush_action(sd, ss, cache);
952                 }
953         }
954
955         cache->first_time = 0;
956 }
957
958 static void add_face_normal(vec3f *norm, MVert *mvert, const MFace* face, float *fn)
959 {
960         vec3f c= {mvert[face->v1].co[0],mvert[face->v1].co[1],mvert[face->v1].co[2]};
961         vec3f b= {mvert[face->v2].co[0],mvert[face->v2].co[1],mvert[face->v2].co[2]};
962         vec3f a= {mvert[face->v3].co[0],mvert[face->v3].co[1],mvert[face->v3].co[2]};
963         vec3f s1, s2;
964         float final[3];
965
966         VecSubf(&s1.x,&a.x,&b.x);
967         VecSubf(&s2.x,&c.x,&b.x);
968
969         final[0] = s1.y * s2.z - s1.z * s2.y;
970         final[1] = s1.z * s2.x - s1.x * s2.z;
971         final[2] = s1.x * s2.y - s1.y * s2.x;
972
973         if(fn)
974                 VecCopyf(fn, final);
975
976         norm->x+= final[0];
977         norm->y+= final[1];
978         norm->z+= final[2];
979 }
980
981 static void update_damaged_vert(SculptSession *ss, ListBase *lb)
982 {
983         ActiveData *vert;
984     
985         float *buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->normals ):0;
986         for(vert= lb->first; vert; vert= vert->next) {
987                 vec3f norm= {0,0,0};            
988                 IndexNode *face= ss->fmap[vert->Index].first;
989
990                 while(face){
991                         float *fn = NULL;
992                         if(ss->face_normals)
993                                 fn = &ss->face_normals[face->index*3];
994                         add_face_normal(&norm, ss->mvert, &ss->mface[face->index], fn);
995                         face= face->next;
996                 }
997                 Normalize(&norm.x);
998                 
999                 ss->mvert[vert->Index].no[0]=norm.x*32767;
1000                 ss->mvert[vert->Index].no[1]=norm.y*32767;
1001                 ss->mvert[vert->Index].no[2]=norm.z*32767;
1002
1003                 if( buffer != 0 ) {
1004                         IndexLink *cur = &ss->drawobject->indices[vert->Index];
1005                         while( cur != 0 && cur->element != -1 ) {
1006                                 int i = ss->drawobject->faceRemap[cur->element/3];
1007                                 if( ss->mface[i].flag & ME_SMOOTH ) {
1008                                         VECCOPY(&buffer[cur->element*3],ss->mvert[vert->Index].no);
1009                                 }
1010                                 else {
1011                                         float norm[3];
1012                                         if( ss->mface[i].v4 )
1013                                                 CalcNormFloat4(ss->mvert[ss->mface[i].v1].co, ss->mvert[ss->mface[i].v2].co, ss->mvert[ss->mface[i].v3].co, ss->mvert[ss->mface[i].v4].co, norm);
1014                                         else
1015                                                 CalcNormFloat(ss->mvert[ss->mface[i].v1].co, ss->mvert[ss->mface[i].v2].co, ss->mvert[ss->mface[i].v3].co, norm);
1016                                         VECCOPY(&buffer[(cur->element-cur->element%3)*3],norm);
1017                                         VECCOPY(&buffer[(cur->element-cur->element%3+1)*3],norm);
1018                                         VECCOPY(&buffer[(cur->element-cur->element%3+2)*3],norm);
1019
1020                                         /* maybe this was a quad - need to update the other triangle of the quad */
1021                                         if( ss->drawobject->faceRemap[cur->element/3-1] == i ) {
1022                                                 VECCOPY(&buffer[(cur->element-cur->element%3-3)*3],norm);
1023                                                 VECCOPY(&buffer[(cur->element-cur->element%3-2)*3],norm);
1024                                                 VECCOPY(&buffer[(cur->element-cur->element%3-1)*3],norm);
1025                                         }
1026                                         if( ss->drawobject->faceRemap[cur->element/3+1] == i ) {
1027                                                 VECCOPY(&buffer[(cur->element-cur->element%3+3)*3],norm);
1028                                                 VECCOPY(&buffer[(cur->element-cur->element%3+4)*3],norm);
1029                                                 VECCOPY(&buffer[(cur->element-cur->element%3+5)*3],norm);
1030                                         }
1031                                 }
1032
1033                                 //VECCOPY(&buffer[cur->element*3],ss->mvert[vert->Index].no);
1034                                 cur = cur->next;
1035                         }
1036                 }
1037         }
1038         if( buffer != 0 )
1039                 GPU_buffer_unlock( ss->drawobject->normals );
1040 }
1041
1042 static void calc_damaged_verts(SculptSession *ss)
1043 {
1044         int i;
1045         
1046         for(i=0; i<8; ++i)
1047                 update_damaged_vert(ss, &ss->cache->grab_active_verts[i]);
1048         update_damaged_vert(ss, &ss->damaged_verts);
1049         BLI_freelistN(&ss->damaged_verts);
1050         ss->damaged_verts.first = ss->damaged_verts.last = NULL;
1051 }
1052
1053 #if 0
1054 static void projverts_clear_inside(SculptSession *ss)
1055 {
1056         int i;
1057         for(i = 0; i < ss->totvert; ++i)
1058                 ss->projverts[i].inside = 0;
1059 }
1060 #endif
1061
1062 static void sculpt_update_tex(Sculpt *sd, SculptSession *ss)
1063 {
1064         Brush *brush = paint_brush(&sd->paint);
1065
1066         if(ss->texcache) {
1067                 MEM_freeN(ss->texcache);
1068                 ss->texcache= NULL;
1069         }
1070
1071         /* Need to allocate a bigger buffer for bigger brush size */
1072         ss->texcache_side = brush->size * 2;
1073         if(!ss->texcache || ss->texcache_side > ss->texcache_actual) {
1074                 ss->texcache = brush_gen_texture_cache(brush, brush->size);
1075                 ss->texcache_actual = ss->texcache_side;
1076         }
1077 }
1078
1079 static void sculptmode_update_all_projverts(SculptSession *ss)
1080 {
1081         unsigned i;
1082
1083         if(!ss->projverts)
1084                 ss->projverts = MEM_mallocN(sizeof(ProjVert)*ss->totvert,"ProjVerts");
1085
1086         for(i=0; i<ss->totvert; ++i) {
1087                 project(ss->cache->mats, ss->vertexcosnos ? &ss->vertexcosnos[i * 6] : ss->mvert[i].co,
1088                         ss->projverts[i].co);
1089                 ss->projverts[i].inside= 0;
1090         }
1091 }
1092
1093 /* Checks whether full update mode (slower) needs to be used to work with modifiers */
1094 char sculpt_modifiers_active(Object *ob)
1095 {
1096         ModifierData *md;
1097         
1098         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1099                 if(md->mode & eModifierMode_Realtime && md->type != eModifierType_Multires)
1100                         return 1;
1101         }
1102         
1103         return 0;
1104 }
1105
1106 /* Sculpt mode handles multires differently from regular meshes, but only if
1107    it's the last modifier on the stack and it is not on the first level */
1108 static struct MultiresModifierData *sculpt_multires_active(Object *ob)
1109 {
1110         ModifierData *md, *nmd;
1111         
1112         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1113                 if(md->type == eModifierType_Multires) {
1114                         MultiresModifierData *mmd= (MultiresModifierData*)md;
1115
1116                         /* Check if any of the modifiers after multires are active
1117                          * if not it can use the multires struct */
1118                         for (nmd= md->next; nmd; nmd= nmd->next)
1119                                 if(nmd->mode & eModifierMode_Realtime)
1120                                         break;
1121
1122                         if(!nmd && mmd->lvl != 1)
1123                                 return mmd;
1124                 }
1125         }
1126
1127         return NULL;
1128 }
1129
1130 static void sculpt_update_mesh_elements(bContext *C)
1131 {
1132         Object *ob = CTX_data_active_object(C);
1133         SculptSession *ss = ob->sculpt;
1134         int oldtotvert = ss->totvert;
1135         DerivedMesh *dm = mesh_get_derived_final(CTX_data_scene(C), ob, CD_MASK_BAREMESH);
1136
1137         if((ss->multires = sculpt_multires_active(ob))) {
1138                 //DerivedMesh *dm = mesh_get_derived_final(CTX_data_scene(C), ob, CD_MASK_BAREMESH);
1139                 ss->totvert = dm->getNumVerts(dm);
1140                 ss->totface = dm->getNumFaces(dm);
1141                 ss->mvert = dm->getVertDataArray(dm, CD_MVERT);
1142                 ss->mface = dm->getFaceDataArray(dm, CD_MFACE);
1143                 ss->face_normals = dm->getFaceDataArray(dm, CD_NORMAL);
1144         }
1145         else {
1146                 Mesh *me = get_mesh(ob);
1147                 ss->totvert = me->totvert;
1148                 ss->totface = me->totface;
1149                 ss->mvert = me->mvert;
1150                 ss->mface = me->mface;
1151                 ss->face_normals = NULL;
1152         }
1153         if( GPU_buffer_legacy( dm ) ) {
1154                 ss->drawobject = 0;
1155         }
1156         else {
1157                 ss->drawobject = dm->drawObject;
1158         }
1159
1160         if(ss->totvert != oldtotvert) {
1161                 if(ss->projverts) MEM_freeN(ss->projverts);
1162                 ss->projverts = NULL;
1163
1164                 if(ss->fmap) MEM_freeN(ss->fmap);
1165                 if(ss->fmap_mem) MEM_freeN(ss->fmap_mem);
1166                 create_vert_face_map(&ss->fmap, &ss->fmap_mem, ss->mface, ss->totvert, ss->totface);
1167                 ss->fmap_size = ss->totvert;
1168         }
1169 }
1170
1171 static int sculpt_mode_poll(bContext *C)
1172 {
1173         Object *ob = CTX_data_active_object(C);
1174         return ob && ob->mode & OB_MODE_SCULPT;
1175 }
1176
1177 int sculpt_poll(bContext *C)
1178 {
1179         return sculpt_mode_poll(C) && paint_poll(C);
1180 }
1181
1182 static void sculpt_undo_push(bContext *C, Sculpt *sd)
1183 {
1184         Brush *brush = paint_brush(&sd->paint);
1185
1186         switch(brush->sculpt_tool) {
1187         case SCULPT_TOOL_DRAW:
1188                 ED_undo_push(C, "Draw Brush"); break;
1189         case SCULPT_TOOL_SMOOTH:
1190                 ED_undo_push(C, "Smooth Brush"); break;
1191         case SCULPT_TOOL_PINCH:
1192                 ED_undo_push(C, "Pinch Brush"); break;
1193         case SCULPT_TOOL_INFLATE:
1194                 ED_undo_push(C, "Inflate Brush"); break;
1195         case SCULPT_TOOL_GRAB:
1196                 ED_undo_push(C, "Grab Brush"); break;
1197         case SCULPT_TOOL_LAYER:
1198                 ED_undo_push(C, "Layer Brush"); break;
1199         case SCULPT_TOOL_FLATTEN:
1200                 ED_undo_push(C, "Flatten Brush"); break;
1201         default:
1202                 ED_undo_push(C, "Sculpting"); break;
1203         }
1204 }
1205
1206 /**** Radial control ****/
1207 static int sculpt_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1208 {
1209         Paint *p = paint_get_active(CTX_data_scene(C));
1210         Brush *brush = paint_brush(p);
1211
1212         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1213         p->paint_cursor = NULL;
1214         brush_radial_control_invoke(op, brush, 1);
1215         return WM_radial_control_invoke(C, op, event);
1216 }
1217
1218 static int sculpt_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1219 {
1220         int ret = WM_radial_control_modal(C, op, event);
1221         if(ret != OPERATOR_RUNNING_MODAL)
1222                 paint_cursor_start(C, sculpt_poll);
1223         return ret;
1224 }
1225
1226 static int sculpt_radial_control_exec(bContext *C, wmOperator *op)
1227 {
1228         Brush *brush = paint_brush(&CTX_data_tool_settings(C)->sculpt->paint);
1229         int ret = brush_radial_control_exec(op, brush, 1);
1230
1231         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1232
1233         return ret;
1234 }
1235
1236 static void SCULPT_OT_radial_control(wmOperatorType *ot)
1237 {
1238         WM_OT_radial_control_partial(ot);
1239
1240         ot->name= "Sculpt Radial Control";
1241         ot->idname= "SCULPT_OT_radial_control";
1242
1243         ot->invoke= sculpt_radial_control_invoke;
1244         ot->modal= sculpt_radial_control_modal;
1245         ot->exec= sculpt_radial_control_exec;
1246         ot->poll= sculpt_poll;
1247
1248         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1249 }
1250
1251 /**** Operator for applying a stroke (various attributes including mouse path)
1252       using the current brush. ****/
1253
1254 static float unproject_brush_radius(SculptSession *ss, float offset)
1255 {
1256         float brush_edge[3];
1257
1258         /* In anchored mode, brush size changes with mouse loc, otherwise it's fixed using the brush radius */
1259         view3d_unproject(ss->cache->mats, brush_edge, ss->cache->initial_mouse[0] + offset,
1260                   ss->cache->initial_mouse[1], ss->cache->depth);
1261
1262         return VecLenf(ss->cache->true_location, brush_edge);
1263 }
1264
1265 static void sculpt_cache_free(StrokeCache *cache)
1266 {
1267         int i;
1268         if(cache->orig_norms)
1269                 MEM_freeN(cache->orig_norms);
1270         if(cache->face_norms)
1271                 MEM_freeN(cache->face_norms);
1272         if(cache->mats)
1273                 MEM_freeN(cache->mats);
1274         for(i = 0; i < 8; ++i) 
1275                 BLI_freelistN(&cache->grab_active_verts[i]);
1276         MEM_freeN(cache);
1277 }
1278
1279 /* Initialize the stroke cache invariants from operator properties */
1280 static void sculpt_update_cache_invariants(Sculpt *sd, SculptSession *ss, bContext *C, wmOperator *op)
1281 {
1282         StrokeCache *cache = MEM_callocN(sizeof(StrokeCache), "stroke cache");
1283         Brush *brush = paint_brush(&sd->paint);
1284         ViewContext *vc = paint_stroke_view_context(op->customdata);
1285         int i;
1286
1287         ss->cache = cache;
1288
1289         RNA_float_get_array(op->ptr, "scale", cache->scale);
1290         cache->flag = RNA_int_get(op->ptr, "flag");
1291         RNA_float_get_array(op->ptr, "clip_tolerance", cache->clip_tolerance);
1292         RNA_float_get_array(op->ptr, "initial_mouse", cache->initial_mouse);
1293         cache->depth = RNA_float_get(op->ptr, "depth");
1294
1295         cache->mouse[0] = cache->initial_mouse[0];
1296         cache->mouse[1] = cache->initial_mouse[1];
1297
1298         /* Truly temporary data that isn't stored in properties */
1299
1300         cache->mats = MEM_callocN(sizeof(bglMats), "sculpt bglMats");
1301         view3d_get_transformation(vc, vc->obact, cache->mats);
1302
1303         sculpt_update_mesh_elements(C);
1304
1305         /* Initialize layer brush displacements */
1306         if(brush->sculpt_tool == SCULPT_TOOL_LAYER &&
1307            (!ss->layer_disps || !(brush->flag & BRUSH_PERSISTENT))) {
1308                 if(ss->layer_disps)
1309                         MEM_freeN(ss->layer_disps);
1310                 ss->layer_disps = MEM_callocN(sizeof(float) * ss->totvert, "layer brush displacements");
1311         }
1312
1313         /* Make copies of the mesh vertex locations and normals for some tools */
1314         if(brush->sculpt_tool == SCULPT_TOOL_LAYER || (brush->flag & BRUSH_ANCHORED)) {
1315                 if(brush->sculpt_tool != SCULPT_TOOL_LAYER ||
1316                    !ss->mesh_co_orig || !(brush->flag & BRUSH_PERSISTENT)) {
1317                         if(!ss->mesh_co_orig)
1318                                 ss->mesh_co_orig= MEM_mallocN(sizeof(float) * 3 * ss->totvert,
1319                                                                        "sculpt mesh vertices copy");
1320                         for(i = 0; i < ss->totvert; ++i)
1321                                 VecCopyf(ss->mesh_co_orig[i], ss->mvert[i].co);
1322                 }
1323
1324                 if(brush->flag & BRUSH_ANCHORED) {
1325                         cache->orig_norms= MEM_mallocN(sizeof(short) * 3 * ss->totvert, "Sculpt orig norm");
1326                         for(i = 0; i < ss->totvert; ++i) {
1327                                 cache->orig_norms[i][0] = ss->mvert[i].no[0];
1328                                 cache->orig_norms[i][1] = ss->mvert[i].no[1];
1329                                 cache->orig_norms[i][2] = ss->mvert[i].no[2];
1330                         }
1331
1332                         if(ss->face_normals) {
1333                                 float *fn = ss->face_normals;
1334                                 cache->face_norms= MEM_mallocN(sizeof(float) * 3 * ss->totface, "Sculpt face norms");
1335                                 for(i = 0; i < ss->totface; ++i, fn += 3)
1336                                         VecCopyf(cache->face_norms[i], fn);
1337                         }
1338                 }
1339         }
1340
1341         view3d_unproject(cache->mats, cache->true_location, cache->initial_mouse[0], cache->initial_mouse[1], cache->depth);
1342         cache->initial_radius = unproject_brush_radius(ss, brush->size);
1343         cache->rotation = 0;
1344         cache->first_time = 1;
1345 }
1346
1347 /* Initialize the stroke cache variants from operator properties */
1348 static void sculpt_update_cache_variants(Sculpt *sd, SculptSession *ss, PointerRNA *ptr)
1349 {
1350         StrokeCache *cache = ss->cache;
1351         Brush *brush = paint_brush(&sd->paint);
1352         float grab_location[3];
1353         
1354         int dx, dy;
1355
1356         if(!(brush->flag & BRUSH_ANCHORED))
1357                 RNA_float_get_array(ptr, "location", cache->true_location);
1358         cache->flip = RNA_boolean_get(ptr, "flip");
1359         RNA_float_get_array(ptr, "mouse", cache->mouse);
1360         cache->pressure = RNA_float_get(ptr, "pressure");
1361         
1362         /* Truly temporary data that isn't stored in properties */
1363
1364         cache->previous_pixel_radius = cache->pixel_radius;
1365         cache->pixel_radius = brush->size;
1366
1367         if(brush->flag & BRUSH_SIZE_PRESSURE) {
1368                 cache->pixel_radius *= cache->pressure;
1369                 cache->radius = cache->initial_radius * cache->pressure;
1370         }
1371         else
1372                 cache->radius = cache->initial_radius;
1373
1374         if(brush->flag & BRUSH_ANCHORED) {
1375                 dx = cache->mouse[0] - cache->initial_mouse[0];
1376                 dy = cache->mouse[1] - cache->initial_mouse[1];
1377                 cache->pixel_radius = sqrt(dx*dx + dy*dy);
1378                 cache->radius = unproject_brush_radius(ss, cache->pixel_radius);
1379                 cache->rotation = atan2(dy, dx);
1380         }
1381         else if(brush->flag & BRUSH_RAKE) {
1382                 int update;
1383
1384                 dx = cache->last_rake[0] - cache->mouse[0];
1385                 dy = cache->last_rake[1] - cache->mouse[1];
1386
1387                 update = dx*dx + dy*dy > 100;
1388
1389                 /* To prevent jitter, only update the angle if the mouse has moved over 10 pixels */
1390                 if(update && !cache->first_time)
1391                         cache->rotation = M_PI_2 + atan2(dy, dx);
1392
1393                 if(update || cache->first_time) {
1394                         cache->last_rake[0] = cache->mouse[0];
1395                         cache->last_rake[1] = cache->mouse[1];
1396                 }
1397         }
1398
1399         /* Find the grab delta */
1400         if(brush->sculpt_tool == SCULPT_TOOL_GRAB) {
1401                 view3d_unproject(cache->mats, grab_location, cache->mouse[0], cache->mouse[1], cache->depth);
1402                 if(!cache->first_time)
1403                         VecSubf(cache->grab_delta, grab_location, cache->old_grab_location);
1404                 VecCopyf(cache->old_grab_location, grab_location);
1405         }
1406 }
1407
1408 /* Initialize stroke operator properties */
1409 static void sculpt_brush_stroke_init_properties(bContext *C, wmOperator *op, wmEvent *event, SculptSession *ss)
1410 {
1411         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
1412         Object *ob= CTX_data_active_object(C);
1413         ModifierData *md;
1414         float scale[3], clip_tolerance[3] = {0,0,0};
1415         float mouse[2];
1416         int flag = 0;
1417
1418         /* Set scaling adjustment */
1419         scale[0] = 1.0f / ob->size[0];
1420         scale[1] = 1.0f / ob->size[1];
1421         scale[2] = 1.0f / ob->size[2];
1422         RNA_float_set_array(op->ptr, "scale", scale);
1423
1424         /* Initialize mirror modifier clipping */
1425         for(md= ob->modifiers.first; md; md= md->next) {
1426                 if(md->type==eModifierType_Mirror && (md->mode & eModifierMode_Realtime)) {
1427                         const MirrorModifierData *mmd = (MirrorModifierData*) md;
1428                         
1429                         /* Mark each axis that needs clipping along with its tolerance */
1430                         if(mmd->flag & MOD_MIR_CLIPPING) {
1431                                 flag |= CLIP_X << mmd->axis;
1432                                 if(mmd->tolerance > clip_tolerance[mmd->axis])
1433                                         clip_tolerance[mmd->axis] = mmd->tolerance;
1434                         }
1435                 }
1436         }
1437         RNA_int_set(op->ptr, "flag", flag);
1438         RNA_float_set_array(op->ptr, "clip_tolerance", clip_tolerance);
1439
1440         /* Initial mouse location */
1441         mouse[0] = event->x;
1442         mouse[1] = event->y;
1443         RNA_float_set_array(op->ptr, "initial_mouse", mouse);
1444
1445         /* Initial screen depth under the mouse */
1446         RNA_float_set(op->ptr, "depth", read_cached_depth(paint_stroke_view_context(op->customdata), event->x, event->y));
1447
1448         sculpt_update_cache_invariants(sd, ss, C, op);
1449 }
1450
1451 static void sculpt_brush_stroke_init(bContext *C)
1452 {
1453         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
1454         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1455
1456         view3d_operator_needs_opengl(C);
1457
1458         /* TODO: Shouldn't really have to do this at the start of every
1459            stroke, but sculpt would need some sort of notification when
1460            changes are made to the texture. */
1461         sculpt_update_tex(sd, ss);
1462
1463         sculpt_update_mesh_elements(C);
1464 }
1465
1466 static void sculpt_restore_mesh(Sculpt *sd, SculptSession *ss)
1467 {
1468         StrokeCache *cache = ss->cache;
1469         Brush *brush = paint_brush(&sd->paint);
1470         float *buffer= NULL;
1471         int i;
1472
1473         /* Restore the mesh before continuing with anchored stroke */
1474         if((brush->flag & BRUSH_ANCHORED) && ss->mesh_co_orig) {
1475
1476                 if(ss->drawobject)
1477                         buffer= (float *)GPU_buffer_lock(ss->drawobject->normals);
1478
1479                 for(i = 0; i < ss->totvert; ++i) {
1480                         VecCopyf(ss->mvert[i].co, ss->mesh_co_orig[i]);
1481                         ss->mvert[i].no[0] = cache->orig_norms[i][0];
1482                         ss->mvert[i].no[1] = cache->orig_norms[i][1];
1483                         ss->mvert[i].no[2] = cache->orig_norms[i][2];
1484                         if( buffer != 0 ) {
1485                                 IndexLink *cur = &ss->drawobject->indices[i];
1486                                 while( cur != 0 && cur->element != -1 ) {
1487                                         VECCOPY(&buffer[cur->element*3],cache->orig_norms[i]);
1488                                         cur = cur->next;
1489                                 }
1490                         }
1491                 }
1492                 if( buffer != 0 )
1493                         GPU_buffer_unlock( ss->drawobject->normals );
1494
1495                 if(ss->face_normals) {
1496                         float *fn = ss->face_normals;
1497                         for(i = 0; i < ss->totface; ++i, fn += 3)
1498                                 VecCopyf(fn, cache->face_norms[i]);
1499                 }
1500
1501                 if(brush->sculpt_tool == SCULPT_TOOL_LAYER)
1502                         memset(ss->layer_disps, 0, sizeof(float) * ss->totvert);
1503         }
1504 }
1505
1506 static void sculpt_post_stroke_free(SculptSession *ss)
1507 {
1508         BLI_freelistN(&ss->damaged_rects);
1509         BLI_freelistN(&ss->damaged_verts);
1510 }
1511
1512 static void sculpt_flush_update(bContext *C)
1513 {
1514         Object *ob = CTX_data_active_object(C);
1515         SculptSession *ss = ob->sculpt;
1516         ARegion *ar = CTX_wm_region(C);
1517         MultiresModifierData *mmd = ss->multires;
1518
1519         calc_damaged_verts(ss);
1520
1521         if(mmd) {
1522                 if(mmd->undo_verts && mmd->undo_verts != ss->mvert)
1523                         MEM_freeN(mmd->undo_verts);
1524                 
1525                 mmd->undo_verts = ss->mvert;
1526                 mmd->undo_verts_tot = ss->totvert;
1527                 multires_mark_as_modified(ob);
1528         }
1529
1530         ED_region_tag_redraw(ar);
1531 }
1532
1533 static int sculpt_stroke_test_start(bContext *C, struct wmOperator *op, wmEvent *event)
1534 {
1535         ViewContext vc;
1536         float cur_depth;
1537
1538         view3d_set_viewcontext(C, &vc);
1539         cur_depth = read_cached_depth(&vc, event->x, event->y);
1540         
1541         /* Don't start the stroke until a valid depth is found */
1542         if(cur_depth < 1.0 - FLT_EPSILON) {
1543                 SculptSession *ss = CTX_data_active_object(C)->sculpt;
1544
1545                 sculpt_brush_stroke_init_properties(C, op, event, ss);
1546                 sculptmode_update_all_projverts(ss);
1547
1548                 return 1;
1549         }
1550         else
1551                 return 0;
1552 }
1553
1554 static void sculpt_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1555 {
1556         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
1557         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1558
1559         sculpt_update_cache_variants(sd, ss, itemptr);
1560         sculpt_restore_mesh(sd, ss);
1561         do_symmetrical_brush_actions(sd, ss);
1562
1563         /* Cleanup */
1564         sculpt_flush_update(C);
1565         sculpt_post_stroke_free(ss);
1566 }
1567
1568 static void sculpt_stroke_done(bContext *C, struct PaintStroke *stroke)
1569 {
1570         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1571
1572         /* Finished */
1573         if(ss->cache) {
1574                 Sculpt *sd = CTX_data_tool_settings(C)->sculpt;         
1575
1576                 request_depth_update(paint_stroke_view_context(stroke)->rv3d);
1577                 sculpt_cache_free(ss->cache);
1578                 ss->cache = NULL;
1579                 sculpt_undo_push(C, sd);
1580         }
1581 }
1582
1583 static int sculpt_brush_stroke_invoke(bContext *C, wmOperator *op, wmEvent *event)
1584 {
1585         sculpt_brush_stroke_init(C);
1586
1587         op->customdata = paint_stroke_new(C, sculpt_stroke_test_start,
1588                                           sculpt_stroke_update_step,
1589                                           sculpt_stroke_done);
1590
1591         /* add modal handler */
1592         WM_event_add_modal_handler(C, op);
1593
1594         op->type->modal(C, op, event);
1595         
1596         return OPERATOR_RUNNING_MODAL;
1597 }
1598
1599 static int sculpt_brush_stroke_exec(bContext *C, wmOperator *op)
1600 {
1601         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
1602         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1603
1604         op->customdata = paint_stroke_new(C, sculpt_stroke_test_start, sculpt_stroke_update_step, sculpt_stroke_done);
1605
1606         sculpt_brush_stroke_init(C);
1607
1608         sculpt_update_cache_invariants(sd, ss, C, op);
1609         sculptmode_update_all_projverts(ss);
1610
1611         paint_stroke_exec(C, op);
1612
1613         sculpt_flush_update(C);
1614         sculpt_cache_free(ss->cache);
1615
1616         sculpt_undo_push(C, sd);
1617
1618         return OPERATOR_FINISHED;
1619 }
1620
1621 static void SCULPT_OT_brush_stroke(wmOperatorType *ot)
1622 {
1623         ot->flag |= OPTYPE_REGISTER;
1624
1625         /* identifiers */
1626         ot->name= "Sculpt Mode";
1627         ot->idname= "SCULPT_OT_brush_stroke";
1628         
1629         /* api callbacks */
1630         ot->invoke= sculpt_brush_stroke_invoke;
1631         ot->modal= paint_stroke_modal;
1632         ot->exec= sculpt_brush_stroke_exec;
1633         ot->poll= sculpt_poll;
1634         
1635         /* flags (sculpt does own undo? (ton) */
1636         ot->flag= OPTYPE_REGISTER|OPTYPE_BLOCKING;
1637
1638         /* properties */
1639         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1640
1641         /* If the object has a scaling factor, brushes also need to be scaled
1642            to work as expected. */
1643         RNA_def_float_vector(ot->srna, "scale", 3, NULL, 0.0f, FLT_MAX, "Scale", "", 0.0f, 1000.0f);
1644
1645         RNA_def_int(ot->srna, "flag", 0, 0, INT_MAX, "flag", "", 0, INT_MAX);
1646
1647         /* For mirror modifiers */
1648         RNA_def_float_vector(ot->srna, "clip_tolerance", 3, NULL, 0.0f, FLT_MAX, "clip_tolerance", "", 0.0f, 1000.0f);
1649
1650         /* The initial 2D location of the mouse */
1651         RNA_def_float_vector(ot->srna, "initial_mouse", 2, NULL, INT_MIN, INT_MAX, "initial_mouse", "", INT_MIN, INT_MAX);
1652
1653         /* The initial screen depth of the mouse */
1654         RNA_def_float(ot->srna, "depth", 0.0f, 0.0f, FLT_MAX, "depth", "", 0.0f, FLT_MAX);
1655 }
1656
1657 /**** Reset the copy of the mesh that is being sculpted on (currently just for the layer brush) ****/
1658
1659 static int sculpt_set_persistent_base(bContext *C, wmOperator *op)
1660 {
1661         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1662
1663         if(ss) {
1664                 if(ss->layer_disps)
1665                         MEM_freeN(ss->layer_disps);
1666                 ss->layer_disps = NULL;
1667
1668                 if(ss->mesh_co_orig)
1669                         MEM_freeN(ss->mesh_co_orig);
1670                 ss->mesh_co_orig = NULL;
1671         }
1672
1673         return OPERATOR_FINISHED;
1674 }
1675
1676 static void SCULPT_OT_set_persistent_base(wmOperatorType *ot)
1677 {
1678         /* identifiers */
1679         ot->name= "Set Persistent Base";
1680         ot->idname= "SCULPT_OT_set_persistent_base";
1681         
1682         /* api callbacks */
1683         ot->exec= sculpt_set_persistent_base;
1684         ot->poll= sculpt_mode_poll;
1685         
1686         ot->flag= OPTYPE_REGISTER;
1687 }
1688
1689 /**** Toggle operator for turning sculpt mode on or off ****/
1690
1691 static int sculpt_toggle_mode(bContext *C, wmOperator *op)
1692 {
1693         ToolSettings *ts = CTX_data_tool_settings(C);
1694         Object *ob = CTX_data_active_object(C);
1695
1696         if(ob->mode & OB_MODE_SCULPT) {
1697                 multires_force_update(ob);
1698
1699                 /* Leave sculptmode */
1700                 ob->mode &= ~OB_MODE_SCULPT;
1701
1702                 free_sculptsession(&ob->sculpt);
1703         }
1704         else {
1705                 /* Enter sculptmode */
1706
1707                 ob->mode |= OB_MODE_SCULPT;
1708                 
1709                 /* Create persistent sculpt mode data */
1710                 if(!ts->sculpt)
1711                         ts->sculpt = MEM_callocN(sizeof(Sculpt), "sculpt mode data");
1712
1713                 /* Create sculpt mode session data */
1714                 if(ob->sculpt)
1715                         free_sculptsession(&ob->sculpt);
1716                 ob->sculpt = MEM_callocN(sizeof(SculptSession), "sculpt session");
1717
1718                 paint_init(&ts->sculpt->paint, PAINT_CURSOR_SCULPT);
1719                 
1720                 paint_cursor_start(C, sculpt_poll);
1721         }
1722
1723         WM_event_add_notifier(C, NC_SCENE|ND_MODE, CTX_data_scene(C));
1724
1725         return OPERATOR_FINISHED;
1726 }
1727
1728 static void SCULPT_OT_sculptmode_toggle(wmOperatorType *ot)
1729 {
1730         /* identifiers */
1731         ot->name= "Sculpt Mode";
1732         ot->idname= "SCULPT_OT_sculptmode_toggle";
1733         
1734         /* api callbacks */
1735         ot->exec= sculpt_toggle_mode;
1736         ot->poll= ED_operator_object_active;
1737         
1738         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1739 }
1740
1741 void ED_operatortypes_sculpt()
1742 {
1743         WM_operatortype_append(SCULPT_OT_radial_control);
1744         WM_operatortype_append(SCULPT_OT_brush_stroke);
1745         WM_operatortype_append(SCULPT_OT_sculptmode_toggle);
1746         WM_operatortype_append(SCULPT_OT_set_persistent_base);
1747 }