sculpt mode crash fix when VBOs disabled
[blender.git] / source / blender / editors / sculpt_paint / sculpt.c
1 /*
2  * $Id$
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software  Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2006 by Nicholas Bishop
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  *
29  * Implements the Sculpt Mode tools
30  *
31  */
32
33 #include "MEM_guardedalloc.h"
34
35 #include "BLI_arithb.h"
36 #include "BLI_blenlib.h"
37 #include "BLI_dynstr.h"
38
39 #include "DNA_armature_types.h"
40 #include "DNA_brush_types.h"
41 #include "DNA_image_types.h"
42 #include "DNA_key_types.h"
43 #include "DNA_mesh_types.h"
44 #include "DNA_meshdata_types.h"
45 #include "DNA_modifier_types.h"
46 #include "DNA_object_types.h"
47 #include "DNA_screen_types.h"
48 #include "DNA_scene_types.h"
49 #include "DNA_texture_types.h"
50 #include "DNA_view3d_types.h"
51 #include "DNA_userdef_types.h"
52 #include "DNA_color_types.h"
53
54 #include "BKE_brush.h"
55 #include "BKE_context.h"
56 #include "BKE_customdata.h"
57 #include "BKE_DerivedMesh.h"
58 #include "BKE_depsgraph.h"
59 #include "BKE_global.h"
60 #include "BKE_image.h"
61 #include "BKE_key.h"
62 #include "BKE_library.h"
63 #include "BKE_main.h"
64 #include "BKE_mesh.h"
65 #include "BKE_modifier.h"
66 #include "BKE_multires.h"
67 #include "BKE_paint.h"
68 #include "BKE_texture.h"
69 #include "BKE_utildefines.h"
70 #include "BKE_colortools.h"
71
72 #include "BIF_gl.h"
73 #include "BIF_glutil.h"
74
75 #include "WM_api.h"
76 #include "WM_types.h"
77 #include "ED_screen.h"
78 #include "ED_sculpt.h"
79 #include "ED_space_api.h"
80 #include "ED_util.h"
81 #include "ED_view3d.h"
82 #include "paint_intern.h"
83 #include "sculpt_intern.h"
84
85 #include "RNA_access.h"
86 #include "RNA_define.h"
87
88 #include "IMB_imbuf_types.h"
89
90 #include "RE_render_ext.h"
91 #include "RE_shader_ext.h" /*for multitex_ext*/
92
93 #include "GPU_draw.h"
94 #include "gpu_buffers.h"
95
96 #include <math.h>
97 #include <stdlib.h>
98 #include <string.h>
99
100 /* Number of vertices to average in order to determine the flatten distance */
101 #define FLATTEN_SAMPLE_SIZE 10
102
103 /* ===== STRUCTS =====
104  *
105  */
106
107 /* ActiveData stores an Index into the mvert array of Mesh, plus Fade, which
108    stores how far the vertex is from the brush center, scaled to the range [0,1]. */
109 typedef struct ActiveData {
110         struct ActiveData *next, *prev;
111         unsigned int Index;
112         float Fade;
113         float dist;
114 } ActiveData;
115
116 typedef enum StrokeFlags {
117         CLIP_X = 1,
118         CLIP_Y = 2,
119         CLIP_Z = 4
120 } StrokeFlags;
121
122 /* Cache stroke properties. Used because
123    RNA property lookup isn't particularly fast.
124
125    For descriptions of these settings, check the operator properties.
126 */
127 typedef struct StrokeCache {
128         /* Invariants */
129         float initial_radius;
130         float scale[3];
131         int flag;
132         float clip_tolerance[3];
133         float initial_mouse[2];
134         float depth;
135
136         /* Variants */
137         float radius;
138         float true_location[3];
139         float location[3];
140         float flip;
141         float pressure;
142         float mouse[2];
143
144         /* The rest is temporary storage that isn't saved as a property */
145
146         int first_time; /* Beginning of stroke may do some things special */
147
148         bglMats *mats;
149
150         short (*orig_norms)[3]; /* Copy of the mesh vertices' normals */
151         float (*face_norms)[3]; /* Copy of the mesh faces' normals */
152         float rotation; /* Texture rotation (radians) for anchored and rake modes */
153         int pixel_radius, previous_pixel_radius;
154         ListBase grab_active_verts[8]; /* The same list of verts is used throught grab stroke */
155         float grab_delta[3], grab_delta_symmetry[3];
156         float old_grab_location[3];
157         int symmetry; /* Symmetry index between 0 and 7 */
158         float view_normal[3], view_normal_symmetry[3];
159         int last_rake[2]; /* Last location of updating rake rotation */
160 } StrokeCache;
161
162 typedef struct RectNode {
163         struct RectNode *next, *prev;
164         rcti r;
165 } RectNode;
166
167 /* Used to store to 2D screen coordinates of each vertex in the mesh. */
168 typedef struct ProjVert {
169         short co[2];
170         
171         /* Used to mark whether a vertex is inside a rough bounding box
172            containing the brush. */
173         char inside;
174 } ProjVert;
175
176 /* ===== OPENGL =====
177  *
178  * Simple functions to get data from the GL
179  */
180
181 /* Convert a point in model coordinates to 2D screen coordinates. */
182 static void projectf(bglMats *mats, const float v[3], float p[2])
183 {
184         double ux, uy, uz;
185
186         gluProject(v[0],v[1],v[2], mats->modelview, mats->projection,
187                    (GLint *)mats->viewport, &ux, &uy, &uz);
188         p[0]= ux;
189         p[1]= uy;
190 }
191
192 static void project(bglMats *mats, const float v[3], short p[2])
193 {
194         float f[2];
195         projectf(mats, v, f);
196
197         p[0]= f[0];
198         p[1]= f[1];
199 }
200
201 /* ===== Sculpting =====
202  *
203  */
204
205 /* Return modified brush strength. Includes the direction of the brush, positive
206    values pull vertices, negative values push. Uses tablet pressure and a
207    special multiplier found experimentally to scale the strength factor. */
208 static float brush_strength(Sculpt *sd, StrokeCache *cache)
209 {
210         Brush *brush = paint_brush(&sd->paint);
211         /* Primary strength input; square it to make lower values more sensitive */
212         float alpha = brush->alpha * brush->alpha;
213
214         float dir= brush->flag & BRUSH_DIR_IN ? -1 : 1;
215         float pressure= 1;
216         float flip= cache->flip ? -1:1;
217
218         if(brush->flag & BRUSH_ALPHA_PRESSURE)
219                 pressure *= cache->pressure;
220         
221         switch(brush->sculpt_tool){
222         case SCULPT_TOOL_DRAW:
223         case SCULPT_TOOL_INFLATE:
224         case SCULPT_TOOL_CLAY:
225         case SCULPT_TOOL_FLATTEN:
226         case SCULPT_TOOL_LAYER:
227                 return alpha * dir * pressure * flip; /*XXX: not sure why? was multiplied by G.vd->grid */;
228         case SCULPT_TOOL_SMOOTH:
229                 return alpha * 4 * pressure;
230         case SCULPT_TOOL_PINCH:
231                 return alpha / 2 * dir * pressure * flip;
232         case SCULPT_TOOL_GRAB:
233                 return 1;
234         default:
235                 return 0;
236         }
237 }
238
239 /* Handles clipping against a mirror modifier and SCULPT_LOCK axis flags */
240 static void sculpt_clip(Sculpt *sd, SculptSession *ss, float *co, const float val[3])
241 {
242         int i;
243
244         for(i=0; i<3; ++i) {
245                 if(sd->flags & (SCULPT_LOCK_X << i))
246                         continue;
247
248                 if((ss->cache->flag & (CLIP_X << i)) && (fabs(co[i]) <= ss->cache->clip_tolerance[i]))
249                         co[i]= 0.0f;
250                 else
251                         co[i]= val[i];
252         }               
253 }
254
255 static void add_norm_if(float view_vec[3], float out[3], float out_flip[3], const short no[3])
256 {
257         float fno[3] = {no[0], no[1], no[2]};
258
259         Normalize(fno);
260
261         if((Inpf(view_vec, fno)) > 0) {
262                 VecAddf(out, out, fno);
263         } else {
264                 VecAddf(out_flip, out_flip, fno); /* out_flip is used when out is {0,0,0} */
265         }
266 }
267
268 /* Currently only for the draw brush; finds average normal for all active
269    vertices */
270 static void calc_area_normal(Sculpt *sd, SculptSession *ss, float out[3], const ListBase* active_verts)
271 {
272         Brush *brush = paint_brush(&sd->paint);
273         StrokeCache *cache = ss->cache;
274         ActiveData *node = active_verts->first;
275         const int view = 0; /* XXX: should probably be a flag, not number: brush_type==SCULPT_TOOL_DRAW ? sculptmode_brush()->view : 0; */
276         float out_flip[3];
277         float *out_dir = cache->view_normal_symmetry;
278         
279         out[0]=out[1]=out[2] = out_flip[0]=out_flip[1]=out_flip[2] = 0;
280
281         if(brush->flag & BRUSH_ANCHORED) {
282                 for(; node; node = node->next)
283                         add_norm_if(out_dir, out, out_flip, cache->orig_norms[node->Index]);
284         }
285         else {
286                 for(; node; node = node->next)
287                         add_norm_if(out_dir, out, out_flip, ss->mvert[node->Index].no);
288         }
289
290         if (out[0]==0.0 && out[1]==0.0 && out[2]==0.0) {
291                 VECCOPY(out, out_flip);
292         }
293         
294         Normalize(out);
295
296         if(out_dir) {
297                 out[0] = out_dir[0] * view + out[0] * (10-view);
298                 out[1] = out_dir[1] * view + out[1] * (10-view);
299                 out[2] = out_dir[2] * view + out[2] * (10-view);
300         }
301         
302         Normalize(out);
303 }
304
305 static void do_draw_brush(Sculpt *sd, SculptSession *ss, const ListBase* active_verts)
306 {
307         float area_normal[3];
308         int j;
309         ActiveData *node= active_verts->first;
310         float* buffer;
311
312         calc_area_normal(sd, ss, area_normal, active_verts);
313         
314         buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
315
316         while(node){
317                 float *co= ss->mvert[node->Index].co;
318
319                 const float val[3]= {co[0]+area_normal[0]*ss->cache->radius*node->Fade*ss->cache->scale[0],
320                                      co[1]+area_normal[1]*ss->cache->radius*node->Fade*ss->cache->scale[1],
321                                      co[2]+area_normal[2]*ss->cache->radius*node->Fade*ss->cache->scale[2]};
322
323                 if( buffer != 0 ) {
324                         IndexLink *cur = &ss->drawobject->indices[node->Index];
325                         while( cur != 0 && cur->element != -1 ) {
326                                 sculpt_clip(sd, ss, &buffer[cur->element*3], val);
327                                 cur = cur->next;
328                         }
329                 }
330
331                 sculpt_clip(sd, ss, co, val);
332
333                 node= node->next;
334         }
335         if( buffer != 0 )
336                 GPU_buffer_unlock( ss->drawobject->vertices );
337 }
338
339 /* For the smooth brush, uses the neighboring vertices around vert to calculate
340    a smoothed location for vert. Skips corner vertices (used by only one
341    polygon.) */
342 static void neighbor_average(SculptSession *ss, float avg[3], const int vert)
343 {
344         int i, skip= -1, total=0;
345         IndexNode *node= ss->fmap[vert].first;
346         char ncount= BLI_countlist(&ss->fmap[vert]);
347         MFace *f;
348
349         avg[0] = avg[1] = avg[2] = 0;
350                 
351         /* Don't modify corner vertices */
352         if(ncount==1) {
353                 VecCopyf(avg, ss->mvert[vert].co);
354                 return;
355         }
356
357         while(node){
358                 f= &ss->mface[node->index];
359                 
360                 if(f->v4) {
361                         skip= (f->v1==vert?2:
362                                f->v2==vert?3:
363                                f->v3==vert?0:
364                                f->v4==vert?1:-1);
365                 }
366
367                 for(i=0; i<(f->v4?4:3); ++i) {
368                         if(i != skip && (ncount!=2 || BLI_countlist(&ss->fmap[(&f->v1)[i]]) <= 2)) {
369                                 VecAddf(avg, avg, ss->mvert[(&f->v1)[i]].co);
370                                 ++total;
371                         }
372                 }
373
374                 node= node->next;
375         }
376
377         if(total>0)
378                 VecMulf(avg, 1.0f / total);
379         else
380                 VecCopyf(avg, ss->mvert[vert].co);
381 }
382
383 static void do_smooth_brush(Sculpt *s, SculptSession *ss, const ListBase* active_verts)
384 {
385         ActiveData *node= active_verts->first;
386         float *buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
387         int i;
388         
389         for(i = 0; i < 2; ++i) {
390                 while(node){
391                         float *co= ss->mvert[node->Index].co;
392                         float avg[3], val[3];
393                         
394                         neighbor_average(ss, avg, node->Index);
395                         val[0] = co[0]+(avg[0]-co[0])*node->Fade;
396                         val[1] = co[1]+(avg[1]-co[1])*node->Fade;
397                         val[2] = co[2]+(avg[2]-co[2])*node->Fade;
398                         
399                         sculpt_clip(s, ss, co, val);                    
400                         if( buffer != 0 ) {                             
401                                 IndexLink *cur = &ss->drawobject->indices[node->Index]; 
402                                 while( cur != 0 && cur->element != -1 ) {
403                                         sculpt_clip(s, ss, &buffer[cur->element*3], val);
404                                         cur = cur->next;
405                                 }
406                         }
407                         node= node->next;
408                 }
409         }
410         if( buffer != 0 )
411                 GPU_buffer_unlock( ss->drawobject->vertices );
412 }
413
414 static void do_pinch_brush(Sculpt *s, SculptSession *ss, const ListBase* active_verts)
415 {
416         ActiveData *node= active_verts->first;
417         float *buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
418
419         while(node) {
420                 float *co= ss->mvert[node->Index].co;
421                 const float val[3]= {co[0]+(ss->cache->location[0]-co[0])*node->Fade,
422                                      co[1]+(ss->cache->location[1]-co[1])*node->Fade,
423                                      co[2]+(ss->cache->location[2]-co[2])*node->Fade};
424
425                 if( buffer != 0 ) {
426                         IndexLink *cur = &ss->drawobject->indices[node->Index];
427                         while( cur != 0 && cur->element != -1 ) {
428                                 sculpt_clip(s, ss, &buffer[cur->element*3], val);
429                                 cur = cur->next;
430                         }
431                 }
432
433                 sculpt_clip(s, ss, co, val);
434                 node= node->next;
435         }
436         if( buffer != 0 )
437                 GPU_buffer_unlock( ss->drawobject->vertices );
438 }
439
440 static void do_grab_brush(Sculpt *sd, SculptSession *ss)
441 {
442         ActiveData *node= ss->cache->grab_active_verts[ss->cache->symmetry].first;
443         float add[3];
444         float grab_delta[3];
445         float *buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
446         
447         VecCopyf(grab_delta, ss->cache->grab_delta_symmetry);
448         
449         while(node) {
450                 float *co= ss->mvert[node->Index].co;
451                 
452                 VecCopyf(add, grab_delta);
453                 VecMulf(add, node->Fade);
454                 VecAddf(add, add, co);
455
456                 if( buffer != 0 ) {
457                         IndexLink *cur = &ss->drawobject->indices[node->Index];
458                         while( cur != 0 && cur->element != -1 ) {
459                                 sculpt_clip(sd, ss, &buffer[cur->element*3], add);
460                                 cur = cur->next;
461                         }
462                 }
463
464                 sculpt_clip(sd, ss, co, add);
465
466                 node= node->next;
467         }
468         if( buffer != 0 )
469                 GPU_buffer_unlock( ss->drawobject->vertices );
470         
471 }
472
473 static void do_layer_brush(Sculpt *sd, SculptSession *ss, const ListBase *active_verts)
474 {
475         float area_normal[3];
476         ActiveData *node= active_verts->first;
477         float *buffer;
478         float lim= ss->cache->radius / 4;
479
480         if(ss->cache->flip)
481                 lim = -lim;
482
483         calc_area_normal(sd, ss, area_normal, active_verts);
484
485         buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
486         while(node){
487                 float *disp= &ss->layer_disps[node->Index];
488                 float *co= ss->mvert[node->Index].co;
489                 float val[3];
490                 
491                 *disp+= node->Fade;
492                 
493                 /* Don't let the displacement go past the limit */
494                 if((lim < 0 && *disp < lim) || (lim > 0 && *disp > lim))
495                         *disp = lim;
496                 
497                 val[0] = ss->mesh_co_orig[node->Index][0]+area_normal[0] * *disp*ss->cache->scale[0];
498                 val[1] = ss->mesh_co_orig[node->Index][1]+area_normal[1] * *disp*ss->cache->scale[1];
499                 val[2] = ss->mesh_co_orig[node->Index][2]+area_normal[2] * *disp*ss->cache->scale[2];
500
501                 if( buffer != 0 ) {
502                         IndexLink *cur = &ss->drawobject->indices[node->Index];
503                         while( cur != 0 && cur->element != -1 ) {
504                                 sculpt_clip(sd, ss, &buffer[cur->element*3], val);
505                                 cur = cur->next;
506                         }
507                 }
508
509                 sculpt_clip(sd, ss, co, val);
510
511                 node= node->next;
512         }
513         if( buffer != 0 )
514                 GPU_buffer_unlock( ss->drawobject->vertices );
515 }
516
517 static void do_inflate_brush(Sculpt *s, SculptSession *ss, const ListBase *active_verts)
518 {
519         ActiveData *node= active_verts->first;
520         float add[3];
521         float *buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
522
523         while(node) {
524                 float *co= ss->mvert[node->Index].co;
525                 short *no= ss->mvert[node->Index].no;
526
527                 add[0]= no[0]/ 32767.0f;
528                 add[1]= no[1]/ 32767.0f;
529                 add[2]= no[2]/ 32767.0f;
530                 VecMulf(add, node->Fade * ss->cache->radius);
531                 add[0]*= ss->cache->scale[0];
532                 add[1]*= ss->cache->scale[1];
533                 add[2]*= ss->cache->scale[2];
534                 VecAddf(add, add, co);
535                 
536                 if( buffer != 0 ) {
537                         IndexLink *cur = &ss->drawobject->indices[node->Index];
538                         while( cur != 0 && cur->element != -1 ) {
539                                 sculpt_clip(s, ss, &buffer[cur->element*3], add);
540                                 cur = cur->next;
541                         }
542                 }
543
544                 sculpt_clip(s, ss, co, add);
545
546                 node= node->next;
547         }
548         if( buffer != 0 )
549                 GPU_buffer_unlock( ss->drawobject->vertices );
550 }
551
552 static void calc_flatten_center(SculptSession *ss, ActiveData *node, float co[3])
553 {
554         ActiveData *outer[FLATTEN_SAMPLE_SIZE];
555         int i;
556         
557         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
558                 outer[i] = node;
559                 
560         for(; node; node = node->next) {
561                 for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i) {
562                         if(node->dist > outer[i]->dist) {
563                                 outer[i] = node;
564                                 break;
565                         }
566                 }
567         }
568         
569         co[0] = co[1] = co[2] = 0.0f;
570         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
571                 VecAddf(co, co, ss->mvert[outer[i]->Index].co);
572         VecMulf(co, 1.0f / FLATTEN_SAMPLE_SIZE);
573 }
574
575 /* Projects a point onto a plane along the plane's normal */
576 static void point_plane_project(float intr[3], float co[3], float plane_normal[3], float plane_center[3])
577 {
578         float p1[3], sub1[3], sub2[3];
579
580         /* Find the intersection between squash-plane and vertex (along the area normal) */
581         VecSubf(p1, co, plane_normal);
582         VecSubf(sub1, plane_center, p1);
583         VecSubf(sub2, co, p1);
584         VecSubf(intr, co, p1);
585         VecMulf(intr, Inpf(plane_normal, sub1) / Inpf(plane_normal, sub2));
586         VecAddf(intr, intr, p1);
587 }
588
589 static int plane_point_side(float co[3], float plane_normal[3], float plane_center[3], int flip)
590 {
591         float delta[3];
592         float d;
593
594         VecSubf(delta, co, plane_center);
595         d = Inpf(plane_normal, delta);
596
597         if(flip)
598                 d = -d;
599
600         return d <= 0.0f;
601 }
602
603 static void do_flatten_clay_brush(Sculpt *sd, SculptSession *ss, const ListBase *active_verts, int clay)
604 {
605         ActiveData *node= active_verts->first;
606         /* area_normal and cntr define the plane towards which vertices are squashed */
607         float area_normal[3];
608         float cntr[3], cntr2[3], bstr = 0;
609         int flip = 0;
610         float *buffer;
611         calc_area_normal(sd, ss, area_normal, active_verts);
612         calc_flatten_center(ss, node, cntr);
613
614         if(clay) {
615                 bstr= brush_strength(sd, ss->cache);
616                 /* Limit clay application to here */
617                 cntr2[0]=cntr[0]+area_normal[0]*bstr*ss->cache->scale[0];
618                 cntr2[1]=cntr[1]+area_normal[1]*bstr*ss->cache->scale[1];
619                 cntr2[2]=cntr[2]+area_normal[2]*bstr*ss->cache->scale[2];
620                 flip = bstr < 0;
621         }
622
623         buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->vertices ):0;
624
625         while(node){
626                 float *co= ss->mvert[node->Index].co;
627                 float intr[3], val[3];
628                 
629                 if(!clay || plane_point_side(co, area_normal, cntr2, flip)) {
630                         /* Find the intersection between squash-plane and vertex (along the area normal) */             
631                         point_plane_project(intr, co, area_normal, cntr);
632
633                         VecSubf(val, intr, co);
634
635                         if(clay) {
636                                 if(bstr > FLT_EPSILON)
637                                         VecMulf(val, node->Fade / bstr);
638                                 else
639                                         VecMulf(val, node->Fade);
640                                 /* Clay displacement */
641                                 val[0]+=area_normal[0] * ss->cache->scale[0]*node->Fade;
642                                 val[1]+=area_normal[1] * ss->cache->scale[1]*node->Fade;
643                                 val[2]+=area_normal[2] * ss->cache->scale[2]*node->Fade;
644                         }
645                         else
646                                 VecMulf(val, fabs(node->Fade));
647
648                         VecAddf(val, val, co);
649
650                         if( buffer != 0 ) {
651                                 IndexLink *cur = &ss->drawobject->indices[node->Index];
652                                 while( cur != 0 && cur->element != -1 ) {
653                                         sculpt_clip(sd, ss, &buffer[cur->element*3], val);
654                                         cur = cur->next;
655                                 }
656                         }                       
657                         sculpt_clip(sd, ss, co, val);
658
659                 }
660                 
661                 node= node->next;
662         }
663         if( buffer != 0 )
664                 GPU_buffer_unlock( ss->drawobject->vertices );
665 }
666
667 /* Uses symm to selectively flip any axis of a coordinate. */
668 static void flip_coord(float out[3], float in[3], const char symm)
669 {
670         if(symm & SCULPT_SYMM_X)
671                 out[0]= -in[0];
672         else
673                 out[0]= in[0];
674         if(symm & SCULPT_SYMM_Y)
675                 out[1]= -in[1];
676         else
677                 out[1]= in[1];
678         if(symm & SCULPT_SYMM_Z)
679                 out[2]= -in[2];
680         else
681                 out[2]= in[2];
682 }
683
684 /* Get a pixel from the texcache at (px, py) */
685 static unsigned char get_texcache_pixel(const SculptSession *ss, int px, int py)
686 {
687         unsigned *p;
688         p = ss->texcache + py * ss->texcache_side + px;
689         return ((unsigned char*)(p))[0];
690 }
691
692 static float get_texcache_pixel_bilinear(const SculptSession *ss, float u, float v)
693 {
694         int x, y, x2, y2;
695         const int tc_max = ss->texcache_side - 1;
696         float urat, vrat, uopp;
697
698         if(u < 0) u = 0;
699         else if(u >= ss->texcache_side) u = tc_max;
700         if(v < 0) v = 0;
701         else if(v >= ss->texcache_side) v = tc_max;
702
703         x = floor(u);
704         y = floor(v);
705         x2 = x + 1;
706         y2 = y + 1;
707
708         if(x2 > ss->texcache_side) x2 = tc_max;
709         if(y2 > ss->texcache_side) y2 = tc_max;
710         
711         urat = u - x;
712         vrat = v - y;
713         uopp = 1 - urat;
714                 
715         return ((get_texcache_pixel(ss, x, y) * uopp +
716                  get_texcache_pixel(ss, x2, y) * urat) * (1 - vrat) + 
717                 (get_texcache_pixel(ss, x, y2) * uopp +
718                  get_texcache_pixel(ss, x2, y2) * urat) * vrat) / 255.0;
719 }
720
721 /* Return a multiplier for brush strength on a particular vertex. */
722 static float tex_strength(Sculpt *sd, SculptSession *ss, float *point, const float len)
723 {
724         Brush *br = paint_brush(&sd->paint);
725         MTex *tex = NULL;
726         float avg= 1;
727
728         if(br->texact >= 0)
729                 tex = br->mtex[br->texact];
730
731         if(!tex) {
732                 avg= 1;
733         }
734         else if(tex->brush_map_mode == MTEX_MAP_MODE_3D) {
735                 float jnk;
736
737                 /* Get strength by feeding the vertex 
738                    location directly into a texture */
739                 externtex(tex, point, &avg,
740                           &jnk, &jnk, &jnk, &jnk);
741         }
742         else if(ss->texcache) {
743                 const float bsize= ss->cache->pixel_radius * 2;
744                 const float rot= tex->rot + ss->cache->rotation;
745                 int px, py;
746                 float flip[3], point_2d[2];
747
748                 /* If the active area is being applied for symmetry, flip it
749                    across the symmetry axis in order to project it. This insures
750                    that the brush texture will be oriented correctly. */
751                 VecCopyf(flip, point);
752                 flip_coord(flip, flip, ss->cache->symmetry);
753                 projectf(ss->cache->mats, flip, point_2d);
754
755                 /* For Tile and Drag modes, get the 2D screen coordinates of the
756                    and scale them up or down to the texture size. */
757                 if(tex->brush_map_mode == MTEX_MAP_MODE_TILED) {
758                         const int sx= (const int)tex->size[0];
759                         const int sy= (const int)tex->size[1];
760                         
761                         float fx= point_2d[0];
762                         float fy= point_2d[1];
763                         
764                         float angle= atan2(fy, fx) - rot;
765                         float flen= sqrtf(fx*fx + fy*fy);
766                         
767                         if(rot<0.001 && rot>-0.001) {
768                                 px= point_2d[0];
769                                 py= point_2d[1];
770                         } else {
771                                 px= flen * cos(angle) + 2000;
772                                 py= flen * sin(angle) + 2000;
773                         }
774                         if(sx != 1)
775                                 px %= sx-1;
776                         if(sy != 1)
777                                 py %= sy-1;
778                         avg= get_texcache_pixel_bilinear(ss, ss->texcache_side*px/sx, ss->texcache_side*py/sy);
779                 }
780                 else if(tex->brush_map_mode == MTEX_MAP_MODE_FIXED) {
781                         float fx= (point_2d[0] - ss->cache->mouse[0]) / bsize;
782                         float fy= (point_2d[1] - ss->cache->mouse[1]) / bsize;
783
784                         float angle= atan2(fy, fx) - rot;
785                         float flen= sqrtf(fx*fx + fy*fy);
786                         
787                         fx = flen * cos(angle) + 0.5;
788                         fy = flen * sin(angle) + 0.5;
789
790                         avg= get_texcache_pixel_bilinear(ss, fx * ss->texcache_side, fy * ss->texcache_side);
791                 }
792         }
793
794         avg*= brush_curve_strength(br, len, ss->cache->radius); /* Falloff curve */
795
796         return avg;
797 }
798
799 /* Mark area around the brush as damaged. projverts are marked if they are
800    inside the area and the damaged rectangle in 2D screen coordinates is 
801    added to damaged_rects. */
802 static void sculpt_add_damaged_rect(SculptSession *ss)
803 {
804         short p[2];
805         RectNode *rn= MEM_mallocN(sizeof(RectNode),"RectNode");
806         const float radius = MAX2(ss->cache->pixel_radius, ss->cache->previous_pixel_radius);
807         unsigned i;
808
809         /* Find center */
810         project(ss->cache->mats, ss->cache->location, p);
811         rn->r.xmin= p[0] - radius;
812         rn->r.ymin= p[1] - radius;
813         rn->r.xmax= p[0] + radius;
814         rn->r.ymax= p[1] + radius;
815
816         BLI_addtail(&ss->damaged_rects, rn);
817
818         /* Update insides */
819         for(i=0; i<ss->totvert; ++i) {
820                 if(!ss->projverts[i].inside) {
821                         if(ss->projverts[i].co[0] > rn->r.xmin && ss->projverts[i].co[1] > rn->r.ymin &&
822                            ss->projverts[i].co[0] < rn->r.xmax && ss->projverts[i].co[1] < rn->r.ymax) {
823                                 ss->projverts[i].inside= 1;
824                         }
825                 }
826                 // XXX: remember to fix this!
827                 // temporary pass
828                 ss->projverts[i].inside = 1;
829         }
830 }
831
832 static void do_brush_action(Sculpt *sd, SculptSession *ss, StrokeCache *cache)
833 {
834         Brush *brush = paint_brush(&sd->paint);
835         float av_dist;
836         ListBase active_verts={0,0};
837         ListBase *grab_active_verts = &ss->cache->grab_active_verts[ss->cache->symmetry];
838         ActiveData *adata= 0;
839         float *vert;
840         Mesh *me= NULL; /*XXX: get_mesh(OBACT); */
841         const float bstrength= brush_strength(sd, cache);
842         KeyBlock *keyblock= NULL; /*XXX: ob_get_keyblock(OBACT); */
843         Brush *b = brush;
844         int i;
845
846         sculpt_add_damaged_rect(ss);
847
848         /* Build a list of all vertices that are potentially within the brush's
849            area of influence. Only do this once for the grab brush. */
850         if((b->sculpt_tool != SCULPT_TOOL_GRAB) || cache->first_time) {
851                 for(i=0; i<ss->totvert; ++i) {
852                         /* Projverts.inside provides a rough bounding box */
853                         if(ss->multires || ss->projverts[i].inside) {
854                                 //vert= ss->vertexcosnos ? &ss->vertexcosnos[i*6] : a->verts[i].co;
855                                 vert= ss->mvert[i].co;
856                                 av_dist= VecLenf(ss->cache->location, vert);
857                                 if(av_dist < cache->radius) {
858                                         adata= (ActiveData*)MEM_mallocN(sizeof(ActiveData), "ActiveData");
859
860                                         adata->Index = i;
861                                         /* Fade is used to store the final strength at which the brush
862                                            should modify a particular vertex. */
863                                         adata->Fade= tex_strength(sd, ss, vert, av_dist) * bstrength;
864                                         adata->dist = av_dist;
865
866                                         if(b->sculpt_tool == SCULPT_TOOL_GRAB && cache->first_time)
867                                                 BLI_addtail(grab_active_verts, adata);
868                                         else
869                                                 BLI_addtail(&active_verts, adata);
870                                 }
871                         }
872                 }
873         }
874
875         /* Only act if some verts are inside the brush area */
876         if(active_verts.first || (b->sculpt_tool == SCULPT_TOOL_GRAB && grab_active_verts->first)) {
877                 /* Apply one type of brush action */
878                 switch(b->sculpt_tool){
879                 case SCULPT_TOOL_DRAW:
880                         do_draw_brush(sd, ss, &active_verts);
881                         break;
882                 case SCULPT_TOOL_SMOOTH:
883                         do_smooth_brush(sd, ss, &active_verts);
884                         break;
885                 case SCULPT_TOOL_PINCH:
886                         do_pinch_brush(sd, ss, &active_verts);
887                         break;
888                 case SCULPT_TOOL_INFLATE:
889                         do_inflate_brush(sd, ss, &active_verts);
890                         break;
891                 case SCULPT_TOOL_GRAB:
892                         do_grab_brush(sd, ss);
893                         break;
894                 case SCULPT_TOOL_LAYER:
895                         do_layer_brush(sd, ss, &active_verts);
896                         break;
897                 case SCULPT_TOOL_FLATTEN:
898                         do_flatten_clay_brush(sd, ss, &active_verts, 0);
899                         break;
900                 case SCULPT_TOOL_CLAY:
901                         do_flatten_clay_brush(sd, ss, &active_verts, 1);
902                 }
903         
904                 /* Copy the modified vertices from mesh to the active key */
905                 if(keyblock && !ss->multires) {
906                         float *co= keyblock->data;
907                         if(co) {
908                                 if(b->sculpt_tool == SCULPT_TOOL_GRAB)
909                                         adata = grab_active_verts->first;
910                                 else
911                                         adata = active_verts.first;
912
913                                 for(; adata; adata= adata->next)
914                                         if(adata->Index < keyblock->totelem)
915                                                 VecCopyf(&co[adata->Index*3], me->mvert[adata->Index].co);
916                         }
917                 }
918
919                 if(ss->vertexcosnos && !ss->multires)
920                         BLI_freelistN(&active_verts);
921                 else {
922                         if(b->sculpt_tool != SCULPT_TOOL_GRAB)
923                                 addlisttolist(&ss->damaged_verts, &active_verts);
924                 }
925         }
926 }
927
928 /* Flip all the editdata across the axis/axes specified by symm. Used to
929    calculate multiple modifications to the mesh when symmetry is enabled. */
930 static void calc_brushdata_symm(StrokeCache *cache, const char symm)
931 {
932         flip_coord(cache->location, cache->true_location, symm);
933         flip_coord(cache->view_normal_symmetry, cache->view_normal, symm);
934         flip_coord(cache->grab_delta_symmetry, cache->grab_delta, symm);
935         cache->symmetry= symm;
936 }
937
938 static void do_symmetrical_brush_actions(Sculpt *sd, SculptSession *ss)
939 {
940         StrokeCache *cache = ss->cache;
941         const char symm = sd->flags & 7;
942         int i;
943
944         VecCopyf(cache->location, cache->true_location);
945         VecCopyf(cache->grab_delta_symmetry, cache->grab_delta);
946         cache->symmetry = 0;
947         do_brush_action(sd, ss, cache);
948
949         for(i = 1; i <= symm; ++i) {
950                 if(symm & i && (symm != 5 || i != 3) && (symm != 6 || (i != 3 && i != 5))) {
951                         calc_brushdata_symm(cache, i);
952                         do_brush_action(sd, ss, cache);
953                 }
954         }
955
956         cache->first_time = 0;
957 }
958
959 static void add_face_normal(vec3f *norm, MVert *mvert, const MFace* face, float *fn)
960 {
961         vec3f c= {mvert[face->v1].co[0],mvert[face->v1].co[1],mvert[face->v1].co[2]};
962         vec3f b= {mvert[face->v2].co[0],mvert[face->v2].co[1],mvert[face->v2].co[2]};
963         vec3f a= {mvert[face->v3].co[0],mvert[face->v3].co[1],mvert[face->v3].co[2]};
964         vec3f s1, s2;
965         float final[3];
966
967         VecSubf(&s1.x,&a.x,&b.x);
968         VecSubf(&s2.x,&c.x,&b.x);
969
970         final[0] = s1.y * s2.z - s1.z * s2.y;
971         final[1] = s1.z * s2.x - s1.x * s2.z;
972         final[2] = s1.x * s2.y - s1.y * s2.x;
973
974         if(fn)
975                 VecCopyf(fn, final);
976
977         norm->x+= final[0];
978         norm->y+= final[1];
979         norm->z+= final[2];
980 }
981
982 static void update_damaged_vert(SculptSession *ss, ListBase *lb)
983 {
984         ActiveData *vert;
985     
986         float *buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->normals ):0;
987         for(vert= lb->first; vert; vert= vert->next) {
988                 vec3f norm= {0,0,0};            
989                 IndexNode *face= ss->fmap[vert->Index].first;
990
991                 while(face){
992                         float *fn = NULL;
993                         if(ss->face_normals)
994                                 fn = &ss->face_normals[face->index*3];
995                         add_face_normal(&norm, ss->mvert, &ss->mface[face->index], fn);
996                         face= face->next;
997                 }
998                 Normalize(&norm.x);
999                 
1000                 ss->mvert[vert->Index].no[0]=norm.x*32767;
1001                 ss->mvert[vert->Index].no[1]=norm.y*32767;
1002                 ss->mvert[vert->Index].no[2]=norm.z*32767;
1003
1004                 if( buffer != 0 ) {
1005                         IndexLink *cur = &ss->drawobject->indices[vert->Index];
1006                         while( cur != 0 && cur->element != -1 ) {
1007                                 int i = ss->drawobject->faceRemap[cur->element/3];
1008                                 if( ss->mface[i].flag & ME_SMOOTH ) {
1009                                         VECCOPY(&buffer[cur->element*3],ss->mvert[vert->Index].no);
1010                                 }
1011                                 else {
1012                                         float norm[3];
1013                                         if( ss->mface[i].v4 )
1014                                                 CalcNormFloat4(ss->mvert[ss->mface[i].v1].co, ss->mvert[ss->mface[i].v2].co, ss->mvert[ss->mface[i].v3].co, ss->mvert[ss->mface[i].v4].co, norm);
1015                                         else
1016                                                 CalcNormFloat(ss->mvert[ss->mface[i].v1].co, ss->mvert[ss->mface[i].v2].co, ss->mvert[ss->mface[i].v3].co, norm);
1017                                         VECCOPY(&buffer[cur->element*3],norm);
1018                                         VECCOPY(&buffer[cur->element*3],norm);
1019                                         VECCOPY(&buffer[cur->element*3],norm);
1020                                 }
1021
1022                                 //VECCOPY(&buffer[cur->element*3],ss->mvert[vert->Index].no);
1023                                 cur = cur->next;
1024                         }
1025                 }
1026         }
1027         if( buffer != 0 )
1028                 GPU_buffer_unlock( ss->drawobject->normals );
1029 }
1030
1031 static void calc_damaged_verts(SculptSession *ss)
1032 {
1033         int i;
1034         
1035         for(i=0; i<8; ++i)
1036                 update_damaged_vert(ss, &ss->cache->grab_active_verts[i]);
1037         update_damaged_vert(ss, &ss->damaged_verts);
1038         BLI_freelistN(&ss->damaged_verts);
1039         ss->damaged_verts.first = ss->damaged_verts.last = NULL;
1040 }
1041
1042 #if 0
1043 static void projverts_clear_inside(SculptSession *ss)
1044 {
1045         int i;
1046         for(i = 0; i < ss->totvert; ++i)
1047                 ss->projverts[i].inside = 0;
1048 }
1049 #endif
1050
1051 static void sculpt_update_tex(Sculpt *sd, SculptSession *ss)
1052 {
1053         Brush *brush = paint_brush(&sd->paint);
1054
1055         if(ss->texcache) {
1056                 MEM_freeN(ss->texcache);
1057                 ss->texcache= NULL;
1058         }
1059
1060         /* Need to allocate a bigger buffer for bigger brush size */
1061         ss->texcache_side = brush->size * 2;
1062         if(!ss->texcache || ss->texcache_side > ss->texcache_actual) {
1063                 ss->texcache = brush_gen_texture_cache(brush, brush->size);
1064                 ss->texcache_actual = ss->texcache_side;
1065         }
1066 }
1067
1068 static void sculptmode_update_all_projverts(SculptSession *ss)
1069 {
1070         unsigned i;
1071
1072         if(!ss->projverts)
1073                 ss->projverts = MEM_mallocN(sizeof(ProjVert)*ss->totvert,"ProjVerts");
1074
1075         for(i=0; i<ss->totvert; ++i) {
1076                 project(ss->cache->mats, ss->vertexcosnos ? &ss->vertexcosnos[i * 6] : ss->mvert[i].co,
1077                         ss->projverts[i].co);
1078                 ss->projverts[i].inside= 0;
1079         }
1080 }
1081
1082 /* Checks whether full update mode (slower) needs to be used to work with modifiers */
1083 char sculpt_modifiers_active(Object *ob)
1084 {
1085         ModifierData *md;
1086         
1087         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1088                 if(md->mode & eModifierMode_Realtime && md->type != eModifierType_Multires)
1089                         return 1;
1090         }
1091         
1092         return 0;
1093 }
1094
1095 /* Sculpt mode handles multires differently from regular meshes, but only if
1096    it's the last modifier on the stack and it is not on the first level */
1097 static struct MultiresModifierData *sculpt_multires_active(Object *ob)
1098 {
1099         ModifierData *md;
1100         
1101         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1102                 if(md->type == eModifierType_Multires && !md->next) {
1103                         MultiresModifierData *mmd = (MultiresModifierData*)md;
1104                         if(mmd->lvl != 1)
1105                                 return mmd;
1106                 }
1107         }
1108
1109         return NULL;
1110 }
1111
1112 static void sculpt_update_mesh_elements(bContext *C)
1113 {
1114         Object *ob = CTX_data_active_object(C);
1115         SculptSession *ss = ob->sculpt;
1116         int oldtotvert = ss->totvert;
1117         DerivedMesh *dm = mesh_get_derived_final(CTX_data_scene(C), ob, CD_MASK_BAREMESH);
1118
1119         if((ss->multires = sculpt_multires_active(ob))) {
1120                 //DerivedMesh *dm = mesh_get_derived_final(CTX_data_scene(C), ob, CD_MASK_BAREMESH);
1121                 ss->totvert = dm->getNumVerts(dm);
1122                 ss->totface = dm->getNumFaces(dm);
1123                 ss->mvert = dm->getVertDataArray(dm, CD_MVERT);
1124                 ss->mface = dm->getFaceDataArray(dm, CD_MFACE);
1125                 ss->face_normals = dm->getFaceDataArray(dm, CD_NORMAL);
1126         }
1127         else {
1128                 Mesh *me = get_mesh(ob);
1129                 ss->totvert = me->totvert;
1130                 ss->totface = me->totface;
1131                 ss->mvert = me->mvert;
1132                 ss->mface = me->mface;
1133                 ss->face_normals = NULL;
1134         }
1135         if( GPU_buffer_legacy( dm ) ) {
1136                 ss->drawobject = 0;
1137         }
1138         else {
1139                 ss->drawobject = dm->drawObject;
1140         }
1141
1142         if(ss->totvert != oldtotvert) {
1143                 if(ss->projverts) MEM_freeN(ss->projverts);
1144                 ss->projverts = NULL;
1145
1146                 if(ss->fmap) MEM_freeN(ss->fmap);
1147                 if(ss->fmap_mem) MEM_freeN(ss->fmap_mem);
1148                 create_vert_face_map(&ss->fmap, &ss->fmap_mem, ss->mface, ss->totvert, ss->totface);
1149                 ss->fmap_size = ss->totvert;
1150         }
1151 }
1152
1153 static int sculpt_mode_poll(bContext *C)
1154 {
1155         Object *ob = CTX_data_active_object(C);
1156         return ob && ob->mode & OB_MODE_SCULPT;
1157 }
1158
1159 int sculpt_poll(bContext *C)
1160 {
1161         return sculpt_mode_poll(C) && paint_poll(C);
1162 }
1163
1164 static void sculpt_undo_push(bContext *C, Sculpt *sd)
1165 {
1166         Brush *brush = paint_brush(&sd->paint);
1167
1168         switch(brush->sculpt_tool) {
1169         case SCULPT_TOOL_DRAW:
1170                 ED_undo_push(C, "Draw Brush"); break;
1171         case SCULPT_TOOL_SMOOTH:
1172                 ED_undo_push(C, "Smooth Brush"); break;
1173         case SCULPT_TOOL_PINCH:
1174                 ED_undo_push(C, "Pinch Brush"); break;
1175         case SCULPT_TOOL_INFLATE:
1176                 ED_undo_push(C, "Inflate Brush"); break;
1177         case SCULPT_TOOL_GRAB:
1178                 ED_undo_push(C, "Grab Brush"); break;
1179         case SCULPT_TOOL_LAYER:
1180                 ED_undo_push(C, "Layer Brush"); break;
1181         case SCULPT_TOOL_FLATTEN:
1182                 ED_undo_push(C, "Flatten Brush"); break;
1183         default:
1184                 ED_undo_push(C, "Sculpting"); break;
1185         }
1186 }
1187
1188 /**** Radial control ****/
1189 static int sculpt_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1190 {
1191         Paint *p = paint_get_active(CTX_data_scene(C));
1192         Brush *brush = paint_brush(p);
1193
1194         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1195         p->paint_cursor = NULL;
1196         brush_radial_control_invoke(op, brush, 1);
1197         return WM_radial_control_invoke(C, op, event);
1198 }
1199
1200 static int sculpt_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1201 {
1202         int ret = WM_radial_control_modal(C, op, event);
1203         if(ret != OPERATOR_RUNNING_MODAL)
1204                 paint_cursor_start(C, sculpt_poll);
1205         return ret;
1206 }
1207
1208 static int sculpt_radial_control_exec(bContext *C, wmOperator *op)
1209 {
1210         Brush *brush = paint_brush(&CTX_data_tool_settings(C)->sculpt->paint);
1211         int ret = brush_radial_control_exec(op, brush, 1);
1212
1213         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1214
1215         return ret;
1216 }
1217
1218 static void SCULPT_OT_radial_control(wmOperatorType *ot)
1219 {
1220         WM_OT_radial_control_partial(ot);
1221
1222         ot->name= "Sculpt Radial Control";
1223         ot->idname= "SCULPT_OT_radial_control";
1224
1225         ot->invoke= sculpt_radial_control_invoke;
1226         ot->modal= sculpt_radial_control_modal;
1227         ot->exec= sculpt_radial_control_exec;
1228         ot->poll= sculpt_poll;
1229
1230         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1231 }
1232
1233 /**** Operator for applying a stroke (various attributes including mouse path)
1234       using the current brush. ****/
1235
1236 static float unproject_brush_radius(SculptSession *ss, float offset)
1237 {
1238         float brush_edge[3];
1239
1240         /* In anchored mode, brush size changes with mouse loc, otherwise it's fixed using the brush radius */
1241         view3d_unproject(ss->cache->mats, brush_edge, ss->cache->initial_mouse[0] + offset,
1242                   ss->cache->initial_mouse[1], ss->cache->depth);
1243
1244         return VecLenf(ss->cache->true_location, brush_edge);
1245 }
1246
1247 static void sculpt_cache_free(StrokeCache *cache)
1248 {
1249         int i;
1250         if(cache->orig_norms)
1251                 MEM_freeN(cache->orig_norms);
1252         if(cache->face_norms)
1253                 MEM_freeN(cache->face_norms);
1254         if(cache->mats)
1255                 MEM_freeN(cache->mats);
1256         for(i = 0; i < 8; ++i) 
1257                 BLI_freelistN(&cache->grab_active_verts[i]);
1258         MEM_freeN(cache);
1259 }
1260
1261 /* Initialize the stroke cache invariants from operator properties */
1262 static void sculpt_update_cache_invariants(Sculpt *sd, SculptSession *ss, bContext *C, wmOperator *op)
1263 {
1264         StrokeCache *cache = MEM_callocN(sizeof(StrokeCache), "stroke cache");
1265         Brush *brush = paint_brush(&sd->paint);
1266         ViewContext *vc = paint_stroke_view_context(op->customdata);
1267         int i;
1268
1269         ss->cache = cache;
1270
1271         RNA_float_get_array(op->ptr, "scale", cache->scale);
1272         cache->flag = RNA_int_get(op->ptr, "flag");
1273         RNA_float_get_array(op->ptr, "clip_tolerance", cache->clip_tolerance);
1274         RNA_float_get_array(op->ptr, "initial_mouse", cache->initial_mouse);
1275         cache->depth = RNA_float_get(op->ptr, "depth");
1276
1277         cache->mouse[0] = cache->initial_mouse[0];
1278         cache->mouse[1] = cache->initial_mouse[1];
1279
1280         /* Truly temporary data that isn't stored in properties */
1281
1282         cache->mats = MEM_callocN(sizeof(bglMats), "sculpt bglMats");
1283         view3d_get_transformation(vc, vc->obact, cache->mats);
1284
1285         sculpt_update_mesh_elements(C);
1286
1287         /* Initialize layer brush displacements */
1288         if(brush->sculpt_tool == SCULPT_TOOL_LAYER &&
1289            (!ss->layer_disps || !(brush->flag & BRUSH_PERSISTENT))) {
1290                 if(ss->layer_disps)
1291                         MEM_freeN(ss->layer_disps);
1292                 ss->layer_disps = MEM_callocN(sizeof(float) * ss->totvert, "layer brush displacements");
1293         }
1294
1295         /* Make copies of the mesh vertex locations and normals for some tools */
1296         if(brush->sculpt_tool == SCULPT_TOOL_LAYER || (brush->flag & BRUSH_ANCHORED)) {
1297                 if(brush->sculpt_tool != SCULPT_TOOL_LAYER ||
1298                    !ss->mesh_co_orig || !(brush->flag & BRUSH_PERSISTENT)) {
1299                         if(!ss->mesh_co_orig)
1300                                 ss->mesh_co_orig= MEM_mallocN(sizeof(float) * 3 * ss->totvert,
1301                                                                        "sculpt mesh vertices copy");
1302                         for(i = 0; i < ss->totvert; ++i)
1303                                 VecCopyf(ss->mesh_co_orig[i], ss->mvert[i].co);
1304                 }
1305
1306                 if(brush->flag & BRUSH_ANCHORED) {
1307                         cache->orig_norms= MEM_mallocN(sizeof(short) * 3 * ss->totvert, "Sculpt orig norm");
1308                         for(i = 0; i < ss->totvert; ++i) {
1309                                 cache->orig_norms[i][0] = ss->mvert[i].no[0];
1310                                 cache->orig_norms[i][1] = ss->mvert[i].no[1];
1311                                 cache->orig_norms[i][2] = ss->mvert[i].no[2];
1312                         }
1313
1314                         if(ss->face_normals) {
1315                                 float *fn = ss->face_normals;
1316                                 cache->face_norms= MEM_mallocN(sizeof(float) * 3 * ss->totface, "Sculpt face norms");
1317                                 for(i = 0; i < ss->totface; ++i, fn += 3)
1318                                         VecCopyf(cache->face_norms[i], fn);
1319                         }
1320                 }
1321         }
1322
1323         view3d_unproject(cache->mats, cache->true_location, cache->initial_mouse[0], cache->initial_mouse[1], cache->depth);
1324         cache->initial_radius = unproject_brush_radius(ss, brush->size);
1325         cache->rotation = 0;
1326         cache->first_time = 1;
1327 }
1328
1329 /* Initialize the stroke cache variants from operator properties */
1330 static void sculpt_update_cache_variants(Sculpt *sd, SculptSession *ss, PointerRNA *ptr)
1331 {
1332         StrokeCache *cache = ss->cache;
1333         Brush *brush = paint_brush(&sd->paint);
1334         float grab_location[3];
1335         
1336         int dx, dy;
1337
1338         if(!(brush->flag & BRUSH_ANCHORED))
1339                 RNA_float_get_array(ptr, "location", cache->true_location);
1340         cache->flip = RNA_boolean_get(ptr, "flip");
1341         RNA_float_get_array(ptr, "mouse", cache->mouse);
1342         cache->pressure = RNA_float_get(ptr, "pressure");
1343         
1344         /* Truly temporary data that isn't stored in properties */
1345
1346         cache->previous_pixel_radius = cache->pixel_radius;
1347         cache->pixel_radius = brush->size;
1348
1349         if(brush->flag & BRUSH_SIZE_PRESSURE) {
1350                 cache->pixel_radius *= cache->pressure;
1351                 cache->radius = cache->initial_radius * cache->pressure;
1352         }
1353         else
1354                 cache->radius = cache->initial_radius;
1355
1356         if(brush->flag & BRUSH_ANCHORED) {
1357                 dx = cache->mouse[0] - cache->initial_mouse[0];
1358                 dy = cache->mouse[1] - cache->initial_mouse[1];
1359                 cache->pixel_radius = sqrt(dx*dx + dy*dy);
1360                 cache->radius = unproject_brush_radius(ss, cache->pixel_radius);
1361                 cache->rotation = atan2(dy, dx);
1362         }
1363         else if(brush->flag & BRUSH_RAKE) {
1364                 int update;
1365
1366                 dx = cache->last_rake[0] - cache->mouse[0];
1367                 dy = cache->last_rake[1] - cache->mouse[1];
1368
1369                 update = dx*dx + dy*dy > 100;
1370
1371                 /* To prevent jitter, only update the angle if the mouse has moved over 10 pixels */
1372                 if(update && !cache->first_time)
1373                         cache->rotation = M_PI_2 + atan2(dy, dx);
1374
1375                 if(update || cache->first_time) {
1376                         cache->last_rake[0] = cache->mouse[0];
1377                         cache->last_rake[1] = cache->mouse[1];
1378                 }
1379         }
1380
1381         /* Find the grab delta */
1382         if(brush->sculpt_tool == SCULPT_TOOL_GRAB) {
1383                 view3d_unproject(cache->mats, grab_location, cache->mouse[0], cache->mouse[1], cache->depth);
1384                 if(!cache->first_time)
1385                         VecSubf(cache->grab_delta, grab_location, cache->old_grab_location);
1386                 VecCopyf(cache->old_grab_location, grab_location);
1387         }
1388 }
1389
1390 /* Initialize stroke operator properties */
1391 static void sculpt_brush_stroke_init_properties(bContext *C, wmOperator *op, wmEvent *event, SculptSession *ss)
1392 {
1393         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
1394         Object *ob= CTX_data_active_object(C);
1395         ModifierData *md;
1396         float scale[3], clip_tolerance[3] = {0,0,0};
1397         float mouse[2];
1398         int flag = 0;
1399
1400         /* Set scaling adjustment */
1401         scale[0] = 1.0f / ob->size[0];
1402         scale[1] = 1.0f / ob->size[1];
1403         scale[2] = 1.0f / ob->size[2];
1404         RNA_float_set_array(op->ptr, "scale", scale);
1405
1406         /* Initialize mirror modifier clipping */
1407         for(md= ob->modifiers.first; md; md= md->next) {
1408                 if(md->type==eModifierType_Mirror && (md->mode & eModifierMode_Realtime)) {
1409                         const MirrorModifierData *mmd = (MirrorModifierData*) md;
1410                         
1411                         /* Mark each axis that needs clipping along with its tolerance */
1412                         if(mmd->flag & MOD_MIR_CLIPPING) {
1413                                 flag |= CLIP_X << mmd->axis;
1414                                 if(mmd->tolerance > clip_tolerance[mmd->axis])
1415                                         clip_tolerance[mmd->axis] = mmd->tolerance;
1416                         }
1417                 }
1418         }
1419         RNA_int_set(op->ptr, "flag", flag);
1420         RNA_float_set_array(op->ptr, "clip_tolerance", clip_tolerance);
1421
1422         /* Initial mouse location */
1423         mouse[0] = event->x;
1424         mouse[1] = event->y;
1425         RNA_float_set_array(op->ptr, "initial_mouse", mouse);
1426
1427         /* Initial screen depth under the mouse */
1428         RNA_float_set(op->ptr, "depth", read_cached_depth(paint_stroke_view_context(op->customdata), event->x, event->y));
1429
1430         sculpt_update_cache_invariants(sd, ss, C, op);
1431 }
1432
1433 static void sculpt_brush_stroke_init(bContext *C)
1434 {
1435         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
1436         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1437
1438         view3d_operator_needs_opengl(C);
1439
1440         /* TODO: Shouldn't really have to do this at the start of every
1441            stroke, but sculpt would need some sort of notification when
1442            changes are made to the texture. */
1443         sculpt_update_tex(sd, ss);
1444
1445         sculpt_update_mesh_elements(C);
1446 }
1447
1448 static void sculpt_restore_mesh(Sculpt *sd, SculptSession *ss)
1449 {
1450         StrokeCache *cache = ss->cache;
1451         Brush *brush = paint_brush(&sd->paint);
1452         float *buffer;
1453         int i;
1454
1455         /* Restore the mesh before continuing with anchored stroke */
1456         if((brush->flag & BRUSH_ANCHORED) && ss->mesh_co_orig) {
1457                 buffer = buffer = ss->drawobject!=0?(float *)GPU_buffer_lock( ss->drawobject->normals ):0;
1458                 for(i = 0; i < ss->totvert; ++i) {
1459                         VecCopyf(ss->mvert[i].co, ss->mesh_co_orig[i]);
1460                         ss->mvert[i].no[0] = cache->orig_norms[i][0];
1461                         ss->mvert[i].no[1] = cache->orig_norms[i][1];
1462                         ss->mvert[i].no[2] = cache->orig_norms[i][2];
1463                         if( buffer != 0 ) {
1464                                 IndexLink *cur = &ss->drawobject->indices[i];
1465                                 while( cur != 0 && cur->element != -1 ) {
1466                                         VECCOPY(&buffer[cur->element*3],cache->orig_norms[i]);
1467                                         cur = cur->next;
1468                                 }
1469                         }
1470                 }
1471                 if( buffer != 0 )
1472                         GPU_buffer_unlock( ss->drawobject->normals );
1473
1474                 if(ss->face_normals) {
1475                         float *fn = ss->face_normals;
1476                         for(i = 0; i < ss->totface; ++i, fn += 3)
1477                                 VecCopyf(fn, cache->face_norms[i]);
1478                 }
1479
1480                 if(brush->sculpt_tool == SCULPT_TOOL_LAYER)
1481                         memset(ss->layer_disps, 0, sizeof(float) * ss->totvert);
1482         }
1483 }
1484
1485 static void sculpt_post_stroke_free(SculptSession *ss)
1486 {
1487         BLI_freelistN(&ss->damaged_rects);
1488         BLI_freelistN(&ss->damaged_verts);
1489 }
1490
1491 static void sculpt_flush_update(bContext *C)
1492 {
1493         Object *ob = CTX_data_active_object(C);
1494         SculptSession *ss = ob->sculpt;
1495         ARegion *ar = CTX_wm_region(C);
1496         MultiresModifierData *mmd = ss->multires;
1497
1498         calc_damaged_verts(ss);
1499
1500         if(mmd) {
1501                 if(mmd->undo_verts && mmd->undo_verts != ss->mvert)
1502                         MEM_freeN(mmd->undo_verts);
1503                 
1504                 mmd->undo_verts = ss->mvert;
1505                 mmd->undo_verts_tot = ss->totvert;
1506                 multires_mark_as_modified(ob);
1507         }
1508
1509         ED_region_tag_redraw(ar);
1510 }
1511
1512 static int sculpt_stroke_test_start(bContext *C, struct wmOperator *op, wmEvent *event)
1513 {
1514         ViewContext vc;
1515         float cur_depth;
1516
1517         view3d_set_viewcontext(C, &vc);
1518         cur_depth = read_cached_depth(&vc, event->x, event->y);
1519         
1520         /* Don't start the stroke until a valid depth is found */
1521         if(cur_depth < 1.0 - FLT_EPSILON) {
1522                 SculptSession *ss = CTX_data_active_object(C)->sculpt;
1523
1524                 sculpt_brush_stroke_init_properties(C, op, event, ss);
1525                 sculptmode_update_all_projverts(ss);
1526
1527                 return 1;
1528         }
1529         else
1530                 return 0;
1531 }
1532
1533 static void sculpt_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1534 {
1535         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
1536         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1537
1538         sculpt_update_cache_variants(sd, ss, itemptr);
1539         sculpt_restore_mesh(sd, ss);
1540         do_symmetrical_brush_actions(sd, ss);
1541
1542         /* Cleanup */
1543         sculpt_flush_update(C);
1544         sculpt_post_stroke_free(ss);
1545 }
1546
1547 static void sculpt_stroke_done(bContext *C, struct PaintStroke *stroke)
1548 {
1549         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1550
1551         /* Finished */
1552         if(ss->cache) {
1553                 Sculpt *sd = CTX_data_tool_settings(C)->sculpt;         
1554
1555                 request_depth_update(paint_stroke_view_context(stroke)->rv3d);
1556                 sculpt_cache_free(ss->cache);
1557                 ss->cache = NULL;
1558                 sculpt_undo_push(C, sd);
1559         }
1560 }
1561
1562 static int sculpt_brush_stroke_invoke(bContext *C, wmOperator *op, wmEvent *event)
1563 {
1564         sculpt_brush_stroke_init(C);
1565
1566         op->customdata = paint_stroke_new(C, sculpt_stroke_test_start,
1567                                           sculpt_stroke_update_step,
1568                                           sculpt_stroke_done);
1569
1570         /* add modal handler */
1571         WM_event_add_modal_handler(C, op);
1572
1573         op->type->modal(C, op, event);
1574         
1575         return OPERATOR_RUNNING_MODAL;
1576 }
1577
1578 static int sculpt_brush_stroke_exec(bContext *C, wmOperator *op)
1579 {
1580         Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
1581         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1582
1583         op->customdata = paint_stroke_new(C, sculpt_stroke_test_start, sculpt_stroke_update_step, sculpt_stroke_done);
1584
1585         sculpt_brush_stroke_init(C);
1586
1587         sculpt_update_cache_invariants(sd, ss, C, op);
1588         sculptmode_update_all_projverts(ss);
1589
1590         paint_stroke_exec(C, op);
1591
1592         sculpt_flush_update(C);
1593         sculpt_cache_free(ss->cache);
1594
1595         sculpt_undo_push(C, sd);
1596
1597         return OPERATOR_FINISHED;
1598 }
1599
1600 static void SCULPT_OT_brush_stroke(wmOperatorType *ot)
1601 {
1602         ot->flag |= OPTYPE_REGISTER;
1603
1604         /* identifiers */
1605         ot->name= "Sculpt Mode";
1606         ot->idname= "SCULPT_OT_brush_stroke";
1607         
1608         /* api callbacks */
1609         ot->invoke= sculpt_brush_stroke_invoke;
1610         ot->modal= paint_stroke_modal;
1611         ot->exec= sculpt_brush_stroke_exec;
1612         ot->poll= sculpt_poll;
1613         
1614         /* flags (sculpt does own undo? (ton) */
1615         ot->flag= OPTYPE_REGISTER|OPTYPE_BLOCKING;
1616
1617         /* properties */
1618         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1619
1620         /* If the object has a scaling factor, brushes also need to be scaled
1621            to work as expected. */
1622         RNA_def_float_vector(ot->srna, "scale", 3, NULL, 0.0f, FLT_MAX, "Scale", "", 0.0f, 1000.0f);
1623
1624         RNA_def_int(ot->srna, "flag", 0, 0, INT_MAX, "flag", "", 0, INT_MAX);
1625
1626         /* For mirror modifiers */
1627         RNA_def_float_vector(ot->srna, "clip_tolerance", 3, NULL, 0.0f, FLT_MAX, "clip_tolerance", "", 0.0f, 1000.0f);
1628
1629         /* The initial 2D location of the mouse */
1630         RNA_def_float_vector(ot->srna, "initial_mouse", 2, NULL, INT_MIN, INT_MAX, "initial_mouse", "", INT_MIN, INT_MAX);
1631
1632         /* The initial screen depth of the mouse */
1633         RNA_def_float(ot->srna, "depth", 0.0f, 0.0f, FLT_MAX, "depth", "", 0.0f, FLT_MAX);
1634 }
1635
1636 /**** Reset the copy of the mesh that is being sculpted on (currently just for the layer brush) ****/
1637
1638 static int sculpt_set_persistent_base(bContext *C, wmOperator *op)
1639 {
1640         SculptSession *ss = CTX_data_active_object(C)->sculpt;
1641
1642         if(ss) {
1643                 if(ss->layer_disps)
1644                         MEM_freeN(ss->layer_disps);
1645                 ss->layer_disps = NULL;
1646
1647                 if(ss->mesh_co_orig)
1648                         MEM_freeN(ss->mesh_co_orig);
1649                 ss->mesh_co_orig = NULL;
1650         }
1651
1652         return OPERATOR_FINISHED;
1653 }
1654
1655 static void SCULPT_OT_set_persistent_base(wmOperatorType *ot)
1656 {
1657         /* identifiers */
1658         ot->name= "Set Persistent Base";
1659         ot->idname= "SCULPT_OT_set_persistent_base";
1660         
1661         /* api callbacks */
1662         ot->exec= sculpt_set_persistent_base;
1663         ot->poll= sculpt_mode_poll;
1664         
1665         ot->flag= OPTYPE_REGISTER;
1666 }
1667
1668 /**** Toggle operator for turning sculpt mode on or off ****/
1669
1670 static int sculpt_toggle_mode(bContext *C, wmOperator *op)
1671 {
1672         ToolSettings *ts = CTX_data_tool_settings(C);
1673         Object *ob = CTX_data_active_object(C);
1674
1675         if(ob->mode & OB_MODE_SCULPT) {
1676                 multires_force_update(ob);
1677
1678                 /* Leave sculptmode */
1679                 ob->mode &= ~OB_MODE_SCULPT;
1680
1681                 free_sculptsession(&ob->sculpt);
1682         }
1683         else {
1684                 /* Enter sculptmode */
1685
1686                 ob->mode |= OB_MODE_SCULPT;
1687                 
1688                 /* Create persistent sculpt mode data */
1689                 if(!ts->sculpt)
1690                         ts->sculpt = MEM_callocN(sizeof(Sculpt), "sculpt mode data");
1691
1692                 /* Create sculpt mode session data */
1693                 if(ob->sculpt)
1694                         free_sculptsession(&ob->sculpt);
1695                 ob->sculpt = MEM_callocN(sizeof(SculptSession), "sculpt session");
1696
1697                 paint_init(&ts->sculpt->paint, PAINT_CURSOR_SCULPT);
1698                 
1699                 paint_cursor_start(C, sculpt_poll);
1700         }
1701
1702         WM_event_add_notifier(C, NC_SCENE|ND_MODE, CTX_data_scene(C));
1703
1704         return OPERATOR_FINISHED;
1705 }
1706
1707 static void SCULPT_OT_sculptmode_toggle(wmOperatorType *ot)
1708 {
1709         /* identifiers */
1710         ot->name= "Sculpt Mode";
1711         ot->idname= "SCULPT_OT_sculptmode_toggle";
1712         
1713         /* api callbacks */
1714         ot->exec= sculpt_toggle_mode;
1715         ot->poll= ED_operator_object_active;
1716         
1717         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1718 }
1719
1720 void ED_operatortypes_sculpt()
1721 {
1722         WM_operatortype_append(SCULPT_OT_radial_control);
1723         WM_operatortype_append(SCULPT_OT_brush_stroke);
1724         WM_operatortype_append(SCULPT_OT_sculptmode_toggle);
1725         WM_operatortype_append(SCULPT_OT_set_persistent_base);
1726 }