Fixed warnings in sculpt, also brought back brush curve.
[blender.git] / source / blender / editors / sculpt / sculpt.c
1 /*
2  * $Id: sculptmode.c 18309 2009-01-04 07:47:11Z nicholasbishop $
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software  Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2006 by Nicholas Bishop
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  *
29  * Implements the Sculpt Mode tools
30  *
31  * BDR_sculptmode.h
32  *
33  */
34
35 #include "MEM_guardedalloc.h"
36
37 #include "BLI_arithb.h"
38 #include "BLI_blenlib.h"
39 #include "BLI_dynstr.h"
40
41 #include "DNA_armature_types.h"
42 #include "DNA_brush_types.h"
43 #include "DNA_image_types.h"
44 #include "DNA_key_types.h"
45 #include "DNA_mesh_types.h"
46 #include "DNA_meshdata_types.h"
47 #include "DNA_modifier_types.h"
48 #include "DNA_object_types.h"
49 #include "DNA_screen_types.h"
50 #include "DNA_scene_types.h"
51 #include "DNA_texture_types.h"
52 #include "DNA_view3d_types.h"
53 #include "DNA_userdef_types.h"
54 #include "DNA_color_types.h"
55
56 #include "BKE_brush.h"
57 #include "BKE_context.h"
58 #include "BKE_customdata.h"
59 #include "BKE_DerivedMesh.h"
60 #include "BKE_depsgraph.h"
61 #include "BKE_global.h"
62 #include "BKE_image.h"
63 #include "BKE_key.h"
64 #include "BKE_library.h"
65 #include "BKE_main.h"
66 #include "BKE_mesh.h"
67 #include "BKE_modifier.h"
68 #include "BKE_multires.h"
69 #include "BKE_sculpt.h"
70 #include "BKE_texture.h"
71 #include "BKE_utildefines.h"
72 #include "BKE_colortools.h"
73
74 #include "BIF_gl.h"
75 #include "BIF_glutil.h"
76
77 #include "WM_api.h"
78 #include "WM_types.h"
79 #include "ED_screen.h"
80 #include "ED_sculpt.h"
81 #include "ED_space_api.h"
82 #include "sculpt_intern.h"
83 #include "../space_view3d/view3d_intern.h" /* XXX: oh no, the next generation of bad level call! should move ViewDepths perhaps (also used for view matrices) */
84
85 #include "RNA_access.h"
86 #include "RNA_define.h"
87
88 #include "IMB_imbuf_types.h"
89
90 #include "RE_render_ext.h"
91 #include "RE_shader_ext.h" /*for multitex_ext*/
92
93 #include "GPU_draw.h"
94
95 #include <math.h>
96 #include <stdlib.h>
97 #include <string.h>
98
99 /* Number of vertices to average in order to determine the flatten distance */
100 #define FLATTEN_SAMPLE_SIZE 10
101
102 /* Texture cache size */
103 #define TC_SIZE 256
104
105 /* ===== STRUCTS =====
106  *
107  */
108
109 /* ActiveData stores an Index into the mvert array of Mesh, plus Fade, which
110    stores how far the vertex is from the brush center, scaled to the range [0,1]. */
111 typedef struct ActiveData {
112         struct ActiveData *next, *prev;
113         unsigned int Index;
114         float Fade;
115         float dist;
116 } ActiveData;
117
118 typedef struct BrushActionSymm {
119         float center_3d[3];
120         char index;
121
122         float up[3], right[3], out[3];
123
124         /* Grab brush */
125         float grab_delta[3];
126 } BrushActionSymm;
127
128 typedef enum StrokeFlags {
129         CLIP_X = 1,
130         CLIP_Y = 2,
131         CLIP_Z = 4
132 } StrokeFlags;
133
134 /* Cache stroke properties that don't change after
135    the initialization at the start of a stroke. Used because
136    RNA property lookup isn't particularly fast.
137
138    For descriptions of these settings, check the operator properties.
139 */
140 typedef struct StrokeCache {
141         float radius;
142         float scale[3];
143         float flip;
144         int flag;
145         float clip_tolerance[3];
146         int mouse[2];
147 } StrokeCache;
148
149 typedef struct BrushAction {
150         BrushActionSymm symm;
151
152         char firsttime;
153
154         /* Some brushes need access to original mesh vertices */
155         vec3f *mesh_store;
156         short (*orig_norms)[3];
157
158         float prev_radius;
159         float radius;
160
161         float *layer_disps;
162
163         float anchored_rot;
164
165         /* Grab brush */
166         ListBase grab_active_verts[8];
167         float depth;
168 } BrushAction;
169
170 typedef struct RectNode {
171         struct RectNode *next, *prev;
172         rcti r;
173 } RectNode;
174
175 /* Used to store to 2D screen coordinates of each vertex in the mesh. */
176 typedef struct ProjVert {
177         short co[2];
178         
179         /* Used to mark whether a vertex is inside a rough bounding box
180            containing the brush. */
181         char inside;
182 } ProjVert;
183
184 static Object *active_ob= NULL;
185
186 static void init_brushaction(SculptData *sd, BrushAction *a, short *, short *);
187
188 /* vertex_users is an array of Lists that store all the faces that use a
189    particular vertex. vertex_users is in the same order as mesh.mvert */
190 static void calc_vertex_users(SculptSession *ss)
191 {
192         int i,j;
193         IndexNode *node= NULL;
194
195         sculpt_vertexusers_free(ss);
196         
197         /* For efficiency, use vertex_users_mem as a memory pool (may be larger
198            than necessary if mesh has triangles, but only one alloc is needed.) */
199         ss->vertex_users= MEM_callocN(sizeof(ListBase) * ss->totvert, "vertex_users");
200         ss->vertex_users_size= ss->totvert;
201         ss->vertex_users_mem= MEM_callocN(sizeof(IndexNode)*ss->totface*4, "vertex_users_mem");
202         node= ss->vertex_users_mem;
203
204         /* Find the users */
205         for(i=0; i<ss->totface; ++i){
206                 for(j=0; j<(ss->mface[i].v4?4:3); ++j, ++node) {
207                         node->index=i;
208                         BLI_addtail(&ss->vertex_users[((unsigned int*)(&ss->mface[i]))[j]], node);
209                 }
210         }
211 }
212
213 /* ===== INTERFACE =====
214  */
215
216 /* XXX: this can probably removed entirely */
217 #if 0
218 void sculptmode_rem_tex(void *junk0,void *junk1)
219 {
220         MTex *mtex= G.scene->sculptdata.mtex[G.scene->sculptdata.texact];
221         if(mtex) {
222                 SculptSession *ss= sculpt_session();
223                 if(mtex->tex) mtex->tex->id.us--;
224                 MEM_freeN(mtex);
225                 G.scene->sculptdata.mtex[G.scene->sculptdata.texact]= NULL;
226                 /* Clear brush preview */
227                 if(ss->texcache) {
228                         MEM_freeN(ss->texcache);
229                         ss->texcache= NULL;
230                 }
231                 // XXX BIF_undo_push("Unlink brush texture");
232                 allqueue(REDRAWBUTSEDIT, 0);
233                 allqueue(REDRAWOOPS, 0);
234         }
235 }
236 #endif
237
238 /* ===== OPENGL =====
239  *
240  * Simple functions to get data from the GL
241  */
242
243 /* Uses window coordinates (x,y) to find the depth in the GL depth buffer. If
244    available, G.vd->depths is used so that the brush doesn't sculpt on top of
245    itself (G.vd->depths is only updated at the end of a brush stroke.) */
246 static float get_depth(bContext *C, short x, short y)
247 {
248         ScrArea *sa= CTX_wm_area(C);
249
250         if(sa->spacetype==SPACE_VIEW3D) { // should check this in context instead?
251                 ViewDepths *vd = ((View3D*)sa->spacedata.first)->depths;
252                 
253                 y -= CTX_wm_region(C)->winrct.ymin;
254
255                 if(vd && vd->depths && x > 0 && y > 0 && x < vd->w && y < vd->h)
256                         return vd->depths[y * vd->w + x];
257         }
258
259         fprintf(stderr, "Error: Bad depth store!\n");
260         return 1;
261 }
262
263 /* Uses window coordinates (x,y) and depth component z to find a point in
264    modelspace */
265 static void unproject(SculptSession *ss, float out[3], const short x, const short y, const float z)
266 {
267         double ux, uy, uz;
268
269         gluUnProject(x,y,z, ss->mats->modelview, ss->mats->projection,
270                      (GLint *)ss->mats->viewport, &ux, &uy, &uz );
271         out[0] = ux;
272         out[1] = uy;
273         out[2] = uz;
274 }
275
276 /* Convert a point in model coordinates to 2D screen coordinates. */
277 static void projectf(SculptSession *ss, const float v[3], float p[2])
278 {
279         double ux, uy, uz;
280
281         gluProject(v[0],v[1],v[2], ss->mats->modelview, ss->mats->projection,
282                    (GLint *)ss->mats->viewport, &ux, &uy, &uz);
283         p[0]= ux;
284         p[1]= uy;
285 }
286
287 static void project(SculptSession *ss, const float v[3], short p[2])
288 {
289         float f[2];
290         projectf(ss, v, f);
291
292         p[0]= f[0];
293         p[1]= f[1];
294 }
295
296 /* ===== Sculpting =====
297  *
298  */
299
300 /* Return modified brush size. Uses current tablet pressure (if available) to
301    shrink the brush. Skipped for grab brush because only the first mouse down
302    size is used, which is small if the user has just touched the pen to the
303    tablet */
304 static char brush_size(SculptData *sd)
305 {
306         float size= sd->brush->size;
307 #if 0
308         float pressure= 0; /* XXX: get_pressure(); */
309         short activedevice= 0; /* XXX: get_activedevice(); */
310         
311         if(b->sculpt_tool!=SCULPT_TOOL_GRAB) {
312                 const float size_factor= sd->tablet_size / 10.0f;
313                 
314                 /* XXX: tablet stuff
315                 if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
316                         size*= sd->tablet_size==0?1:
317                         (1-size_factor) + pressure*size_factor;*/
318         }
319 #endif
320         return size;
321 }
322
323 /* Return modified brush strength. Includes the direction of the brush, positive
324    values pull vertices, negative values push. Uses tablet pressure and a
325    special multiplier found experimentally to scale the strength factor. */
326 static float brush_strength(SculptData *sd, StrokeCache *cache)
327 {
328         float dir= sd->brush->flag & BRUSH_DIR_IN ? -1 : 1;
329         float pressure= 1;
330         /* short activedevice= 0;XXX: get_activedevice(); */
331         float flip= cache->flip ? -1:1;
332         float anchored = sd->brush->flag & BRUSH_ANCHORED ? 25 : 1;
333
334         /* XXX: tablet stuff */
335 #if 0
336         const float strength_factor= sd->tablet_strength / 10.0f;
337
338         if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
339                 pressure= sd->sculptdata.tablet_strength==0?1:
340                         (1-strength_factor) + 1/*XXX: get_pressure()*/ *strength_factor;
341         
342         /* Flip direction for eraser */
343         if(activedevice==DEV_ERASER)
344                 dir= -dir;
345 #endif
346
347         switch(sd->brush->sculpt_tool){
348         case SCULPT_TOOL_DRAW:
349         case SCULPT_TOOL_LAYER:
350                 return sd->brush->alpha / 50.0f * dir * pressure * flip * anchored; /*XXX: not sure why? multiplied by G.vd->grid */;
351         case SCULPT_TOOL_SMOOTH:
352                 return sd->brush->alpha / .5f * pressure * anchored;
353         case SCULPT_TOOL_PINCH:
354                 return sd->brush->alpha / 10.0f * dir * pressure * flip * anchored;
355         case SCULPT_TOOL_GRAB:
356                 return 1;
357         case SCULPT_TOOL_INFLATE:
358                 return sd->brush->alpha / 50.0f * dir * pressure * flip * anchored;
359         case SCULPT_TOOL_FLATTEN:
360                 return sd->brush->alpha / 5.0f * pressure * anchored;
361         default:
362                 return 0;
363         }
364 }
365
366 /* For clipping against a mirror modifier */
367 static void sculpt_clip(StrokeCache *cache, float *co, const float val[3])
368 {
369         char i;
370         for(i=0; i<3; ++i) {
371                 if((cache->flag & (CLIP_X << i)) && (fabs(co[i]) <= cache->clip_tolerance[i]))
372                         co[i]= 0.0f;
373                 else
374                         co[i]= val[i];
375         }               
376 }
377
378 static void sculpt_axislock(SculptData *sd, float *co)
379 {
380         if (sd->flags & (SCULPT_LOCK_X|SCULPT_LOCK_Y|SCULPT_LOCK_Z)) return;
381         /* XXX: if(G.vd->twmode == V3D_MANIP_LOCAL) { */
382         if(0) {
383                 float mat[3][3], imat[3][3];
384                 /* XXX: Mat3CpyMat4(mat, OBACT->obmat); */
385                 Mat3Inv(imat, mat);
386                 Mat3MulVecfl(mat, co);
387                 if (sd->flags & SCULPT_LOCK_X) co[0] = 0.0;
388                 if (sd->flags & SCULPT_LOCK_Y) co[1] = 0.0;
389                 if (sd->flags & SCULPT_LOCK_Z) co[2] = 0.0;             
390                 Mat3MulVecfl(imat, co);
391         } else {
392                 if (sd->flags & SCULPT_LOCK_X) co[0] = 0.0;
393                 if (sd->flags & SCULPT_LOCK_Y) co[1] = 0.0;
394                 if (sd->flags & SCULPT_LOCK_Z) co[2] = 0.0;             
395         }
396 }
397
398 static void add_norm_if(float view_vec[3], float out[3], float out_flip[3], const short no[3])
399 {
400         float fno[3] = {no[0], no[1], no[2]};
401
402         Normalize(fno);
403
404         if((Inpf(view_vec, fno)) > 0) {
405                 VecAddf(out, out, fno);
406         } else {
407                 VecAddf(out_flip, out_flip, fno); /* out_flip is used when out is {0,0,0} */
408         }
409 }
410
411 /* Currently only for the draw brush; finds average normal for all active
412    vertices */
413 static void calc_area_normal(SculptData *sd, float out[3], BrushAction *a, const float *outdir, const ListBase* active_verts)
414 {
415         ActiveData *node = active_verts->first;
416         const int view = 0; /* XXX: should probably be a flag, not number: sd->brush_type==SCULPT_TOOL_DRAW ? sculptmode_brush()->view : 0; */
417         float out_flip[3];
418         
419         out[0]=out[1]=out[2] = out_flip[0]=out_flip[1]=out_flip[2] = 0;
420
421         if(sd->brush->flag & BRUSH_ANCHORED) {
422                 for(; node; node = node->next)
423                         add_norm_if(a->symm.out, out, out_flip, a->orig_norms[node->Index]);
424         }
425         else {
426                 for(; node; node = node->next)
427                         add_norm_if(a->symm.out, out, out_flip, sd->session->mvert[node->Index].no);
428         }
429
430         if (out[0]==0.0 && out[1]==0.0 && out[2]==0.0) {
431                 VECCOPY(out, out_flip);
432         }
433         
434         Normalize(out);
435
436         if(outdir) {
437                 out[0] = outdir[0] * view + out[0] * (10-view);
438                 out[1] = outdir[1] * view + out[1] * (10-view);
439                 out[2] = outdir[2] * view + out[2] * (10-view);
440         }
441         
442         Normalize(out);
443 }
444
445 static void do_draw_brush(SculptData *sd, SculptSession *ss, BrushAction *a, const ListBase* active_verts)
446 {
447         float area_normal[3];
448         ActiveData *node= active_verts->first;
449
450         calc_area_normal(sd, area_normal, a, a->symm.out, active_verts);
451         
452         sculpt_axislock(sd, area_normal);
453         
454         while(node){
455                 float *co= ss->mvert[node->Index].co;
456                 
457                 const float val[3]= {co[0]+area_normal[0]*node->Fade*ss->cache->scale[0],
458                                      co[1]+area_normal[1]*node->Fade*ss->cache->scale[1],
459                                      co[2]+area_normal[2]*node->Fade*ss->cache->scale[2]};
460                                      
461                 sculpt_clip(ss->cache, co, val);
462                 
463                 node= node->next;
464         }
465 }
466
467 /* For the smooth brush, uses the neighboring vertices around vert to calculate
468    a smoothed location for vert. Skips corner vertices (used by only one
469    polygon.) */
470 static void neighbor_average(SculptSession *ss, float avg[3], const int vert)
471 {
472         int i, skip= -1, total=0;
473         IndexNode *node= ss->vertex_users[vert].first;
474         char ncount= BLI_countlist(&ss->vertex_users[vert]);
475         MFace *f;
476
477         avg[0] = avg[1] = avg[2] = 0;
478                 
479         /* Don't modify corner vertices */
480         if(ncount==1) {
481                 VecCopyf(avg, ss->mvert[vert].co);
482                 return;
483         }
484
485         while(node){
486                 f= &ss->mface[node->index];
487                 
488                 if(f->v4) {
489                         skip= (f->v1==vert?2:
490                                f->v2==vert?3:
491                                f->v3==vert?0:
492                                f->v4==vert?1:-1);
493                 }
494
495                 for(i=0; i<(f->v4?4:3); ++i) {
496                         if(i != skip && (ncount!=2 || BLI_countlist(&ss->vertex_users[(&f->v1)[i]]) <= 2)) {
497                                 VecAddf(avg, avg, ss->mvert[(&f->v1)[i]].co);
498                                 ++total;
499                         }
500                 }
501
502                 node= node->next;
503         }
504
505         if(total>0)
506                 VecMulf(avg, 1.0f / total);
507         else
508                 VecCopyf(avg, ss->mvert[vert].co);
509 }
510
511 static void do_smooth_brush(SculptSession *ss, const ListBase* active_verts)
512 {
513         ActiveData *node= active_verts->first;
514
515         while(node){
516                 float *co= ss->mvert[node->Index].co;
517                 float avg[3], val[3];
518
519                 neighbor_average(ss, avg, node->Index);
520                 val[0] = co[0]+(avg[0]-co[0])*node->Fade;
521                 val[1] = co[1]+(avg[1]-co[1])*node->Fade;
522                 val[2] = co[2]+(avg[2]-co[2])*node->Fade;
523
524                 sculpt_clip(ss->cache, co, val);
525                 node= node->next;
526         }
527 }
528
529 static void do_pinch_brush(SculptSession *ss, const BrushAction *a, const ListBase* active_verts)
530 {
531         ActiveData *node= active_verts->first;
532
533         while(node) {
534                 float *co= ss->mvert[node->Index].co;
535                 const float val[3]= {co[0]+(a->symm.center_3d[0]-co[0])*node->Fade,
536                                      co[1]+(a->symm.center_3d[1]-co[1])*node->Fade,
537                                      co[2]+(a->symm.center_3d[2]-co[2])*node->Fade};
538                 sculpt_clip(ss->cache, co, val);
539                 node= node->next;
540         }
541 }
542
543 static void do_grab_brush(SculptData *sd, SculptSession *ss, BrushAction *a)
544 {
545         ActiveData *node= a->grab_active_verts[a->symm.index].first;
546         float add[3];
547         float grab_delta[3];
548         
549         VecCopyf(grab_delta, a->symm.grab_delta);
550         sculpt_axislock(sd, grab_delta);
551         
552         while(node) {
553                 float *co= ss->mvert[node->Index].co;
554                 
555                 VecCopyf(add, grab_delta);
556                 VecMulf(add, node->Fade);
557                 VecAddf(add, add, co);
558                 sculpt_clip(ss->cache, co, add);
559
560                 node= node->next;
561         }
562         
563 }
564
565 static void do_layer_brush(SculptData *sd, SculptSession *ss, BrushAction *a, const ListBase *active_verts)
566 {
567         float area_normal[3];
568         ActiveData *node= active_verts->first;
569         const float bstr= brush_strength(sd, ss->cache);
570
571         calc_area_normal(sd, area_normal, a, NULL, active_verts);
572
573         while(node){
574                 float *disp= &a->layer_disps[node->Index];
575                 
576                 if((bstr > 0 && *disp < bstr) ||
577                   (bstr < 0 && *disp > bstr)) {
578                         float *co= ss->mvert[node->Index].co;
579                         
580                         *disp+= node->Fade;
581
582                         if(bstr < 0) {
583                                 if(*disp < bstr)
584                                         *disp = bstr;
585                         } else {
586                                 if(*disp > bstr)
587                                         *disp = bstr;
588                         }
589
590                         {
591                                 const float val[3]= {a->mesh_store[node->Index].x+area_normal[0] * *disp*ss->cache->scale[0],
592                                                      a->mesh_store[node->Index].y+area_normal[1] * *disp*ss->cache->scale[1],
593                                                      a->mesh_store[node->Index].z+area_normal[2] * *disp*ss->cache->scale[2]};
594                                 sculpt_clip(ss->cache, co, val);
595                         }
596                 }
597
598                 node= node->next;
599         }
600 }
601
602 static void do_inflate_brush(SculptSession *ss, const ListBase *active_verts)
603 {
604         ActiveData *node= active_verts->first;
605         float add[3];
606         
607         while(node) {
608                 float *co= ss->mvert[node->Index].co;
609                 short *no= ss->mvert[node->Index].no;
610
611                 add[0]= no[0]/ 32767.0f;
612                 add[1]= no[1]/ 32767.0f;
613                 add[2]= no[2]/ 32767.0f;
614                 VecMulf(add, node->Fade);
615                 add[0]*= ss->cache->scale[0];
616                 add[1]*= ss->cache->scale[1];
617                 add[2]*= ss->cache->scale[2];
618                 VecAddf(add, add, co);
619                 
620                 sculpt_clip(ss->cache, co, add);
621
622                 node= node->next;
623         }
624 }
625
626 static void calc_flatten_center(SculptSession *ss, ActiveData *node, float co[3])
627 {
628         ActiveData *outer[FLATTEN_SAMPLE_SIZE];
629         int i;
630         
631         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
632                 outer[i] = node;
633                 
634         for(; node; node = node->next) {
635                 for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i) {
636                         if(node->dist > outer[i]->dist) {
637                                 outer[i] = node;
638                                 break;
639                         }
640                 }
641         }
642         
643         co[0] = co[1] = co[2] = 0.0f;
644         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
645                 VecAddf(co, co, ss->mvert[outer[i]->Index].co);
646         VecMulf(co, 1.0f / FLATTEN_SAMPLE_SIZE);
647 }
648
649 static void do_flatten_brush(SculptData *sd, SculptSession *ss, BrushAction *a, const ListBase *active_verts)
650 {
651         ActiveData *node= active_verts->first;
652         /* area_normal and cntr define the plane towards which vertices are squashed */
653         float area_normal[3];
654         float cntr[3];
655
656         calc_area_normal(sd, area_normal, a, a->symm.out, active_verts);
657         calc_flatten_center(ss, node, cntr);
658
659         while(node){
660                 float *co= ss->mvert[node->Index].co;
661                 float p1[3], sub1[3], sub2[3], intr[3], val[3];
662                 
663                 /* Find the intersection between squash-plane and vertex (along the area normal) */
664                 VecSubf(p1, co, area_normal);
665                 VecSubf(sub1, cntr, p1);
666                 VecSubf(sub2, co, p1);
667                 VecSubf(intr, co, p1);
668                 VecMulf(intr, Inpf(area_normal, sub1) / Inpf(area_normal, sub2));
669                 VecAddf(intr, intr, p1);
670                 
671                 VecSubf(val, intr, co);
672                 VecMulf(val, node->Fade);
673                 VecAddf(val, val, co);
674                 
675                 sculpt_clip(ss->cache, co, val);
676                 
677                 node= node->next;
678         }
679 }
680
681 /* Uses the brush curve control to find a strength value between 0 and 1 */
682 static float curve_strength(CurveMapping *cumap, float p, const float len)
683 {
684         if(p > len) p= len;
685         return curvemapping_evaluateF(cumap, 0, p/len);
686 }
687
688 /* Uses symm to selectively flip any axis of a coordinate. */
689 static void flip_coord(float co[3], const char symm)
690 {
691         if(symm & SCULPT_SYMM_X)
692                 co[0]= -co[0];
693         if(symm & SCULPT_SYMM_Y)
694                 co[1]= -co[1];
695         if(symm & SCULPT_SYMM_Z)
696                 co[2]= -co[2];
697 }
698
699 /* Use the warpfac field in MTex to store a rotation value for sculpt textures. Value is in degrees */
700 static float sculpt_tex_angle(SculptData *sd)
701 {
702         if(sd->texact!=-1 && sd->mtex[sd->texact])
703                 return sd->mtex[sd->texact]->warpfac;
704         return 0;
705 }
706
707 static void set_tex_angle(SculptData *sd, const float f)
708 {
709         if(sd->texact != -1 && sd->mtex[sd->texact])
710                 sd->mtex[sd->texact]->warpfac = f;
711 }
712         
713 static float to_rad(const float deg)
714 {
715         return deg * (M_PI/180.0f);
716 }
717
718 static float to_deg(const float rad)
719 {
720         return rad * (180.0f/M_PI);
721 }
722
723 /* Get a pixel from the texcache at (px, py) */
724 static unsigned char get_texcache_pixel(const SculptSession *ss, int px, int py)
725 {
726         unsigned *p;
727         p = ss->texcache + py * ss->texcache_w + px;
728         return ((unsigned char*)(p))[0];
729 }
730
731 static float get_texcache_pixel_bilinear(const SculptSession *ss, float u, float v)
732 {
733         int x, y, x2, y2;
734         const int tc_max = TC_SIZE - 1;
735         float urat, vrat, uopp;
736
737         if(u < 0) u = 0;
738         else if(u >= TC_SIZE) u = tc_max;
739         if(v < 0) v = 0;
740         else if(v >= TC_SIZE) v = tc_max;
741
742         x = floor(u);
743         y = floor(v);
744         x2 = x + 1;
745         y2 = y + 1;
746
747         if(x2 > TC_SIZE) x2 = tc_max;
748         if(y2 > TC_SIZE) y2 = tc_max;
749         
750         urat = u - x;
751         vrat = v - y;
752         uopp = 1 - urat;
753                 
754         return ((get_texcache_pixel(ss, x, y) * uopp +
755                  get_texcache_pixel(ss, x2, y) * urat) * (1 - vrat) + 
756                 (get_texcache_pixel(ss, x, y2) * uopp +
757                  get_texcache_pixel(ss, x2, y2) * urat) * vrat) / 255.0;
758 }
759
760 /* Return a multiplier for brush strength on a particular vertex. */
761 static float tex_strength(SculptData *sd, BrushAction *a, float *point, const float len)
762 {
763         SculptSession *ss= sd->session;
764         float avg= 1;
765
766         if(sd->texact==-1 || !sd->mtex[sd->texact])
767                 avg= 1;
768         else if(sd->texrept==SCULPTREPT_3D) {
769                 /* Get strength by feeding the vertex location directly
770                    into a texture */
771                 float jnk;
772                 const float factor= 0.01;
773                 MTex mtex;
774                 memset(&mtex,0,sizeof(MTex));
775                 mtex.tex= sd->mtex[sd->texact]->tex;
776                 mtex.projx= 1;
777                 mtex.projy= 2;
778                 mtex.projz= 3;
779                 VecCopyf(mtex.size, sd->mtex[sd->texact]->size);
780                 VecMulf(mtex.size, factor);
781                 if(!sd->texsep)
782                         mtex.size[1]= mtex.size[2]= mtex.size[0];
783                 
784                 externtex(&mtex,point,&avg,&jnk,&jnk,&jnk,&jnk);
785         }
786         else if(ss->texcache) {
787                 const float bsize= a->radius * 2;
788                 const float rot= to_rad(sculpt_tex_angle(sd)) + a->anchored_rot;
789                 int px, py;
790                 float flip[3], point_2d[2];
791
792                 /* If the active area is being applied for symmetry, flip it
793                    across the symmetry axis in order to project it. This insures
794                    that the brush texture will be oriented correctly. */
795                 VecCopyf(flip, point);
796                 flip_coord(flip, a->symm.index);
797                 projectf(ss, flip, point_2d);
798
799                 /* For Tile and Drag modes, get the 2D screen coordinates of the
800                    and scale them up or down to the texture size. */
801                 if(sd->texrept==SCULPTREPT_TILE) {
802                         const int sx= (const int)sd->mtex[sd->texact]->size[0];
803                         const int sy= (const int)sd->texsep ? sd->mtex[sd->texact]->size[1] : sx;
804                         
805                         float fx= point_2d[0];
806                         float fy= point_2d[1];
807                         
808                         float angle= atan2(fy, fx) - rot;
809                         float flen= sqrtf(fx*fx + fy*fy);
810                         
811                         if(rot<0.001 && rot>-0.001) {
812                                 px= point_2d[0];
813                                 py= point_2d[1];
814                         } else {
815                                 px= flen * cos(angle) + 2000;
816                                 py= flen * sin(angle) + 2000;
817                         }
818                         if(sx != 1)
819                                 px %= sx-1;
820                         if(sy != 1)
821                                 py %= sy-1;
822                         avg= get_texcache_pixel_bilinear(ss, TC_SIZE*px/sx, TC_SIZE*py/sy);
823                 } else {
824                         float fx= (point_2d[0] - ss->cache->mouse[0]) / bsize;
825                         float fy= (point_2d[1] - ss->cache->mouse[1]) / bsize;
826
827                         float angle= atan2(fy, fx) - rot;
828                         float flen= sqrtf(fx*fx + fy*fy);
829                         
830                         fx = flen * cos(angle) + 0.5;
831                         fy = flen * sin(angle) + 0.5;
832
833                         avg= get_texcache_pixel_bilinear(ss, fx * TC_SIZE, fy * TC_SIZE);
834                 }
835         }
836
837         avg*= curve_strength(sd->brush->curve, len, ss->cache->radius); /* Falloff curve */
838
839         return avg;
840 }
841
842 /* Mark area around the brush as damaged. projverts are marked if they are
843    inside the area and the damaged rectangle in 2D screen coordinates is 
844    added to damaged_rects. */
845 static void sculpt_add_damaged_rect(SculptSession *ss, BrushAction *a)
846 {
847         short p[2];
848         RectNode *rn= MEM_mallocN(sizeof(RectNode),"RectNode");
849         const float radius = a->radius > a->prev_radius ? a->radius : a->prev_radius;
850         unsigned i;
851
852         /* Find center */
853         project(ss, a->symm.center_3d, p);
854         rn->r.xmin= p[0] - radius;
855         rn->r.ymin= p[1] - radius;
856         rn->r.xmax= p[0] + radius;
857         rn->r.ymax= p[1] + radius;
858
859         BLI_addtail(&ss->damaged_rects, rn);
860
861         /* Update insides */
862         for(i=0; i<ss->totvert; ++i) {
863                 if(!ss->projverts[i].inside) {
864                         if(ss->projverts[i].co[0] > rn->r.xmin && ss->projverts[i].co[1] > rn->r.ymin &&
865                            ss->projverts[i].co[0] < rn->r.xmax && ss->projverts[i].co[1] < rn->r.ymax) {
866                                 ss->projverts[i].inside= 1;
867                         }
868                 }
869                 // XXX: remember to fix this!
870                 // temporary pass
871                 ss->projverts[i].inside = 1;
872         }
873 }
874
875 /* Clears the depth buffer in each modified area. */
876 static void sculpt_clear_damaged_areas(SculptSession *ss)
877 {
878         RectNode *rn= NULL;
879
880         for(rn = ss->damaged_rects.first; rn; rn = rn->next) {
881                 rcti clp = rn->r;
882                 rcti *win = NULL; /*XXX: &curarea->winrct; */
883                 
884                 clp.xmin += win->xmin;
885                 clp.xmax += win->xmin;
886                 clp.ymin += win->ymin;
887                 clp.ymax += win->ymin;
888                 
889                 if(clp.xmin < win->xmax && clp.xmax > win->xmin &&
890                    clp.ymin < win->ymax && clp.ymax > win->ymin) {
891                         if(clp.xmin < win->xmin) clp.xmin = win->xmin;
892                         if(clp.ymin < win->ymin) clp.ymin = win->ymin;
893                         if(clp.xmax > win->xmax) clp.xmax = win->xmax;
894                         if(clp.ymax > win->ymax) clp.ymax = win->ymax;
895
896                         glScissor(clp.xmin + 1, clp.ymin + 1,
897                                   clp.xmax - clp.xmin - 2,
898                                   clp.ymax - clp.ymin - 2);
899                 }
900                 
901                 glClear(GL_DEPTH_BUFFER_BIT);
902         }
903 }
904
905 static void do_brush_action(SculptData *sd, StrokeCache *cache, BrushAction *a)
906 {
907         int i;
908         float av_dist;
909         ListBase active_verts={0,0};
910         ActiveData *adata= 0;
911         float *vert;
912         Mesh *me= NULL; /*XXX: get_mesh(OBACT); */
913         const float bstrength= brush_strength(sd, cache);
914         KeyBlock *keyblock= NULL; /*XXX: ob_get_keyblock(OBACT); */
915         SculptSession *ss = sd->session;
916         Brush *b = sd->brush;
917
918         sculpt_add_damaged_rect(ss, a);
919
920         /* Build a list of all vertices that are potentially within the brush's
921            area of influence. Only do this once for the grab brush. */
922         if((b->sculpt_tool != SCULPT_TOOL_GRAB) || a->firsttime) {
923                 for(i=0; i<ss->totvert; ++i) {
924                         /* Projverts.inside provides a rough bounding box */
925                         if(ss->multires || ss->projverts[i].inside) {
926                                 //vert= ss->vertexcosnos ? &ss->vertexcosnos[i*6] : a->verts[i].co;
927                                 vert= ss->mvert[i].co;
928                                 av_dist= VecLenf(a->symm.center_3d, vert);
929                                 if(av_dist < cache->radius) {
930                                         adata= (ActiveData*)MEM_mallocN(sizeof(ActiveData), "ActiveData");
931
932                                         adata->Index = i;
933                                         /* Fade is used to store the final strength at which the brush
934                                            should modify a particular vertex. */
935                                         adata->Fade= tex_strength(sd, a, vert, av_dist) * bstrength;
936                                         adata->dist = av_dist;
937
938                                         if(b->sculpt_tool == SCULPT_TOOL_GRAB && a->firsttime)
939                                                 BLI_addtail(&a->grab_active_verts[a->symm.index], adata);
940                                         else
941                                                 BLI_addtail(&active_verts, adata);
942                                 }
943                         }
944                 }
945         }
946
947         /* Only act if some verts are inside the brush area */
948         if(active_verts.first || (b->sculpt_tool == SCULPT_TOOL_GRAB && a->grab_active_verts[a->symm.index].first)) {
949                 /* Apply one type of brush action */
950                 switch(b->sculpt_tool){
951                 case SCULPT_TOOL_DRAW:
952                         do_draw_brush(sd, ss, a, &active_verts);
953                         break;
954                 case SCULPT_TOOL_SMOOTH:
955                         do_smooth_brush(ss, &active_verts);
956                         break;
957                 case SCULPT_TOOL_PINCH:
958                         do_pinch_brush(ss, a, &active_verts);
959                         break;
960                 case SCULPT_TOOL_INFLATE:
961                         do_inflate_brush(ss, &active_verts);
962                         break;
963                 case SCULPT_TOOL_GRAB:
964                         do_grab_brush(sd, ss, a);
965                         break;
966                 case SCULPT_TOOL_LAYER:
967                         do_layer_brush(sd, ss, a, &active_verts);
968                         break;
969                 case SCULPT_TOOL_FLATTEN:
970                         do_flatten_brush(sd, ss, a, &active_verts);
971                         break;
972                 }
973         
974                 /* Copy the modified vertices from mesh to the active key */
975                 if(keyblock && !ss->multires) {
976                         float *co= keyblock->data;
977                         if(co) {
978                                 if(b->sculpt_tool == SCULPT_TOOL_GRAB)
979                                         adata = a->grab_active_verts[a->symm.index].first;
980                                 else
981                                         adata = active_verts.first;
982
983                                 for(; adata; adata= adata->next)
984                                         if(adata->Index < keyblock->totelem)
985                                                 VecCopyf(&co[adata->Index*3], me->mvert[adata->Index].co);
986                         }
987                 }
988
989                 if(ss->vertexcosnos && !ss->multires)
990                         BLI_freelistN(&active_verts);
991                 else {
992                         if(b->sculpt_tool != SCULPT_TOOL_GRAB)
993                                 addlisttolist(&ss->damaged_verts, &active_verts);
994                 }
995         }
996 }
997
998 /* Flip all the editdata across the axis/axes specified by symm. Used to
999    calculate multiple modifications to the mesh when symmetry is enabled. */
1000 static void calc_brushdata_symm(BrushAction *a, const char symm)
1001 {
1002         flip_coord(a->symm.center_3d, symm);
1003         flip_coord(a->symm.up, symm);
1004         flip_coord(a->symm.right, symm);
1005         flip_coord(a->symm.out, symm);
1006         
1007         a->symm.index= symm;
1008
1009         flip_coord(a->symm.grab_delta, symm);
1010 }
1011
1012 static void do_symmetrical_brush_actions(SculptData *sd, StrokeCache *cache, BrushAction *a)
1013 {
1014         const char symm = sd->flags & 7;
1015         BrushActionSymm orig;
1016         int i;
1017
1018         //init_brushaction(sd, a, co, pr_co);
1019         orig = a->symm;
1020         do_brush_action(sd, cache, a);
1021
1022         for(i = 1; i <= symm; ++i) {
1023                 if(symm & i && (symm != 5 || i != 3) && (symm != 6 || (i != 3 && i != 5))) {
1024                         // Restore the original symmetry data
1025                         a->symm = orig;
1026
1027                         calc_brushdata_symm(a, i);
1028                         do_brush_action(sd, cache, a);
1029                 }
1030         }
1031
1032         a->symm = orig;
1033 }
1034
1035 static void add_face_normal(vec3f *norm, MVert *mvert, const MFace* face, float *fn)
1036 {
1037         vec3f c= {mvert[face->v1].co[0],mvert[face->v1].co[1],mvert[face->v1].co[2]};
1038         vec3f b= {mvert[face->v2].co[0],mvert[face->v2].co[1],mvert[face->v2].co[2]};
1039         vec3f a= {mvert[face->v3].co[0],mvert[face->v3].co[1],mvert[face->v3].co[2]};
1040         vec3f s1, s2;
1041         float final[3];
1042
1043         VecSubf(&s1.x,&a.x,&b.x);
1044         VecSubf(&s2.x,&c.x,&b.x);
1045
1046         final[0] = s1.y * s2.z - s1.z * s2.y;
1047         final[1] = s1.z * s2.x - s1.x * s2.z;
1048         final[2] = s1.x * s2.y - s1.y * s2.x;
1049
1050         if(fn)
1051                 VecCopyf(fn, final);
1052
1053         norm->x+= final[0];
1054         norm->y+= final[1];
1055         norm->z+= final[2];
1056 }
1057
1058 static void update_damaged_vert(SculptSession *ss, ListBase *lb)
1059 {
1060         ActiveData *vert;
1061        
1062         for(vert= lb->first; vert; vert= vert->next) {
1063                 vec3f norm= {0,0,0};            
1064                 IndexNode *face= ss->vertex_users[vert->Index].first;
1065
1066                 while(face){
1067                         float *fn = NULL;
1068                         if(ss->face_normals)
1069                                 fn = &ss->face_normals[face->index*3];
1070                         add_face_normal(&norm, ss->mvert, &ss->mface[face->index], fn);
1071                         face= face->next;
1072                 }
1073                 Normalize(&norm.x);
1074                 
1075                 ss->mvert[vert->Index].no[0]=norm.x*32767;
1076                 ss->mvert[vert->Index].no[1]=norm.y*32767;
1077                 ss->mvert[vert->Index].no[2]=norm.z*32767;
1078         }
1079 }
1080
1081 static void calc_damaged_verts(SculptSession *ss, BrushAction *a)
1082 {
1083         int i;
1084         
1085         for(i=0; i<8; ++i)
1086                 update_damaged_vert(ss, &a->grab_active_verts[i]);
1087
1088         update_damaged_vert(ss, &ss->damaged_verts);
1089         BLI_freelistN(&ss->damaged_verts);
1090         ss->damaged_verts.first = ss->damaged_verts.last = NULL;
1091 }
1092
1093 static void projverts_clear_inside(SculptSession *ss)
1094 {
1095         int i;
1096         for(i = 0; i < ss->totvert; ++i)
1097                 ss->projverts[i].inside = 0;
1098 }
1099
1100 static void sculptmode_update_tex(SculptData *sd)
1101 {
1102         SculptSession *ss= sd->session;
1103         MTex *mtex = sd->mtex[sd->texact];
1104         TexResult texres;
1105         float x, y, step=2.0/TC_SIZE, co[3];
1106         int hasrgb, ix, iy;
1107
1108         memset(&texres, 0, sizeof(TexResult));
1109         
1110         /* Skip Default brush shape and non-textures */
1111         if(sd->texact == -1 || !sd->mtex[sd->texact]) return;
1112
1113         if(ss->texcache) {
1114                 MEM_freeN(ss->texcache);
1115                 ss->texcache= NULL;
1116         }
1117         
1118         ss->texcache_w = ss->texcache_h = TC_SIZE;
1119         ss->texcache = MEM_callocN(sizeof(int) * ss->texcache_w * ss->texcache_h, "Sculpt Texture cache");
1120         
1121         if(mtex && mtex->tex) {
1122                 BKE_image_get_ibuf(sd->mtex[sd->texact]->tex->ima, NULL);
1123                 
1124                 /*do normalized cannonical view coords for texture*/
1125                 for (y=-1.0, iy=0; iy<TC_SIZE; iy++, y += step) {
1126                         for (x=-1.0, ix=0; ix<TC_SIZE; ix++, x += step) {
1127                                 co[0]= x;
1128                                 co[1]= y;
1129                                 co[2]= 0.0f;
1130                                 
1131                                 /* This is copied from displace modifier code */
1132                                 hasrgb = multitex_ext(mtex->tex, co, NULL, NULL, 1, &texres);
1133                         
1134                                 /* if the texture gave an RGB value, we assume it didn't give a valid
1135                                  * intensity, so calculate one (formula from do_material_tex).
1136                                  * if the texture didn't give an RGB value, copy the intensity across
1137                                  */
1138                                 if(hasrgb & TEX_RGB)
1139                                         texres.tin = (0.35 * texres.tr + 0.45 *
1140                                                       texres.tg + 0.2 * texres.tb);
1141
1142                                 texres.tin = texres.tin * 255.0;
1143                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4] = (char)texres.tin;
1144                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+1] = (char)texres.tin;
1145                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+2] = (char)texres.tin;
1146                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+3] = (char)texres.tin;
1147                         }
1148                 }
1149         }
1150 }
1151
1152 /* pr_mouse is only used for the grab brush, can be NULL otherwise */
1153 static void init_brushaction(SculptData *sd, BrushAction *a, short *mouse, short *pr_mouse)
1154 {
1155         SculptSession *ss = sd->session;
1156         Brush *b = sd->brush;
1157         const float mouse_depth = 0; // XXX: get_depth(mouse[0], mouse[1]);
1158         float brush_edge_loc[3], zero_loc[3], oldloc[3];
1159         int i;
1160         const int anchored = sd->brush->flag & BRUSH_ANCHORED;
1161         short orig_mouse[2], dx=0, dy=0;
1162         float size = brush_size(sd);
1163
1164         a->symm.index = 0;
1165
1166         if(a->firsttime) 
1167                 a->depth = mouse_depth;
1168         
1169         /* Convert the location and size of the brush to
1170            modelspace coords */
1171         if(a->firsttime || !anchored) {
1172                 unproject(ss, a->symm.center_3d, mouse[0], mouse[1], mouse_depth);
1173                 /*a->mouse[0] = mouse[0];
1174                   a->mouse[1] = mouse[1];*/
1175         }
1176  
1177         if(anchored) {
1178                 project(ss, a->symm.center_3d, orig_mouse);
1179                 dx = mouse[0] - orig_mouse[0];
1180                 dy = mouse[1] - orig_mouse[1];
1181         }
1182  
1183         if(anchored) {
1184                 unproject(ss, brush_edge_loc, mouse[0], mouse[1], a->depth);
1185                 a->anchored_rot = atan2(dy, dx);
1186         }
1187         else
1188                 unproject(ss, brush_edge_loc, mouse[0] + size, mouse[1], mouse_depth);
1189  
1190         //a->size_3d = VecLenf(a->symm.center_3d, brush_edge_loc);
1191
1192         a->prev_radius = a->radius;
1193
1194         if(anchored)
1195                 a->radius = sqrt(dx*dx + dy*dy);
1196         else
1197                 a->radius = size;
1198
1199         /* Set the pivot to allow the model to rotate around the center of the brush */
1200         /*XXX: if(get_depth(mouse[0],mouse[1]) < 1.0)
1201           VecCopyf(sd->pivot, a->symm.center_3d); */
1202
1203         /* Now project the Up, Right, and Out normals from view to model coords */
1204         unproject(ss, zero_loc, 0, 0, 0);
1205         unproject(ss, a->symm.up, 0, -1, 0);
1206         unproject(ss, a->symm.right, 1, 0, 0);
1207         unproject(ss, a->symm.out, 0, 0, -1);
1208         VecSubf(a->symm.up, a->symm.up, zero_loc);
1209         VecSubf(a->symm.right, a->symm.right, zero_loc);
1210         VecSubf(a->symm.out, a->symm.out, zero_loc);
1211         Normalize(a->symm.up);
1212         Normalize(a->symm.right);
1213         Normalize(a->symm.out);
1214         
1215
1216
1217         if(b->sculpt_tool == SCULPT_TOOL_GRAB) {
1218                 float gcenter[3];
1219
1220                 /* Find the delta */
1221                 unproject(ss, gcenter, mouse[0], mouse[1], a->depth);
1222                 unproject(ss, oldloc, pr_mouse[0], pr_mouse[1], a->depth);
1223                 VecSubf(a->symm.grab_delta, gcenter, oldloc);
1224         }
1225         else if(b->sculpt_tool == SCULPT_TOOL_LAYER) {
1226                 if(!a->layer_disps)
1227                         a->layer_disps= MEM_callocN(sizeof(float)*ss->totvert,"Layer disps");
1228         }
1229
1230         if(b->sculpt_tool == SCULPT_TOOL_LAYER || anchored) {
1231                 if(!a->mesh_store) {
1232                         a->mesh_store= MEM_mallocN(sizeof(vec3f) * ss->totvert, "Sculpt mesh store");
1233                         for(i = 0; i < ss->totvert; ++i)
1234                                 VecCopyf(&a->mesh_store[i].x, ss->mvert[i].co);
1235                 }
1236
1237                 if(anchored && a->layer_disps)
1238                         memset(a->layer_disps, 0, sizeof(float) * ss->totvert);
1239
1240                 if(anchored && !a->orig_norms) {
1241                         a->orig_norms= MEM_mallocN(sizeof(short) * 3 * ss->totvert, "Sculpt orig norm");
1242                         for(i = 0; i < ss->totvert; ++i) {
1243                                 a->orig_norms[i][0] = ss->mvert[i].no[0];
1244                                 a->orig_norms[i][1] = ss->mvert[i].no[1];
1245                                 a->orig_norms[i][2] = ss->mvert[i].no[2];
1246                         }
1247                 }
1248         }
1249 }
1250
1251 /* XXX: Used anywhere?
1252 void sculptmode_set_strength(const int delta)
1253 {
1254         int val = sculptmode_brush()->strength + delta;
1255         if(val < 1) val = 1;
1256         if(val > 100) val = 100;
1257         sculptmode_brush()->strength= val;
1258 }*/
1259
1260 /* XXX: haven't brought in the radial control files, not sure where to put them. Note that all the paint modes should have access to radial control! */
1261 #if 0
1262 static void sculpt_radialcontrol_callback(const int mode, const int val)
1263 {
1264         SculptSession *ss = sculpt_session();
1265         BrushData *br = sculptmode_brush();
1266
1267         if(mode == RADIALCONTROL_SIZE)
1268                 br->size = val;
1269         else if(mode == RADIALCONTROL_STRENGTH)
1270                 br->strength = val;
1271         else if(mode == RADIALCONTROL_ROTATION)
1272                 set_tex_angle(val);
1273
1274         ss->radialcontrol = NULL;
1275 }
1276
1277 /* Returns GL handle to brush texture */
1278 static GLuint sculpt_radialcontrol_calctex()
1279 {
1280         SculptData *sd= sculpt_data();
1281         SculptSession *ss= sculpt_session();
1282         int i, j;
1283         const int tsz = TC_SIZE;
1284         float *texdata= MEM_mallocN(sizeof(float)*tsz*tsz, "Brush preview");
1285         GLuint tex;
1286
1287         if(sd->texrept!=SCULPTREPT_3D)
1288                 sculptmode_update_tex();
1289         for(i=0; i<tsz; ++i)
1290                 for(j=0; j<tsz; ++j) {
1291                         float magn= sqrt(pow(i-tsz/2,2)+pow(j-tsz/2,2));
1292                         if(sd->texfade)
1293                                 texdata[i*tsz+j]= curve_strength(magn,tsz/2);
1294                         else
1295                                 texdata[i*tsz+j]= magn < tsz/2 ? 1 : 0;
1296                 }
1297         if(sd->texact != -1 && ss->texcache) {
1298                 for(i=0; i<tsz; ++i)
1299                         for(j=0; j<tsz; ++j) {
1300                                 const int col= ss->texcache[i*tsz+j];
1301                                 texdata[i*tsz+j]*= (((char*)&col)[0]+((char*)&col)[1]+((char*)&col)[2])/3.0f/255.0f;
1302                         }
1303         }
1304                 
1305         glGenTextures(1, &tex);
1306         glBindTexture(GL_TEXTURE_2D, tex);
1307         glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, tsz, tsz, 0, GL_ALPHA, GL_FLOAT, texdata);
1308         MEM_freeN(texdata);
1309
1310         return tex;
1311 }
1312
1313 void sculpt_radialcontrol_start(int mode)
1314 {
1315         SculptData *sd = sculpt_data();
1316         SculptSession *ss = sculpt_session();
1317         BrushData *br = sculptmode_brush();
1318         int orig=1, max=100;
1319
1320         if(mode == RADIALCONTROL_SIZE) {
1321                 orig = br->size;
1322                 max = 200;
1323         }
1324         else if(mode == RADIALCONTROL_STRENGTH) {
1325                 orig = br->strength;
1326                 max = 100;
1327         }
1328         else if(mode == RADIALCONTROL_ROTATION) {
1329                 if(sd->texact!=-1 && sd->mtex[sd->texact]) {
1330                         orig = sculpt_tex_angle();
1331                         max = 360;
1332                 }
1333                 else
1334                         mode = RADIALCONTROL_NONE;
1335         }
1336
1337         if(mode != RADIALCONTROL_NONE) {
1338                 ss->radialcontrol= radialcontrol_start(mode, sculpt_radialcontrol_callback, orig, max,
1339                                                        sculpt_radialcontrol_calctex());
1340         }
1341 }
1342 #endif
1343
1344 /* XXX: drawing code to go elsewhere!
1345 void sculpt_paint_brush(char clear)
1346 {
1347         if(sculpt_data()->flags & SCULPT_SCULPT_TOOL_DRAW) {
1348                 static short mvalo[2];
1349                 short mval[2];
1350                 const short rad= sculptmode_brush()->size;
1351
1352                 getmouseco_areawin(mval);
1353                 
1354                 persp(PERSP_WIN);
1355                 if(clear)
1356                         fdrawXORcirc(mval[0], mval[1], rad);
1357                 else
1358                         draw_sel_circle(mval, mvalo, rad, rad, 0);
1359                 
1360                 mvalo[0]= mval[0];
1361                 mvalo[1]= mval[1];
1362         }
1363 }
1364 */
1365
1366 void sculptmode_selectbrush_menu(void)
1367 {
1368         /* XXX: I guess menus belong elsewhere too?
1369
1370         SculptData *sd= sculpt_data();
1371         int val;
1372         
1373         pupmenu_set_active(sd->brush_type);
1374         
1375         val= pupmenu("Select Brush%t|Draw|Smooth|Pinch|Inflate|Grab|Layer|Flatten");
1376
1377         if(val>0) {
1378                 sd->brush_type= val;
1379
1380                 BIF_undo_push("Brush type");
1381                 
1382                 allqueue(REDRAWVIEW3D, 1);
1383                 allqueue(REDRAWBUTSEDIT, 1);
1384         }*/
1385 }
1386
1387 static void sculptmode_update_all_projverts(SculptSession *ss)
1388 {
1389         unsigned i;
1390
1391         if(!ss->projverts)
1392                 ss->projverts = MEM_mallocN(sizeof(ProjVert)*ss->totvert,"ProjVerts");
1393
1394         for(i=0; i<ss->totvert; ++i) {
1395                 project(ss, ss->vertexcosnos ? &ss->vertexcosnos[i * 6] : ss->mvert[i].co, ss->projverts[i].co);
1396                 ss->projverts[i].inside= 0;
1397         }
1398 }
1399
1400 /* Checks whether full update mode (slower) needs to be used to work with modifiers */
1401 char sculpt_modifiers_active(Object *ob)
1402 {
1403         ModifierData *md;
1404         
1405         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1406                 if(md->mode & eModifierMode_Realtime && md->type != eModifierType_Multires)
1407                         return 1;
1408         }
1409         
1410         return 0;
1411 }
1412
1413 /* Sculpt mode handles multires differently from regular meshes, but only if
1414    it's the last modifier on the stack and it is not on the first level */
1415 static struct MultiresModifierData *sculpt_multires_active(Object *ob)
1416 {
1417         ModifierData *md;
1418         
1419         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1420                 if(md->type == eModifierType_Multires && !md->next) {
1421                         MultiresModifierData *mmd = (MultiresModifierData*)md;
1422                         if(mmd->lvl != 1)
1423                                 return mmd;
1424                 }
1425         }
1426
1427         return NULL;
1428 }
1429
1430 static void sculpt_update_mesh_elements(SculptSession *ss, Object *ob)
1431 {
1432         if(sculpt_multires_active(ob)) {
1433                 DerivedMesh *dm = mesh_get_derived_final(NULL, ob, CD_MASK_BAREMESH); /* XXX scene=? */
1434                 ss->multires = 1;
1435                 ss->totvert = dm->getNumVerts(dm);
1436                 ss->totface = dm->getNumFaces(dm);
1437                 ss->mvert = dm->getVertDataArray(dm, CD_MVERT);
1438                 ss->mface = dm->getFaceDataArray(dm, CD_MFACE);
1439                 ss->face_normals = dm->getFaceDataArray(dm, CD_NORMAL);
1440         }
1441         else {
1442                 Mesh *me = get_mesh(ob);
1443                 ss->multires = 0;
1444                 ss->totvert = me->totvert;
1445                 ss->totface = me->totface;
1446                 ss->mvert = me->mvert;
1447                 ss->mface = me->mface;
1448                 ss->face_normals = NULL;
1449         }
1450 }
1451
1452 /* XXX: lots of drawing code (partial redraw), has to go elsewhere */
1453 #if 0
1454 void sculptmode_draw_wires(SculptSession *ss, int only_damaged)
1455 {
1456         Mesh *me = get_mesh(OBACT);
1457         int i;
1458
1459         bglPolygonOffset(1.0);
1460         glDepthMask(0);
1461         BIF_ThemeColor((OBACT==OBACT)?TH_ACTIVE:TH_SELECT);
1462
1463         for(i=0; i<me->totedge; i++) {
1464                 MEdge *med= &me->medge[i];
1465
1466                 if((!only_damaged || (ss->projverts[med->v1].inside || ss->projverts[med->v2].inside)) &&
1467                    (med->flag & ME_EDGEDRAW)) {
1468                         glDrawElements(GL_LINES, 2, GL_UNSIGNED_INT, &med->v1);
1469                 }
1470         }
1471
1472         glDepthMask(1);
1473         bglPolygonOffset(0.0);
1474 }
1475
1476 void sculptmode_draw_mesh(int only_damaged) 
1477 {
1478         int i, j, dt, drawCurrentMat = 1, matnr= -1;
1479         SculptSession *ss = sculpt_session();
1480
1481         sculpt_update_mesh_elements(ss, OBACT);
1482
1483         persp(PERSP_VIEW);
1484         mymultmatrix(OBACT->obmat);
1485         glEnable(GL_DEPTH_TEST);
1486         glEnable(GL_LIGHTING);
1487         /* XXX: GPU_set_object_materials(G.scene, OBACT, 0, NULL); */
1488         glEnable(GL_CULL_FACE);
1489
1490         glShadeModel(GL_SMOOTH);
1491
1492         glVertexPointer(3, GL_FLOAT, sizeof(MVert), &ss->mvert[0].co);
1493         glNormalPointer(GL_SHORT, sizeof(MVert), &ss->mvert[0].no);
1494
1495         dt= MIN2(G.vd->drawtype, OBACT->dt);
1496         if(dt==OB_WIRE)
1497                 glColorMask(0,0,0,0);
1498
1499         for(i=0; i<ss->totface; ++i) {
1500                 MFace *f= &ss->mface[i];
1501                 char inside= 0;
1502                 int new_matnr= f->mat_nr + 1;
1503                 
1504                 if(new_matnr != matnr)
1505                         drawCurrentMat= GPU_enable_material(matnr = new_matnr, NULL);
1506                 
1507                 /* If only_damaged!=0, only draw faces that are partially
1508                    inside the area(s) modified by the brush */
1509                 if(only_damaged) {
1510                         for(j=0; j<(f->v4?4:3); ++j) {
1511                                 if(ss->projverts[*((&f->v1)+j)].inside) {
1512                                         inside= 1;
1513                                         break;
1514                                 }
1515                         }
1516                 }
1517                 else
1518                         inside= 1;
1519                         
1520                 if(inside && drawCurrentMat)
1521                         glDrawElements(f->v4?GL_QUADS:GL_TRIANGLES, f->v4?4:3, GL_UNSIGNED_INT, &f->v1);
1522         }
1523
1524         glDisable(GL_CULL_FACE);
1525         glDisable(GL_LIGHTING);
1526         glColorMask(1,1,1,1);
1527
1528         if(dt==OB_WIRE || (OBACT->dtx & OB_DRAWWIRE))
1529                 sculptmode_draw_wires(ss, only_damaged);
1530
1531         glDisable(GL_DEPTH_TEST);
1532 }
1533 #endif
1534
1535 /* XXX */
1536 #if 0
1537 static void sculpt_undo_push(SculptData *sd)
1538 {
1539         switch(sd->brush->sculpt_tool) {
1540         case SCULPT_TOOL_DRAW:
1541                 BIF_undo_push("Draw Brush"); break;
1542         case SCULPT_TOOL_SMOOTH:
1543                 BIF_undo_push("Smooth Brush"); break;
1544         case SCULPT_TOOL_PINCH:
1545                 BIF_undo_push("Pinch Brush"); break;
1546         case SCULPT_TOOL_INFLATE:
1547                 BIF_undo_push("Inflate Brush"); break;
1548         case SCULPT_TOOL_GRAB:
1549                 BIF_undo_push("Grab Brush"); break;
1550         case SCULPT_TOOL_LAYER:
1551                 BIF_undo_push("Layer Brush"); break;
1552         case SCULPT_TOOL_FLATTEN:
1553                 BIF_undo_push("Flatten Brush"); break;
1554         default:
1555                 BIF_undo_push("Sculpting"); break;
1556         }
1557 }
1558 #endif
1559
1560 /**** Operator for applying a stroke (various attributes including mouse path)
1561       using the current brush. ****/
1562 static int sculpt_brush_stroke_poll(bContext *C)
1563 {
1564         // XXX: More to check for, of course
1565
1566         return G.f & G_SCULPTMODE;
1567 }
1568
1569 /* This is temporary, matrices should be data in operator for exec */
1570 static void sculpt_load_mats(bContext *C, bglMats *mats)
1571 {
1572         View3D *v3d = ((View3D*)CTX_wm_area(C)->spacedata.first);
1573         ARegion *ar = CTX_wm_region(C);
1574         Object *ob= CTX_data_active_object(C);
1575         float cpy[4][4];
1576         int i, j;
1577
1578         view3d_operator_needs_opengl(C);
1579
1580         Mat4MulMat4(cpy, v3d->viewmat, ob->obmat);
1581
1582         for(i = 0; i < 4; ++i) {
1583                 for(j = 0; j < 4; ++j) {
1584                         mats->projection[i*4+j] = v3d->winmat[i][j];
1585                         mats->modelview[i*4+j] = cpy[i][j];
1586                 }
1587         }
1588
1589         mats->viewport[0] = ar->winrct.xmin;
1590         mats->viewport[1] = ar->winrct.ymin;
1591         mats->viewport[2] = ar->winx;
1592         mats->viewport[3] = ar->winy;   
1593 }
1594
1595 /* Initialize the stroke cache invariants from operator properties */
1596 static void sculpt_update_cache_invariants(StrokeCache *cache, wmOperator *op)
1597 {
1598         memset(cache, 0, sizeof(StrokeCache));
1599
1600         cache->radius = RNA_float_get(op->ptr, "radius");
1601         RNA_float_get_array(op->ptr, "scale", cache->scale);
1602         cache->flag = RNA_int_get(op->ptr, "flag");
1603         RNA_float_get_array(op->ptr, "clip_tolerance", cache->clip_tolerance);
1604         RNA_int_get_array(op->ptr, "mouse", cache->mouse);
1605 }
1606
1607 /* Initialize the stroke cache variants from operator properties */
1608 static void sculpt_update_cache_variants(StrokeCache *cache, PointerRNA *ptr)
1609 {
1610         cache->flip = RNA_boolean_get(ptr, "flip");
1611 }
1612
1613 /* Initialize stroke operator properties */
1614 static void sculpt_brush_stroke_init(bContext *C, wmOperator *op, wmEvent *event, SculptSession *ss)
1615 {
1616         SculptData *sd = &CTX_data_scene(C)->sculptdata;
1617         Object *ob= CTX_data_active_object(C);
1618         ModifierData *md;
1619         float brush_center[3], brush_edge[3];
1620         float depth = get_depth(C, event->x, event->y);
1621         float size = brush_size(sd);
1622         float scale[3], clip_tolerance[3] = {0,0,0};
1623         int mouse[2], flag = 0;
1624
1625         unproject(ss, brush_center, event->x, event->y, depth);
1626         unproject(ss, brush_edge, event->x + size, event->y, depth);
1627
1628         RNA_float_set(op->ptr, "radius", VecLenf(brush_center, brush_edge));
1629
1630         /* Set scaling adjustment */
1631         scale[0] = 1.0f / ob->size[0];
1632         scale[1] = 1.0f / ob->size[1];
1633         scale[2] = 1.0f / ob->size[2];
1634         RNA_float_set_array(op->ptr, "scale", scale);
1635
1636         /* Initialize mirror modifier clipping */
1637         for(md= ob->modifiers.first; md; md= md->next) {
1638                 if(md->type==eModifierType_Mirror && (md->mode & eModifierMode_Realtime)) {
1639                         const MirrorModifierData *mmd = (MirrorModifierData*) md;
1640                         
1641                         /* Mark each axis that needs clipping along with its tolerance */
1642                         if(mmd->flag & MOD_MIR_CLIPPING) {
1643                                 flag |= CLIP_X << mmd->axis;
1644                                 if(mmd->tolerance > clip_tolerance[mmd->axis])
1645                                         clip_tolerance[mmd->axis] = mmd->tolerance;
1646                         }
1647                 }
1648         }
1649         RNA_int_set(op->ptr, "flag", flag);
1650         RNA_float_set_array(op->ptr, "clip_tolerance", clip_tolerance);
1651
1652         mouse[0] = event->x;
1653         mouse[1] = event->y;
1654         RNA_int_set_array(op->ptr, "mouse", mouse);
1655
1656         sculpt_update_cache_invariants(ss->cache, op);
1657 }
1658
1659 static int sculpt_brush_stroke_invoke(bContext *C, wmOperator *op, wmEvent *event)
1660 {
1661         SculptData *sd = &CTX_data_scene(C)->sculptdata;
1662         Object *ob= CTX_data_active_object(C);
1663         Mesh *me = get_mesh(ob);
1664
1665         // XXX: temporary, much of sculptsession data should be in rna properties
1666         sd->session = MEM_callocN(sizeof(SculptSession), "test sculpt session");
1667         sd->session->mvert = me->mvert;
1668         sd->session->totvert = me->totvert;
1669         sd->session->mats = MEM_callocN(sizeof(bglMats), "test sculpt mats");
1670         sd->session->cache = MEM_callocN(sizeof(StrokeCache), "stroke cache");
1671         
1672         // XXX: temporary matrix stuff
1673         sculpt_load_mats(C, sd->session->mats);
1674
1675         sculptmode_update_all_projverts(sd->session);
1676
1677         sculpt_brush_stroke_init(C, op, event, sd->session);
1678
1679         /* add modal handler */
1680         WM_event_add_modal_handler(C, &CTX_wm_window(C)->handlers, op);
1681         
1682         return OPERATOR_RUNNING_MODAL;
1683 }
1684
1685 /* Temporary, most of brush action will become rna properties */
1686 static void sculpt_action_init(BrushAction *a)
1687 {
1688         memset(a, 0, sizeof(BrushAction));
1689 }
1690
1691 static int sculpt_brush_stroke_modal(bContext *C, wmOperator *op, wmEvent *event)
1692 {
1693         PointerRNA itemptr;
1694         SculptData *sd = &CTX_data_scene(C)->sculptdata;
1695         BrushAction a;
1696         Object *ob= CTX_data_active_object(C);
1697         ARegion *ar = CTX_wm_region(C);
1698
1699         sculpt_action_init(&a);
1700         unproject(sd->session, a.symm.center_3d, event->x, event->y, get_depth(C, event->x, event->y));
1701
1702         /* Add to stroke */
1703         RNA_collection_add(op->ptr, "stroke", &itemptr);
1704         RNA_float_set_array(&itemptr, "location", a.symm.center_3d);
1705         RNA_boolean_set(&itemptr, "flip", event->shift);
1706         sculpt_update_cache_variants(sd->session->cache, &itemptr);
1707
1708         do_symmetrical_brush_actions(&CTX_data_scene(C)->sculptdata, sd->session->cache, &a);
1709         //calc_damaged_verts(sd->session, &a);
1710         BLI_freelistN(&sd->session->damaged_verts);
1711
1712         DAG_object_flush_update(CTX_data_scene(C), ob, OB_RECALC_DATA);
1713         ED_region_tag_redraw(ar);
1714
1715         /* Finished */
1716         if(event->type == LEFTMOUSE && event->val == 0) {
1717                 View3D *v3d = ((View3D*)CTX_wm_area(C)->spacedata.first);
1718                 if(v3d->depths)
1719                         v3d->depths->damaged= 1;
1720
1721                 return OPERATOR_FINISHED;
1722         }
1723
1724         return OPERATOR_RUNNING_MODAL;
1725 }
1726
1727 static int sculpt_brush_stroke_exec(bContext *C, wmOperator *op)
1728 {
1729         BrushAction a;
1730         Object *ob= CTX_data_active_object(C);
1731         ARegion *ar = CTX_wm_region(C);
1732         SculptData *sd = &CTX_data_scene(C)->sculptdata;
1733
1734         sculpt_update_cache_invariants(sd->session->cache, op);
1735
1736         RNA_BEGIN(op->ptr, itemptr, "stroke") {
1737                 sculpt_action_init(&a);         
1738                 RNA_float_get_array(&itemptr, "location", a.symm.center_3d);
1739                 sculpt_update_cache_variants(sd->session->cache, &itemptr);
1740
1741                 do_symmetrical_brush_actions(sd, sd->session->cache, &a);
1742                 BLI_freelistN(&sd->session->damaged_verts);
1743         }
1744         RNA_END;
1745
1746         DAG_object_flush_update(CTX_data_scene(C), ob, OB_RECALC_DATA);
1747         ED_region_tag_redraw(ar);
1748
1749         return OPERATOR_FINISHED;
1750 }
1751
1752 static void SCULPT_OT_brush_stroke(wmOperatorType *ot)
1753 {
1754         PropertyRNA *prop;
1755         float vec3f_def[] = {0,0,0};
1756         int vec2i_def[] = {0,0};
1757
1758         ot->flag |= OPTYPE_REGISTER;
1759
1760         /* identifiers */
1761         ot->name= "Sculpt Mode";
1762         ot->idname= "SCULPT_OT_brush_stroke";
1763         
1764         /* api callbacks */
1765         ot->invoke= sculpt_brush_stroke_invoke;
1766         ot->modal= sculpt_brush_stroke_modal;
1767         ot->exec= sculpt_brush_stroke_exec;
1768         ot->poll= sculpt_brush_stroke_poll;
1769
1770         /* properties */
1771         prop= RNA_def_property(ot->srna, "stroke", PROP_COLLECTION, PROP_NONE);
1772         RNA_def_property_struct_runtime(prop, &RNA_OperatorStrokeElement);
1773
1774         /* Brush radius measured in object space, projected from the brush setting in pixels */
1775         prop= RNA_def_property(ot->srna, "radius", PROP_FLOAT, PROP_NONE);
1776
1777         /* If the object has a scaling factor, brushes also need to be scaled
1778            to work as expected. */
1779         prop= RNA_def_property(ot->srna, "scale", PROP_FLOAT, PROP_VECTOR);
1780         RNA_def_property_array(prop, 3);
1781         RNA_def_property_float_array_default(prop, vec3f_def);
1782
1783         prop= RNA_def_property(ot->srna, "flag", PROP_INT, PROP_NONE);
1784
1785         prop= RNA_def_property(ot->srna, "clip_tolerance", PROP_FLOAT, PROP_VECTOR);
1786         RNA_def_property_array(prop, 3);
1787         RNA_def_property_float_array_default(prop, vec3f_def);
1788
1789         /* The initial 2D location of the mouse */
1790         prop= RNA_def_property(ot->srna, "mouse", PROP_INT, PROP_VECTOR);
1791         RNA_def_property_array(prop, 2);
1792         RNA_def_property_int_array_default(prop, vec2i_def);
1793 }
1794
1795 /**** Toggle operator for turning sculpt mode on or off ****/
1796
1797 static int sculpt_toggle_mode(bContext *C, wmOperator *op)
1798 {
1799         if(G.f & G_SCULPTMODE) {
1800                 /* Leave sculptmode */
1801                 G.f &= ~G_SCULPTMODE;
1802         }
1803         else {
1804                 /* Enter sculptmode */
1805
1806                 G.f |= G_SCULPTMODE;
1807
1808                 /* Needed for testing, if there's no brush then create one */
1809                 CTX_data_scene(C)->sculptdata.brush = add_brush("test brush");
1810         }
1811
1812         return OPERATOR_FINISHED;
1813 }
1814
1815 static void SCULPT_OT_toggle_mode(wmOperatorType *ot)
1816 {
1817         /* identifiers */
1818         ot->name= "Sculpt Mode";
1819         ot->idname= "SCULPT_OT_toggle_mode";
1820         
1821         /* api callbacks */
1822         ot->exec= sculpt_toggle_mode;
1823         ot->poll= ED_operator_object_active;
1824         
1825 }
1826
1827 void ED_operatortypes_sculpt()
1828 {
1829         WM_operatortype_append(SCULPT_OT_brush_stroke);
1830         WM_operatortype_append(SCULPT_OT_toggle_mode);
1831 }
1832
1833 void sculpt(SculptData *sd)
1834 {
1835         SculptSession *ss= sd->session;
1836         Object *ob= NULL; /*XXX */
1837         Mesh *me;
1838         MultiresModifierData *mmd = NULL;
1839         /* lastSigMouse is for the rake, to store the last place the mouse movement was significant */
1840         short mouse[2], mvalo[2], lastSigMouse[2],firsttime=1, mousebut;
1841         short modifier_calculations= 0;
1842         BrushAction *a = MEM_callocN(sizeof(BrushAction), "brush action");
1843         short spacing= 32000;
1844         int scissor_box[4];
1845         float offsetRot;
1846         int smooth_stroke = 0, i;
1847         int anchored, rake = 0 /* XXX: rake = ? */;
1848
1849         /* XXX: checking that sculpting is allowed
1850         if(!(G.f & G_SCULPTMODE) || G.obedit || !ob || ob->id.lib || !get_mesh(ob) || (get_mesh(ob)->totface == 0))
1851                 return;
1852         if(!(ob->lay & G.vd->lay))
1853                 error("Active object is not in this layer");
1854         if(ob_get_keyblock(ob)) {
1855                 if(!(ob->shapeflag & OB_SHAPE_LOCK)) {
1856                         error("Cannot sculpt on unlocked shape key");
1857                         return;
1858                 }
1859         }*/
1860         
1861         anchored = sd->brush->flag & BRUSH_ANCHORED;
1862         smooth_stroke = (sd->flags & SCULPT_INPUT_SMOOTH) && (sd->brush->sculpt_tool != SCULPT_TOOL_GRAB) && !anchored;
1863
1864         if(smooth_stroke)
1865                 sculpt_stroke_new(256);
1866
1867         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
1868         ss->damaged_verts.first = ss->damaged_verts.last = NULL;
1869         ss->vertexcosnos = NULL;
1870
1871         mmd = sculpt_multires_active(ob);
1872         sculpt_update_mesh_elements(ss, ob);
1873
1874         /* Check that vertex users are up-to-date */
1875         if(ob != active_ob || !ss->vertex_users || ss->vertex_users_size != ss->totvert) {
1876                 sculpt_vertexusers_free(ss);
1877                 calc_vertex_users(ss);
1878                 if(ss->projverts)
1879                         MEM_freeN(ss->projverts);
1880                 ss->projverts = NULL;
1881                 active_ob= ob;
1882         }
1883                 
1884         glEnableClientState(GL_VERTEX_ARRAY);
1885         glEnableClientState(GL_NORMAL_ARRAY);
1886
1887         /*XXX:
1888         persp(PERSP_VIEW);
1889         getmouseco_areawin(mvalo);*/
1890
1891         /* Init texture
1892            FIXME: Shouldn't be doing this every time! */
1893         if(sd->texrept!=SCULPTREPT_3D)
1894                 sculptmode_update_tex(sd);
1895
1896         /*XXX: getmouseco_areawin(mouse); */
1897         mvalo[0]= mouse[0];
1898         mvalo[1]= mouse[1];
1899         lastSigMouse[0]=mouse[0];
1900         lastSigMouse[1]=mouse[1];
1901         mousebut = 0; /* XXX: L_MOUSE; */
1902
1903         /* If modifier_calculations is true, then extra time must be spent
1904            updating the mesh. This takes a *lot* longer, so it's worth
1905            skipping if the modifier stack is empty. */
1906         modifier_calculations= sculpt_modifiers_active(ob);
1907
1908         if(modifier_calculations)
1909                 ss->vertexcosnos= mesh_get_mapped_verts_nors(NULL, ob); /* XXX: scene = ? */
1910         sculptmode_update_all_projverts(ss);
1911
1912         /* Capture original copy */
1913         if(sd->flags & SCULPT_DRAW_FAST)
1914                 glAccum(GL_LOAD, 1);
1915
1916         /* Get original scissor box */
1917         glGetIntegerv(GL_SCISSOR_BOX, scissor_box);
1918         
1919         /* For raking, get the original angle*/
1920         offsetRot=sculpt_tex_angle(sd);
1921
1922         me = get_mesh(ob);
1923
1924         while (/*XXX:get_mbut() & mousebut*/0) {
1925                 /* XXX: getmouseco_areawin(mouse); */
1926                 /* If rake, and the mouse has moved over 10 pixels (euclidean) (prevents jitter) then get the new angle */
1927                 if (rake && (pow(lastSigMouse[0]-mouse[0],2)+pow(lastSigMouse[1]-mouse[1],2))>100){
1928                         /*Nasty looking, but just orig + new angle really*/
1929                         set_tex_angle(sd, offsetRot+180.+to_deg(atan2((float)(mouse[1]-lastSigMouse[1]),(float)(mouse[0]-lastSigMouse[0]))));
1930                         lastSigMouse[0]=mouse[0];
1931                         lastSigMouse[1]=mouse[1];
1932                 }
1933                 
1934                 if(firsttime || mouse[0]!=mvalo[0] || mouse[1]!=mvalo[1] ||
1935                    sd->brush->flag & BRUSH_AIRBRUSH) {
1936                         a->firsttime = firsttime;
1937                         firsttime= 0;
1938
1939                         if(smooth_stroke)
1940                                 sculpt_stroke_add_point(ss->stroke, mouse[0], mouse[1]);
1941
1942                         spacing+= sqrt(pow(mvalo[0]-mouse[0],2)+pow(mvalo[1]-mouse[1],2));
1943
1944                         if(modifier_calculations && !ss->vertexcosnos)
1945                                 ss->vertexcosnos= mesh_get_mapped_verts_nors(NULL, ob); /*XXX scene = ? */
1946
1947                         if(sd->brush->sculpt_tool != SCULPT_TOOL_GRAB) {
1948                                 if(anchored) {
1949                                         /* Restore the mesh before continuing with anchored stroke */
1950                                         if(a->mesh_store) {
1951                                                 for(i = 0; i < ss->totvert; ++i) {
1952                                                         VecCopyf(ss->mvert[i].co, &a->mesh_store[i].x);
1953                                                         ss->mvert[i].no[0] = a->orig_norms[i][0];
1954                                                         ss->mvert[i].no[1] = a->orig_norms[i][1];
1955                                                         ss->mvert[i].no[2] = a->orig_norms[i][2];
1956                                                 }
1957                                         }
1958                                         
1959                                         //do_symmetrical_brush_actions(sd, a, mouse, NULL);
1960                                 }
1961                                 else {
1962                                         if(smooth_stroke) {
1963                                                 sculpt_stroke_apply(sd, ss->stroke);
1964                                         }
1965                                         else if(sd->spacing==0 || spacing>sd->spacing) {
1966                                                 //do_symmetrical_brush_actions(sd, a, mouse, NULL);
1967                                                 spacing= 0;
1968                                         }
1969                                 }
1970                         }
1971                         else {
1972                                 //do_symmetrical_brush_actions(sd, a, mouse, mvalo);
1973                                 unproject(ss, sd->pivot, mouse[0], mouse[1], a->depth);
1974                         }
1975
1976                         if((!ss->multires && modifier_calculations) || ob_get_keyblock(ob)) {
1977                                 /* XXX: DAG_object_flush_update(G.scene, ob, OB_RECALC_DATA); */ }
1978
1979                         if(modifier_calculations || sd->brush->sculpt_tool == SCULPT_TOOL_GRAB || !(sd->flags & SCULPT_DRAW_FAST)) {
1980                                 calc_damaged_verts(ss, a);
1981                                 /*XXX: scrarea_do_windraw(curarea);
1982                                 screen_swapbuffers(); */
1983                         } else { /* Optimized drawing */
1984                                 calc_damaged_verts(ss, a);
1985
1986                                 /* Draw the stored image to the screen */
1987                                 glAccum(GL_RETURN, 1);
1988
1989                                 sculpt_clear_damaged_areas(ss);
1990                                 
1991                                 /* Draw all the polygons that are inside the modified area(s) */
1992                                 glScissor(scissor_box[0], scissor_box[1], scissor_box[2], scissor_box[3]);
1993                                 /* XXX: sculptmode_draw_mesh(1); */
1994                                 glAccum(GL_LOAD, 1);
1995
1996                                 projverts_clear_inside(ss);
1997
1998                                 /* XXX: persp(PERSP_WIN); */
1999                                 glDisable(GL_DEPTH_TEST);
2000                                 
2001                                 /* Draw cursor */
2002                                 if(sd->flags & SCULPT_TOOL_DRAW)
2003                                         fdrawXORcirc((float)mouse[0],(float)mouse[1],sd->brush->size);
2004                                 /* XXX: if(smooth_stroke)
2005                                    sculpt_stroke_draw();
2006                                 
2007                                 myswapbuffers(); */
2008                         }
2009
2010                         BLI_freelistN(&ss->damaged_rects);
2011                         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
2012         
2013                         mvalo[0]= mouse[0];
2014                         mvalo[1]= mouse[1];
2015
2016                         if(ss->vertexcosnos) {
2017                                 MEM_freeN(ss->vertexcosnos);
2018                                 ss->vertexcosnos= NULL;
2019                         }
2020
2021                 }
2022                 else { /*XXX:BIF_wait_for_statechange();*/ }
2023         }
2024
2025         /* Set the rotation of the brush back to what it was before any rake */
2026         set_tex_angle(sd, offsetRot);
2027         
2028         if(smooth_stroke) {
2029                 sculpt_stroke_apply_all(sd, ss->stroke);
2030                 calc_damaged_verts(ss, a);
2031                 BLI_freelistN(&ss->damaged_rects);
2032         }
2033
2034         if(a->layer_disps) MEM_freeN(a->layer_disps);
2035         if(a->mesh_store) MEM_freeN(a->mesh_store);
2036         if(a->orig_norms) MEM_freeN(a->orig_norms);
2037         for(i=0; i<8; ++i)
2038                 BLI_freelistN(&a->grab_active_verts[i]);
2039         MEM_freeN(a);
2040         sculpt_stroke_free(ss->stroke);
2041         ss->stroke = NULL;
2042
2043         if(mmd) {
2044                 if(mmd->undo_verts && mmd->undo_verts != ss->mvert)
2045                         MEM_freeN(mmd->undo_verts);
2046                 
2047                 mmd->undo_verts = ss->mvert;
2048                 mmd->undo_verts_tot = ss->totvert;
2049         }
2050
2051         //sculpt_undo_push(sd);
2052
2053         /* XXX: if(G.vd->depths) G.vd->depths->damaged= 1;
2054            allqueue(REDRAWVIEW3D, 0); */
2055 }
2056
2057 /* Partial Mesh Visibility */
2058
2059 /* XXX: Partial vis. always was a mess, have to figure something out */
2060 #if 0
2061 /* mode: 0=hide outside selection, 1=hide inside selection */
2062 static void sculptmode_do_pmv(Object *ob, rcti *hb_2d, int mode)
2063 {
2064         Mesh *me= get_mesh(ob);
2065         float hidebox[6][3];
2066         vec3f plane_normals[4];
2067         float plane_ds[4];
2068         unsigned i, j;
2069         unsigned ndx_show, ndx_hide;
2070         MVert *nve;
2071         unsigned face_cnt_show= 0, face_ndx_show= 0;
2072         unsigned edge_cnt_show= 0, edge_ndx_show= 0;
2073         unsigned *old_map= NULL;
2074         const unsigned SHOW= 0, HIDE=1;
2075
2076         /* Convert hide box from 2D to 3D */
2077         unproject(hidebox[0], hb_2d->xmin, hb_2d->ymax, 1);
2078         unproject(hidebox[1], hb_2d->xmax, hb_2d->ymax, 1);
2079         unproject(hidebox[2], hb_2d->xmax, hb_2d->ymin, 1);
2080         unproject(hidebox[3], hb_2d->xmin, hb_2d->ymin, 1);
2081         unproject(hidebox[4], hb_2d->xmin, hb_2d->ymax, 0);
2082         unproject(hidebox[5], hb_2d->xmax, hb_2d->ymin, 0);
2083         
2084         /* Calculate normals for each side of hide box */
2085         CalcNormFloat(hidebox[0], hidebox[1], hidebox[4], &plane_normals[0].x);
2086         CalcNormFloat(hidebox[1], hidebox[2], hidebox[5], &plane_normals[1].x);
2087         CalcNormFloat(hidebox[2], hidebox[3], hidebox[5], &plane_normals[2].x);
2088         CalcNormFloat(hidebox[3], hidebox[0], hidebox[4], &plane_normals[3].x);
2089         
2090         /* Calculate D for each side of hide box */
2091         for(i= 0; i<4; ++i)
2092                 plane_ds[i]= hidebox[i][0]*plane_normals[i].x + hidebox[i][1]*plane_normals[i].y +
2093                         hidebox[i][2]*plane_normals[i].z;
2094         
2095         /* Add partial visibility to mesh */
2096         if(!me->pv) {
2097                 me->pv= MEM_callocN(sizeof(PartialVisibility),"PartialVisibility");
2098         } else {
2099                 old_map= MEM_callocN(sizeof(unsigned)*me->pv->totvert,"PMV oldmap");
2100                 for(i=0; i<me->pv->totvert; ++i) {
2101                         old_map[i]= me->pv->vert_map[i]<me->totvert?0:1;
2102                 }
2103                 mesh_pmv_revert(ob, me);
2104         }
2105         
2106         /* Kill sculpt data */
2107         active_ob= NULL;
2108         
2109         /* Initalize map with which verts are to be hidden */
2110         me->pv->vert_map= MEM_mallocN(sizeof(unsigned)*me->totvert, "PMV vertmap");
2111         me->pv->totvert= me->totvert;
2112         me->totvert= 0;
2113         for(i=0; i<me->pv->totvert; ++i) {
2114                 me->pv->vert_map[i]= mode ? HIDE:SHOW;
2115                 for(j=0; j<4; ++j) {
2116                         if(me->mvert[i].co[0] * plane_normals[j].x +
2117                            me->mvert[i].co[1] * plane_normals[j].y +
2118                            me->mvert[i].co[2] * plane_normals[j].z < plane_ds[j] ) {
2119                                 me->pv->vert_map[i]= mode ? SHOW:HIDE; /* Vert is outside the hide box */
2120                                 break;
2121                         }
2122                 }
2123                 if(old_map && old_map[i]) me->pv->vert_map[i]= 1;
2124                 if(!me->pv->vert_map[i]) ++me->totvert;
2125
2126         }
2127         if(old_map) MEM_freeN(old_map);
2128
2129         /* Find out how many faces to show */
2130         for(i=0; i<me->totface; ++i) {
2131                 if(!me->pv->vert_map[me->mface[i].v1] &&
2132                    !me->pv->vert_map[me->mface[i].v2] &&
2133                    !me->pv->vert_map[me->mface[i].v3]) {
2134                         if(me->mface[i].v4) {
2135                                 if(!me->pv->vert_map[me->mface[i].v4])
2136                                         ++face_cnt_show;
2137                         }
2138                         else ++face_cnt_show;
2139                 }
2140         }
2141         /* Find out how many edges to show */
2142         for(i=0; i<me->totedge; ++i) {
2143                 if(!me->pv->vert_map[me->medge[i].v1] &&
2144                    !me->pv->vert_map[me->medge[i].v2])
2145                         ++edge_cnt_show;
2146         }
2147
2148         /* Create new vert array and reset each vert's map with map[old]=new index */
2149         nve= MEM_mallocN(sizeof(MVert)*me->pv->totvert, "PMV verts");
2150         ndx_show= 0; ndx_hide= me->totvert;
2151         for(i=0; i<me->pv->totvert; ++i) {
2152                 if(me->pv->vert_map[i]) {
2153                         me->pv->vert_map[i]= ndx_hide;
2154                         nve[me->pv->vert_map[i]]= me->mvert[i];
2155                         ++ndx_hide;
2156                 } else {
2157                         me->pv->vert_map[i]= ndx_show;
2158                         nve[me->pv->vert_map[i]]= me->mvert[i];
2159                         ++ndx_show;
2160                 }
2161         }
2162         CustomData_free_layer_active(&me->vdata, CD_MVERT, me->pv->totvert);
2163         me->mvert= CustomData_add_layer(&me->vdata, CD_MVERT, CD_ASSIGN, nve, me->totvert);
2164
2165         /* Create new face array */
2166         me->pv->old_faces= me->mface;
2167         me->pv->totface= me->totface;
2168         me->mface= MEM_mallocN(sizeof(MFace)*face_cnt_show, "PMV faces");
2169         for(i=0; i<me->totface; ++i) {
2170                 MFace *pr_f= &me->pv->old_faces[i];
2171                 char show= 0;
2172
2173                 if(me->pv->vert_map[pr_f->v1] < me->totvert &&
2174                    me->pv->vert_map[pr_f->v2] < me->totvert &&
2175                    me->pv->vert_map[pr_f->v3] < me->totvert) {
2176                         if(pr_f->v4) {
2177                                 if(me->pv->vert_map[pr_f->v4] < me->totvert)
2178                                         show= 1;
2179                         }
2180                         else show= 1;
2181                 }
2182
2183                 if(show) {
2184                         MFace *cr_f= &me->mface[face_ndx_show];
2185                         *cr_f= *pr_f;
2186                         cr_f->v1= me->pv->vert_map[pr_f->v1];
2187                         cr_f->v2= me->pv->vert_map[pr_f->v2];
2188                         cr_f->v3= me->pv->vert_map[pr_f->v3];
2189                         cr_f->v4= pr_f->v4 ? me->pv->vert_map[pr_f->v4] : 0;
2190                         test_index_face(cr_f,NULL,0,pr_f->v4?4:3);
2191                         ++face_ndx_show;
2192                 }
2193         }
2194         me->totface= face_cnt_show;
2195         CustomData_set_layer(&me->fdata, CD_MFACE, me->mface);
2196
2197         /* Create new edge array */
2198         me->pv->old_edges= me->medge;
2199         me->pv->totedge= me->totedge;
2200         me->medge= MEM_mallocN(sizeof(MEdge)*edge_cnt_show, "PMV edges");
2201         me->pv->edge_map= MEM_mallocN(sizeof(int)*me->pv->totedge,"PMV edgemap");
2202         for(i=0; i<me->totedge; ++i) {
2203                 if(me->pv->vert_map[me->pv->old_edges[i].v1] < me->totvert &&
2204                    me->pv->vert_map[me->pv->old_edges[i].v2] < me->totvert) {
2205                         MEdge *cr_e= &me->medge[edge_ndx_show];
2206                         me->pv->edge_map[i]= edge_ndx_show;
2207                         *cr_e= me->pv->old_edges[i];
2208                         cr_e->v1= me->pv->vert_map[me->pv->old_edges[i].v1];
2209                         cr_e->v2= me->pv->vert_map[me->pv->old_edges[i].v2];
2210                         ++edge_ndx_show;
2211                 }
2212                 else me->pv->edge_map[i]= -1;
2213         }
2214         me->totedge= edge_cnt_show;
2215         CustomData_set_layer(&me->edata, CD_MEDGE, me->medge);
2216
2217         /* XXX: DAG_object_flush_update(G.scene, OBACT, OB_RECALC_DATA); */
2218 }
2219
2220 static rcti sculptmode_pmv_box()
2221 {
2222         /*XXX:  short down[2], mouse[2];
2223         rcti ret;
2224
2225         getmouseco_areawin(down);
2226
2227         while((get_mbut()&L_MOUSE) || (get_mbut()&R_MOUSE)) {
2228                 getmouseco_areawin(mouse);
2229
2230                 scrarea_do_windraw(curarea);
2231
2232                 persp(PERSP_WIN);
2233                 glLineWidth(2);
2234                 setlinestyle(2);
2235                 sdrawXORline(down[0],down[1],mouse[0],down[1]);
2236                 sdrawXORline(mouse[0],down[1],mouse[0],mouse[1]);
2237                 sdrawXORline(mouse[0],mouse[1],down[0],mouse[1]);
2238                 sdrawXORline(down[0],mouse[1],down[0],down[1]);
2239                 setlinestyle(0);
2240                 glLineWidth(1);
2241                 persp(PERSP_VIEW);
2242
2243                 screen_swapbuffers();
2244                 backdrawview3d(0);
2245         }
2246
2247         ret.xmin= down[0]<mouse[0]?down[0]:mouse[0];
2248         ret.ymin= down[1]<mouse[1]?down[1]:mouse[1];
2249         ret.xmax= down[0]>mouse[0]?down[0]:mouse[0];
2250         ret.ymax= down[1]>mouse[1]?down[1]:mouse[1];
2251         return ret;*/
2252 }
2253
2254 void sculptmode_pmv(int mode)
2255 {
2256         Object *ob= NULL; /*XXX: OBACT; */
2257         rcti hb_2d;
2258         
2259         if(ob_get_key(ob)) {
2260                 error("Cannot hide mesh with shape keys enabled");
2261                 return;
2262         }
2263         
2264         hb_2d= sculptmode_pmv_box(); /* Get 2D hide box */
2265         
2266         sculptmode_correct_state();
2267
2268         waitcursor(1);
2269
2270         if(hb_2d.xmax-hb_2d.xmin > 3 && hb_2d.ymax-hb_2d.ymin > 3) {
2271                 init_sculptmatrices();
2272
2273                 sculptmode_do_pmv(ob,&hb_2d,mode);
2274         }
2275         else mesh_pmv_off(ob, get_mesh(ob));
2276
2277         /*XXX: scrarea_do_windraw(curarea); */
2278
2279         BIF_undo_push("Partial mesh hide");
2280
2281         waitcursor(0);
2282 }
2283 #endif