acd9f085532d6a850d9eb89253a2e0dd9d0327a0
[blender.git] / source / blender / editors / sculpt / sculpt.c
1 /*
2  * $Id: sculptmode.c 18309 2009-01-04 07:47:11Z nicholasbishop $
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software  Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2006 by Nicholas Bishop
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  *
29  * Implements the Sculpt Mode tools
30  *
31  * BDR_sculptmode.h
32  *
33  */
34
35 #include "MEM_guardedalloc.h"
36
37 #include "BLI_arithb.h"
38 #include "BLI_blenlib.h"
39 #include "BLI_dynstr.h"
40
41 #include "DNA_armature_types.h"
42 #include "DNA_brush_types.h"
43 #include "DNA_image_types.h"
44 #include "DNA_key_types.h"
45 #include "DNA_mesh_types.h"
46 #include "DNA_meshdata_types.h"
47 #include "DNA_modifier_types.h"
48 #include "DNA_object_types.h"
49 #include "DNA_screen_types.h"
50 #include "DNA_scene_types.h"
51 #include "DNA_texture_types.h"
52 #include "DNA_view3d_types.h"
53 #include "DNA_userdef_types.h"
54 #include "DNA_color_types.h"
55
56 #include "BKE_context.h"
57 #include "BKE_customdata.h"
58 #include "BKE_DerivedMesh.h"
59 #include "BKE_depsgraph.h"
60 #include "BKE_global.h"
61 #include "BKE_image.h"
62 #include "BKE_key.h"
63 #include "BKE_library.h"
64 #include "BKE_main.h"
65 #include "BKE_mesh.h"
66 #include "BKE_modifier.h"
67 #include "BKE_multires.h"
68 #include "BKE_sculpt.h"
69 #include "BKE_texture.h"
70 #include "BKE_utildefines.h"
71 #include "BKE_colortools.h"
72
73 #include "BIF_gl.h"
74 #include "BIF_glutil.h"
75
76 #include "WM_api.h"
77 #include "WM_types.h"
78 #include "ED_screen.h"
79 #include "ED_sculpt.h"
80 #include "ED_space_api.h"
81 #include "sculpt_intern.h"
82 #include "../space_view3d/view3d_intern.h" /* XXX: oh no, the next generation of bad level call! should move ViewDepths perhaps (also used for view matrices) */
83
84 #include "RNA_access.h"
85 #include "RNA_define.h"
86
87 #include "IMB_imbuf_types.h"
88
89 #include "RE_render_ext.h"
90 #include "RE_shader_ext.h" /*for multitex_ext*/
91
92 #include "GPU_draw.h"
93
94 #include <math.h>
95 #include <stdlib.h>
96 #include <string.h>
97
98 /* Number of vertices to average in order to determine the flatten distance */
99 #define FLATTEN_SAMPLE_SIZE 10
100
101 /* Texture cache size */
102 #define TC_SIZE 256
103
104 /* ===== STRUCTS =====
105  *
106  */
107
108 /* ActiveData stores an Index into the mvert array of Mesh, plus Fade, which
109    stores how far the vertex is from the brush center, scaled to the range [0,1]. */
110 typedef struct ActiveData {
111         struct ActiveData *next, *prev;
112         unsigned int Index;
113         float Fade;
114         float dist;
115 } ActiveData;
116
117 typedef struct BrushActionSymm {
118         float center_3d[3];
119         char index;
120
121         float up[3], right[3], out[3];
122
123         /* Grab brush */
124         float grab_delta[3];
125 } BrushActionSymm;
126
127 typedef enum StrokeFlags {
128         CLIP_X = 1,
129         CLIP_Y = 2,
130         CLIP_Z = 4
131 } StrokeFlags;
132
133 /* Cache stroke properties that don't change after
134    the initialization at the start of a stroke. Used because
135    RNA property lookup isn't particularly fast.
136
137    For descriptions of these settings, check the operator properties.
138 */
139 typedef struct StrokeCache {
140         float radius;
141         float scale[3];
142         float flip;
143         int flag;
144         float clip_tolerance[3];
145         int mouse[2];
146 } StrokeCache;
147
148 typedef struct BrushAction {
149         BrushActionSymm symm;
150
151         char firsttime;
152
153         /* Some brushes need access to original mesh vertices */
154         vec3f *mesh_store;
155         short (*orig_norms)[3];
156
157         float prev_radius;
158         float radius;
159
160         float *layer_disps;
161
162         float anchored_rot;
163
164         /* Grab brush */
165         ListBase grab_active_verts[8];
166         float depth;
167 } BrushAction;
168
169 typedef struct RectNode {
170         struct RectNode *next, *prev;
171         rcti r;
172 } RectNode;
173
174 /* Used to store to 2D screen coordinates of each vertex in the mesh. */
175 typedef struct ProjVert {
176         short co[2];
177         
178         /* Used to mark whether a vertex is inside a rough bounding box
179            containing the brush. */
180         char inside;
181 } ProjVert;
182
183 static Object *active_ob= NULL;
184
185 static void init_brushaction(SculptData *sd, BrushAction *a, short *, short *);
186
187
188 /* ===== MEMORY =====
189  * 
190  * Allocate/initialize/free data
191  */
192
193 static void sculpt_init_session(SculptData *sd)
194 {
195         if(sd->session)
196                 ;/*XXX: sculptsession_free(G.scene); */
197         sd->session= MEM_callocN(sizeof(SculptSession), "SculptSession");
198 }
199
200 /* vertex_users is an array of Lists that store all the faces that use a
201    particular vertex. vertex_users is in the same order as mesh.mvert */
202 static void calc_vertex_users(SculptSession *ss)
203 {
204         int i,j;
205         IndexNode *node= NULL;
206
207         sculpt_vertexusers_free(ss);
208         
209         /* For efficiency, use vertex_users_mem as a memory pool (may be larger
210            than necessary if mesh has triangles, but only one alloc is needed.) */
211         ss->vertex_users= MEM_callocN(sizeof(ListBase) * ss->totvert, "vertex_users");
212         ss->vertex_users_size= ss->totvert;
213         ss->vertex_users_mem= MEM_callocN(sizeof(IndexNode)*ss->totface*4, "vertex_users_mem");
214         node= ss->vertex_users_mem;
215
216         /* Find the users */
217         for(i=0; i<ss->totface; ++i){
218                 for(j=0; j<(ss->mface[i].v4?4:3); ++j, ++node) {
219                         node->index=i;
220                         BLI_addtail(&ss->vertex_users[((unsigned int*)(&ss->mface[i]))[j]], node);
221                 }
222         }
223 }
224
225 /* ===== INTERFACE =====
226  */
227
228 /* XXX: this can probably removed entirely */
229 #if 0
230 void sculptmode_rem_tex(void *junk0,void *junk1)
231 {
232         MTex *mtex= G.scene->sculptdata.mtex[G.scene->sculptdata.texact];
233         if(mtex) {
234                 SculptSession *ss= sculpt_session();
235                 if(mtex->tex) mtex->tex->id.us--;
236                 MEM_freeN(mtex);
237                 G.scene->sculptdata.mtex[G.scene->sculptdata.texact]= NULL;
238                 /* Clear brush preview */
239                 if(ss->texcache) {
240                         MEM_freeN(ss->texcache);
241                         ss->texcache= NULL;
242                 }
243                 // XXX BIF_undo_push("Unlink brush texture");
244                 allqueue(REDRAWBUTSEDIT, 0);
245                 allqueue(REDRAWOOPS, 0);
246         }
247 }
248 #endif
249
250 /* ===== OPENGL =====
251  *
252  * Simple functions to get data from the GL
253  */
254
255 /* Store the modelview and projection matrices and viewport. */
256 void init_sculptmatrices()
257 {
258         /* XXX: probably becomes context data?
259
260         SculptSession *ss= sculpt_session();
261
262         glMatrixMode(GL_MODELVIEW);
263         glPushMatrix();
264         glMultMatrixf(OBACT->obmat);
265
266         if(!ss->mats)
267                 ss->mats = MEM_callocN(sizeof(bglMats), "sculpt bglmats");
268         bgl_get_mats(ss->mats);
269         
270         glPopMatrix();
271         */
272 }
273
274 /* Uses window coordinates (x,y) to find the depth in the GL depth buffer. If
275    available, G.vd->depths is used so that the brush doesn't sculpt on top of
276    itself (G.vd->depths is only updated at the end of a brush stroke.) */
277 float get_depth(bContext *C, short x, short y)
278 {
279         float depth;
280         ScrArea *sa= CTX_wm_area(C);
281
282         if(sa->spacetype==SPACE_VIEW3D) { // should check this in context instead?
283                 ViewDepths *vd = ((View3D*)sa->spacedata.first)->depths;
284                 
285                 y -= CTX_wm_region(C)->winrct.ymin;
286
287                 if(vd && vd->depths && x > 0 && y > 0 && x < vd->w && y < vd->h)
288                         return vd->depths[y * vd->w + x];
289         }
290
291         fprintf(stderr, "Error: Bad depth store!\n");
292         return 1;
293 }
294
295 /* Uses window coordinates (x,y) and depth component z to find a point in
296    modelspace */
297 void unproject(SculptSession *ss, float out[3], const short x, const short y, const float z)
298 {
299         double ux, uy, uz;
300
301         gluUnProject(x,y,z, ss->mats->modelview, ss->mats->projection,
302                      (GLint *)ss->mats->viewport, &ux, &uy, &uz );
303         out[0] = ux;
304         out[1] = uy;
305         out[2] = uz;
306 }
307
308 /* Convert a point in model coordinates to 2D screen coordinates. */
309 static void projectf(SculptSession *ss, const float v[3], float p[2])
310 {
311         double ux, uy, uz;
312
313         gluProject(v[0],v[1],v[2], ss->mats->modelview, ss->mats->projection,
314                    (GLint *)ss->mats->viewport, &ux, &uy, &uz);
315         p[0]= ux;
316         p[1]= uy;
317 }
318
319 static void project(SculptSession *ss, const float v[3], short p[2])
320 {
321         float f[2];
322         projectf(ss, v, f);
323
324         p[0]= f[0];
325         p[1]= f[1];
326 }
327
328 /* ===== Sculpting =====
329  *
330  */
331
332 /* Return modified brush size. Uses current tablet pressure (if available) to
333    shrink the brush. Skipped for grab brush because only the first mouse down
334    size is used, which is small if the user has just touched the pen to the
335    tablet */
336 char brush_size(SculptData *sd, Brush *b)
337 {
338         float size= b->size;
339         float pressure= 0; /* XXX: get_pressure(); */
340         short activedevice= 0; /* XXX: get_activedevice(); */
341         
342         if(b->sculpt_tool!=SCULPT_TOOL_GRAB) {
343                 const float size_factor= sd->tablet_size / 10.0f;
344                 
345                 /* XXX: tablet stuff
346                 if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
347                         size*= sd->tablet_size==0?1:
348                         (1-size_factor) + pressure*size_factor;*/
349         }
350         
351         return size;
352 }
353
354 /* Return modified brush strength. Includes the direction of the brush, positive
355    values pull vertices, negative values push. Uses tablet pressure and a
356    special multiplier found experimentally to scale the strength factor. */
357 float brush_strength(SculptData *sd, StrokeCache *cache, BrushAction *a)
358 {
359         float dir= sd->brush->flag & BRUSH_DIR_IN ? -1 : 1;
360         float pressure= 1;
361         short activedevice= 0;/*XXX: get_activedevice(); */
362         float flip= cache->flip ? -1:1;
363         float anchored = sd->brush->flag & BRUSH_ANCHORED ? 25 : 1;
364
365         const float strength_factor= sd->tablet_strength / 10.0f;
366
367         /* XXX: tablet stuff */
368 #if 0
369         if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
370                 pressure= sd->sculptdata.tablet_strength==0?1:
371                         (1-strength_factor) + 1/*XXX: get_pressure()*/ *strength_factor;
372         
373         /* Flip direction for eraser */
374         if(activedevice==DEV_ERASER)
375                 dir= -dir;
376 #endif
377
378         switch(sd->brush->sculpt_tool){
379         case SCULPT_TOOL_DRAW:
380         case SCULPT_TOOL_LAYER:
381                 return sd->brush->alpha / 50.0f * dir * pressure * flip * anchored; /*XXX: not sure why? multiplied by G.vd->grid */;
382         case SCULPT_TOOL_SMOOTH:
383                 return sd->brush->alpha / .5f * pressure * anchored;
384         case SCULPT_TOOL_PINCH:
385                 return sd->brush->alpha / 10.0f * dir * pressure * flip * anchored;
386         case SCULPT_TOOL_GRAB:
387                 return 1;
388         case SCULPT_TOOL_INFLATE:
389                 return sd->brush->alpha / 50.0f * dir * pressure * flip * anchored;
390         case SCULPT_TOOL_FLATTEN:
391                 return sd->brush->alpha / 5.0f * pressure * anchored;
392         default:
393                 return 0;
394         }
395 }
396
397 /* For clipping against a mirror modifier */
398 void sculpt_clip(StrokeCache *cache, float *co, const float val[3])
399 {
400         char i;
401         for(i=0; i<3; ++i) {
402                 if((cache->flag & (CLIP_X << i)) && (fabs(co[i]) <= cache->clip_tolerance[i]))
403                         co[i]= 0.0f;
404                 else
405                         co[i]= val[i];
406         }               
407 }
408
409 void sculpt_axislock(SculptData *sd, float *co)
410 {
411         if (sd->flags & (SCULPT_LOCK_X|SCULPT_LOCK_Y|SCULPT_LOCK_Z)) return;
412         /* XXX: if(G.vd->twmode == V3D_MANIP_LOCAL) { */
413         if(0) {
414                 float mat[3][3], imat[3][3];
415                 /* XXX: Mat3CpyMat4(mat, OBACT->obmat); */
416                 Mat3Inv(imat, mat);
417                 Mat3MulVecfl(mat, co);
418                 if (sd->flags & SCULPT_LOCK_X) co[0] = 0.0;
419                 if (sd->flags & SCULPT_LOCK_Y) co[1] = 0.0;
420                 if (sd->flags & SCULPT_LOCK_Z) co[2] = 0.0;             
421                 Mat3MulVecfl(imat, co);
422         } else {
423                 if (sd->flags & SCULPT_LOCK_X) co[0] = 0.0;
424                 if (sd->flags & SCULPT_LOCK_Y) co[1] = 0.0;
425                 if (sd->flags & SCULPT_LOCK_Z) co[2] = 0.0;             
426         }
427 }
428
429 static void add_norm_if(float view_vec[3], float out[3], float out_flip[3], const short no[3])
430 {
431         float fno[3] = {no[0], no[1], no[2]};
432
433         Normalize(fno);
434
435         if((Inpf(view_vec, fno)) > 0) {
436                 VecAddf(out, out, fno);
437         } else {
438                 VecAddf(out_flip, out_flip, fno); /* out_flip is used when out is {0,0,0} */
439         }
440 }
441
442 /* Currently only for the draw brush; finds average normal for all active
443    vertices */
444 void calc_area_normal(SculptData *sd, float out[3], const BrushAction *a, const float *outdir, const ListBase* active_verts)
445 {
446         ActiveData *node = active_verts->first;
447         const int view = 0; /* XXX: should probably be a flag, not number: sd->brush_type==SCULPT_TOOL_DRAW ? sculptmode_brush()->view : 0; */
448         float out_flip[3];
449         
450         out[0]=out[1]=out[2] = out_flip[0]=out_flip[1]=out_flip[2] = 0;
451
452         if(sd->brush->flag & BRUSH_ANCHORED) {
453                 for(; node; node = node->next)
454                         add_norm_if(((BrushAction*)a)->symm.out, out, out_flip, a->orig_norms[node->Index]);
455         }
456         else {
457                 for(; node; node = node->next)
458                         add_norm_if(((BrushAction*)a)->symm.out, out, out_flip, sd->session->mvert[node->Index].no);
459         }
460
461         if (out[0]==0.0 && out[1]==0.0 && out[2]==0.0) {
462                 VECCOPY(out, out_flip);
463         }
464         
465         Normalize(out);
466
467         if(outdir) {
468                 out[0] = outdir[0] * view + out[0] * (10-view);
469                 out[1] = outdir[1] * view + out[1] * (10-view);
470                 out[2] = outdir[2] * view + out[2] * (10-view);
471         }
472         
473         Normalize(out);
474 }
475
476 void do_draw_brush(SculptData *sd, SculptSession *ss, const BrushAction *a, const ListBase* active_verts)
477 {
478         float area_normal[3];
479         ActiveData *node= active_verts->first;
480
481         calc_area_normal(sd, area_normal, a, a->symm.out, active_verts);
482         
483         sculpt_axislock(sd, area_normal);
484         
485         while(node){
486                 float *co= ss->mvert[node->Index].co;
487                 
488                 const float val[3]= {co[0]+area_normal[0]*node->Fade*ss->cache->scale[0],
489                                      co[1]+area_normal[1]*node->Fade*ss->cache->scale[1],
490                                      co[2]+area_normal[2]*node->Fade*ss->cache->scale[2]};
491                                      
492                 sculpt_clip(ss->cache, co, val);
493                 
494                 node= node->next;
495         }
496 }
497
498 /* For the smooth brush, uses the neighboring vertices around vert to calculate
499    a smoothed location for vert. Skips corner vertices (used by only one
500    polygon.) */
501 vec3f neighbor_average(SculptSession *ss, const int vert)
502 {
503         int i, skip= -1, total=0;
504         IndexNode *node= ss->vertex_users[vert].first;
505         vec3f avg= {0,0,0};
506         char ncount= BLI_countlist(&ss->vertex_users[vert]);
507         MFace *f;
508                 
509         /* Don't modify corner vertices */
510         if(ncount==1) {
511                 VecCopyf(&avg.x, ss->mvert[vert].co);
512                 return avg;
513         }
514
515         while(node){
516                 f= &ss->mface[node->index];
517                 
518                 if(f->v4) {
519                         skip= (f->v1==vert?2:
520                                f->v2==vert?3:
521                                f->v3==vert?0:
522                                f->v4==vert?1:-1);
523                 }
524
525                 for(i=0; i<(f->v4?4:3); ++i) {
526                         if(i != skip && (ncount!=2 || BLI_countlist(&ss->vertex_users[(&f->v1)[i]]) <= 2)) {
527                                 VecAddf(&avg.x, &avg.x, ss->mvert[(&f->v1)[i]].co);
528                                 ++total;
529                         }
530                 }
531
532                 node= node->next;
533         }
534
535         if(total>0) {
536                 avg.x/= total;
537                 avg.y/= total;
538                 avg.z/= total;
539         }
540         else
541                 VecCopyf(&avg.x, ss->mvert[vert].co);
542
543         return avg;
544 }
545
546 void do_smooth_brush(SculptSession *ss, const BrushAction *a, const ListBase* active_verts)
547 {
548         ActiveData *node= active_verts->first;
549
550         while(node){
551                 float *co= ss->mvert[node->Index].co;
552                 const vec3f avg= neighbor_average(ss, node->Index);
553                 const float val[3]= {co[0]+(avg.x-co[0])*node->Fade,
554                                      co[1]+(avg.y-co[1])*node->Fade,
555                                      co[2]+(avg.z-co[2])*node->Fade};
556                 sculpt_clip(ss->cache, co, val);
557                 node= node->next;
558         }
559 }
560
561 void do_pinch_brush(SculptSession *ss, const BrushAction *a, const ListBase* active_verts)
562 {
563         ActiveData *node= active_verts->first;
564
565         while(node) {
566                 float *co= ss->mvert[node->Index].co;
567                 const float val[3]= {co[0]+(a->symm.center_3d[0]-co[0])*node->Fade,
568                                      co[1]+(a->symm.center_3d[1]-co[1])*node->Fade,
569                                      co[2]+(a->symm.center_3d[2]-co[2])*node->Fade};
570                 sculpt_clip(ss->cache, co, val);
571                 node= node->next;
572         }
573 }
574
575 void do_grab_brush(SculptData *sd, SculptSession *ss, BrushAction *a)
576 {
577         ActiveData *node= a->grab_active_verts[a->symm.index].first;
578         float add[3];
579         float grab_delta[3];
580         
581         VecCopyf(grab_delta, a->symm.grab_delta);
582         sculpt_axislock(sd, grab_delta);
583         
584         while(node) {
585                 float *co= ss->mvert[node->Index].co;
586                 
587                 VecCopyf(add, grab_delta);
588                 VecMulf(add, node->Fade);
589                 VecAddf(add, add, co);
590                 sculpt_clip(ss->cache, co, add);
591
592                 node= node->next;
593         }
594         
595 }
596
597 void do_layer_brush(SculptData *sd, SculptSession *ss, BrushAction *a, const ListBase *active_verts)
598 {
599         float area_normal[3];
600         ActiveData *node= active_verts->first;
601         const float bstr= brush_strength(sd, ss->cache, a);
602
603         calc_area_normal(sd, area_normal, a, NULL, active_verts);
604
605         while(node){
606                 float *disp= &a->layer_disps[node->Index];
607                 
608                 if((bstr > 0 && *disp < bstr) ||
609                   (bstr < 0 && *disp > bstr)) {
610                         float *co= ss->mvert[node->Index].co;
611                         
612                         *disp+= node->Fade;
613
614                         if(bstr < 0) {
615                                 if(*disp < bstr)
616                                         *disp = bstr;
617                         } else {
618                                 if(*disp > bstr)
619                                         *disp = bstr;
620                         }
621
622                         {
623                                 const float val[3]= {a->mesh_store[node->Index].x+area_normal[0] * *disp*ss->cache->scale[0],
624                                                      a->mesh_store[node->Index].y+area_normal[1] * *disp*ss->cache->scale[1],
625                                                      a->mesh_store[node->Index].z+area_normal[2] * *disp*ss->cache->scale[2]};
626                                 sculpt_clip(ss->cache, co, val);
627                         }
628                 }
629
630                 node= node->next;
631         }
632 }
633
634 void do_inflate_brush(SculptSession *ss, const BrushAction *a, const ListBase *active_verts)
635 {
636         ActiveData *node= active_verts->first;
637         float add[3];
638         
639         while(node) {
640                 float *co= ss->mvert[node->Index].co;
641                 short *no= ss->mvert[node->Index].no;
642
643                 add[0]= no[0]/ 32767.0f;
644                 add[1]= no[1]/ 32767.0f;
645                 add[2]= no[2]/ 32767.0f;
646                 VecMulf(add, node->Fade);
647                 add[0]*= ss->cache->scale[0];
648                 add[1]*= ss->cache->scale[1];
649                 add[2]*= ss->cache->scale[2];
650                 VecAddf(add, add, co);
651                 
652                 sculpt_clip(ss->cache, co, add);
653
654                 node= node->next;
655         }
656 }
657
658 void calc_flatten_center(SculptSession *ss, ActiveData *node, float co[3])
659 {
660         ActiveData *outer[FLATTEN_SAMPLE_SIZE];
661         int i;
662         
663         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
664                 outer[i] = node;
665                 
666         for(; node; node = node->next) {
667                 for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i) {
668                         if(node->dist > outer[i]->dist) {
669                                 outer[i] = node;
670                                 break;
671                         }
672                 }
673         }
674         
675         co[0] = co[1] = co[2] = 0.0f;
676         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
677                 VecAddf(co, co, ss->mvert[outer[i]->Index].co);
678         VecMulf(co, 1.0f / FLATTEN_SAMPLE_SIZE);
679 }
680
681 void do_flatten_brush(SculptData *sd, SculptSession *ss, const BrushAction *a, const ListBase *active_verts)
682 {
683         ActiveData *node= active_verts->first;
684         /* area_normal and cntr define the plane towards which vertices are squashed */
685         float area_normal[3];
686         float cntr[3];
687
688         calc_area_normal(sd, area_normal, a, a->symm.out, active_verts);
689         calc_flatten_center(ss, node, cntr);
690
691         while(node){
692                 float *co= ss->mvert[node->Index].co;
693                 float p1[3], sub1[3], sub2[3], intr[3], val[3];
694                 
695                 /* Find the intersection between squash-plane and vertex (along the area normal) */
696                 VecSubf(p1, co, area_normal);
697                 VecSubf(sub1, cntr, p1);
698                 VecSubf(sub2, co, p1);
699                 VecSubf(intr, co, p1);
700                 VecMulf(intr, Inpf(area_normal, sub1) / Inpf(area_normal, sub2));
701                 VecAddf(intr, intr, p1);
702                 
703                 VecSubf(val, intr, co);
704                 VecMulf(val, node->Fade);
705                 VecAddf(val, val, co);
706                 
707                 sculpt_clip(ss->cache, co, val);
708                 
709                 node= node->next;
710         }
711 }
712
713 /* Uses the brush curve control to find a strength value between 0 and 1 */
714 float curve_strength(CurveMapping *cumap, float p, const float len)
715 {
716         if(p > len) p= len;
717         return curvemapping_evaluateF(cumap, 0, p/len);
718 }
719
720 /* Uses symm to selectively flip any axis of a coordinate. */
721 void flip_coord(float co[3], const char symm)
722 {
723         if(symm & SCULPT_SYMM_X)
724                 co[0]= -co[0];
725         if(symm & SCULPT_SYMM_Y)
726                 co[1]= -co[1];
727         if(symm & SCULPT_SYMM_Z)
728                 co[2]= -co[2];
729 }
730
731 /* Use the warpfac field in MTex to store a rotation value for sculpt textures. Value is in degrees */
732 float sculpt_tex_angle(SculptData *sd)
733 {
734         if(sd->texact!=-1 && sd->mtex[sd->texact])
735                 return sd->mtex[sd->texact]->warpfac;
736         return 0;
737 }
738
739 void set_tex_angle(SculptData *sd, const float f)
740 {
741         if(sd->texact != -1 && sd->mtex[sd->texact])
742                 sd->mtex[sd->texact]->warpfac = f;
743 }
744         
745 float to_rad(const float deg)
746 {
747         return deg * (M_PI/180.0f);
748 }
749
750 float to_deg(const float rad)
751 {
752         return rad * (180.0f/M_PI);
753 }
754
755 /* Get a pixel from the texcache at (px, py) */
756 static unsigned char get_texcache_pixel(const SculptSession *ss, int px, int py)
757 {
758         unsigned *p;
759         p = ss->texcache + py * ss->texcache_w + px;
760         return ((unsigned char*)(p))[0];
761 }
762
763 static float get_texcache_pixel_bilinear(const SculptSession *ss, float u, float v)
764 {
765         int x, y, x2, y2;
766         const int tc_max = TC_SIZE - 1;
767         float urat, vrat, uopp;
768
769         if(u < 0) u = 0;
770         else if(u >= TC_SIZE) u = tc_max;
771         if(v < 0) v = 0;
772         else if(v >= TC_SIZE) v = tc_max;
773
774         x = floor(u);
775         y = floor(v);
776         x2 = x + 1;
777         y2 = y + 1;
778
779         if(x2 > TC_SIZE) x2 = tc_max;
780         if(y2 > TC_SIZE) y2 = tc_max;
781         
782         urat = u - x;
783         vrat = v - y;
784         uopp = 1 - urat;
785                 
786         return ((get_texcache_pixel(ss, x, y) * uopp +
787                  get_texcache_pixel(ss, x2, y) * urat) * (1 - vrat) + 
788                 (get_texcache_pixel(ss, x, y2) * uopp +
789                  get_texcache_pixel(ss, x2, y2) * urat) * vrat) / 255.0;
790 }
791
792 /* Return a multiplier for brush strength on a particular vertex. */
793 float tex_strength(SculptData *sd, BrushAction *a, float *point, const float len,const unsigned vindex)
794 {
795         SculptSession *ss= sd->session;
796         float avg= 1;
797
798         if(sd->texact==-1 || !sd->mtex[sd->texact])
799                 avg= 1;
800         else if(sd->texrept==SCULPTREPT_3D) {
801                 /* Get strength by feeding the vertex location directly
802                    into a texture */
803                 float jnk;
804                 const float factor= 0.01;
805                 MTex mtex;
806                 memset(&mtex,0,sizeof(MTex));
807                 mtex.tex= sd->mtex[sd->texact]->tex;
808                 mtex.projx= 1;
809                 mtex.projy= 2;
810                 mtex.projz= 3;
811                 VecCopyf(mtex.size, sd->mtex[sd->texact]->size);
812                 VecMulf(mtex.size, factor);
813                 if(!sd->texsep)
814                         mtex.size[1]= mtex.size[2]= mtex.size[0];
815                 
816                 externtex(&mtex,point,&avg,&jnk,&jnk,&jnk,&jnk);
817         }
818         else if(ss->texcache) {
819                 const float bsize= a->radius * 2;
820                 const float rot= to_rad(sculpt_tex_angle(sd)) + a->anchored_rot;
821                 int px, py;
822                 float flip[3], point_2d[2];
823
824                 /* If the active area is being applied for symmetry, flip it
825                    across the symmetry axis in order to project it. This insures
826                    that the brush texture will be oriented correctly. */
827                 VecCopyf(flip, point);
828                 flip_coord(flip, a->symm.index);
829                 projectf(ss, flip, point_2d);
830
831                 /* For Tile and Drag modes, get the 2D screen coordinates of the
832                    and scale them up or down to the texture size. */
833                 if(sd->texrept==SCULPTREPT_TILE) {
834                         const int sx= (const int)sd->mtex[sd->texact]->size[0];
835                         const int sy= (const int)sd->texsep ? sd->mtex[sd->texact]->size[1] : sx;
836                         
837                         float fx= point_2d[0];
838                         float fy= point_2d[1];
839                         
840                         float angle= atan2(fy, fx) - rot;
841                         float flen= sqrtf(fx*fx + fy*fy);
842                         
843                         if(rot<0.001 && rot>-0.001) {
844                                 px= point_2d[0];
845                                 py= point_2d[1];
846                         } else {
847                                 px= flen * cos(angle) + 2000;
848                                 py= flen * sin(angle) + 2000;
849                         }
850                         if(sx != 1)
851                                 px %= sx-1;
852                         if(sy != 1)
853                                 py %= sy-1;
854                         avg= get_texcache_pixel_bilinear(ss, TC_SIZE*px/sx, TC_SIZE*py/sy);
855                 } else {
856                         float fx= (point_2d[0] - ss->cache->mouse[0]) / bsize;
857                         float fy= (point_2d[1] - ss->cache->mouse[1]) / bsize;
858
859                         float angle= atan2(fy, fx) - rot;
860                         float flen= sqrtf(fx*fx + fy*fy);
861                         
862                         fx = flen * cos(angle) + 0.5;
863                         fy = flen * sin(angle) + 0.5;
864
865                         avg= get_texcache_pixel_bilinear(ss, fx * TC_SIZE, fy * TC_SIZE);
866                 }
867         }
868
869 //XXX   avg*= curve_strength(NULL, len, a->size_3d); /* Falloff curve */
870
871         return avg;
872 }
873
874 /* Mark area around the brush as damaged. projverts are marked if they are
875    inside the area and the damaged rectangle in 2D screen coordinates is 
876    added to damaged_rects. */
877 void sculpt_add_damaged_rect(SculptSession *ss, BrushAction *a)
878 {
879         short p[2];
880         RectNode *rn= MEM_mallocN(sizeof(RectNode),"RectNode");
881         const float radius = a->radius > a->prev_radius ? a->radius : a->prev_radius;
882         unsigned i;
883
884         /* Find center */
885         project(ss, a->symm.center_3d, p);
886         rn->r.xmin= p[0] - radius;
887         rn->r.ymin= p[1] - radius;
888         rn->r.xmax= p[0] + radius;
889         rn->r.ymax= p[1] + radius;
890
891         BLI_addtail(&ss->damaged_rects, rn);
892
893         /* Update insides */
894         for(i=0; i<ss->totvert; ++i) {
895                 if(!ss->projverts[i].inside) {
896                         if(ss->projverts[i].co[0] > rn->r.xmin && ss->projverts[i].co[1] > rn->r.ymin &&
897                            ss->projverts[i].co[0] < rn->r.xmax && ss->projverts[i].co[1] < rn->r.ymax) {
898                                 ss->projverts[i].inside= 1;
899                         }
900                 }
901                 // XXX: remember to fix this!
902                 // temporary pass
903                 ss->projverts[i].inside = 1;
904         }
905 }
906
907 /* Clears the depth buffer in each modified area. */
908 void sculpt_clear_damaged_areas(SculptSession *ss)
909 {
910         RectNode *rn= NULL;
911
912         for(rn = ss->damaged_rects.first; rn; rn = rn->next) {
913                 rcti clp = rn->r;
914                 rcti *win = NULL; /*XXX: &curarea->winrct; */
915                 
916                 clp.xmin += win->xmin;
917                 clp.xmax += win->xmin;
918                 clp.ymin += win->ymin;
919                 clp.ymax += win->ymin;
920                 
921                 if(clp.xmin < win->xmax && clp.xmax > win->xmin &&
922                    clp.ymin < win->ymax && clp.ymax > win->ymin) {
923                         if(clp.xmin < win->xmin) clp.xmin = win->xmin;
924                         if(clp.ymin < win->ymin) clp.ymin = win->ymin;
925                         if(clp.xmax > win->xmax) clp.xmax = win->xmax;
926                         if(clp.ymax > win->ymax) clp.ymax = win->ymax;
927
928                         glScissor(clp.xmin + 1, clp.ymin + 1,
929                                   clp.xmax - clp.xmin - 2,
930                                   clp.ymax - clp.ymin - 2);
931                 }
932                 
933                 glClear(GL_DEPTH_BUFFER_BIT);
934         }
935 }
936
937 void do_brush_action(SculptData *sd, StrokeCache *cache, BrushAction *a)
938 {
939         int i;
940         float av_dist;
941         ListBase active_verts={0,0};
942         ActiveData *adata= 0;
943         float *vert;
944         Mesh *me= NULL; /*XXX: get_mesh(OBACT); */
945         const float bstrength= brush_strength(sd, cache, a);
946         KeyBlock *keyblock= NULL; /*XXX: ob_get_keyblock(OBACT); */
947         SculptSession *ss = sd->session;
948         Brush *b = sd->brush;
949
950         sculpt_add_damaged_rect(ss, a);
951
952         /* Build a list of all vertices that are potentially within the brush's
953            area of influence. Only do this once for the grab brush. */
954         if((b->sculpt_tool != SCULPT_TOOL_GRAB) || a->firsttime) {
955                 for(i=0; i<ss->totvert; ++i) {
956                         /* Projverts.inside provides a rough bounding box */
957                         if(ss->multires || ss->projverts[i].inside) {
958                                 //vert= ss->vertexcosnos ? &ss->vertexcosnos[i*6] : a->verts[i].co;
959                                 vert= ss->mvert[i].co;
960                                 av_dist= VecLenf(a->symm.center_3d, vert);
961                                 if(av_dist < cache->radius) {
962                                         adata= (ActiveData*)MEM_mallocN(sizeof(ActiveData), "ActiveData");
963
964                                         adata->Index = i;
965                                         /* Fade is used to store the final strength at which the brush
966                                            should modify a particular vertex. */
967                                         adata->Fade= tex_strength(sd, a, vert, av_dist, i) * bstrength;
968                                         adata->dist = av_dist;
969
970                                         if(b->sculpt_tool == SCULPT_TOOL_GRAB && a->firsttime)
971                                                 BLI_addtail(&a->grab_active_verts[a->symm.index], adata);
972                                         else
973                                                 BLI_addtail(&active_verts, adata);
974                                 }
975                         }
976                 }
977         }
978
979         /* Only act if some verts are inside the brush area */
980         if(active_verts.first || (b->sculpt_tool == SCULPT_TOOL_GRAB && a->grab_active_verts[a->symm.index].first)) {
981                 /* Apply one type of brush action */
982                 switch(b->sculpt_tool){
983                 case SCULPT_TOOL_DRAW:
984                         do_draw_brush(sd, ss, a, &active_verts);
985                         break;
986                 case SCULPT_TOOL_SMOOTH:
987                         do_smooth_brush(ss, a, &active_verts);
988                         break;
989                 case SCULPT_TOOL_PINCH:
990                         do_pinch_brush(ss, a, &active_verts);
991                         break;
992                 case SCULPT_TOOL_INFLATE:
993                         do_inflate_brush(ss, a, &active_verts);
994                         break;
995                 case SCULPT_TOOL_GRAB:
996                         do_grab_brush(sd, ss, a);
997                         break;
998                 case SCULPT_TOOL_LAYER:
999                         do_layer_brush(sd, ss, a, &active_verts);
1000                         break;
1001                 case SCULPT_TOOL_FLATTEN:
1002                         do_flatten_brush(sd, ss, a, &active_verts);
1003                         break;
1004                 }
1005         
1006                 /* Copy the modified vertices from mesh to the active key */
1007                 if(keyblock && !ss->multires) {
1008                         float *co= keyblock->data;
1009                         if(co) {
1010                                 if(b->sculpt_tool == SCULPT_TOOL_GRAB)
1011                                         adata = a->grab_active_verts[a->symm.index].first;
1012                                 else
1013                                         adata = active_verts.first;
1014
1015                                 for(; adata; adata= adata->next)
1016                                         if(adata->Index < keyblock->totelem)
1017                                                 VecCopyf(&co[adata->Index*3], me->mvert[adata->Index].co);
1018                         }
1019                 }
1020
1021                 if(ss->vertexcosnos && !ss->multires)
1022                         BLI_freelistN(&active_verts);
1023                 else {
1024                         if(b->sculpt_tool != SCULPT_TOOL_GRAB)
1025                                 addlisttolist(&ss->damaged_verts, &active_verts);
1026                 }
1027         }
1028 }
1029
1030 /* Flip all the editdata across the axis/axes specified by symm. Used to
1031    calculate multiple modifications to the mesh when symmetry is enabled. */
1032 void calc_brushdata_symm(BrushAction *a, const char symm)
1033 {
1034         flip_coord(a->symm.center_3d, symm);
1035         flip_coord(a->symm.up, symm);
1036         flip_coord(a->symm.right, symm);
1037         flip_coord(a->symm.out, symm);
1038         
1039         a->symm.index= symm;
1040
1041         flip_coord(a->symm.grab_delta, symm);
1042 }
1043
1044 void do_symmetrical_brush_actions(SculptData *sd, StrokeCache *cache, BrushAction *a, short co[2], short pr_co[2])
1045 {
1046         const char symm = sd->flags & 7;
1047         BrushActionSymm orig;
1048         int i;
1049
1050         //init_brushaction(sd, a, co, pr_co);
1051         orig = a->symm;
1052         do_brush_action(sd, cache, a);
1053
1054         for(i = 1; i <= symm; ++i) {
1055                 if(symm & i && (symm != 5 || i != 3) && (symm != 6 || (i != 3 && i != 5))) {
1056                         // Restore the original symmetry data
1057                         a->symm = orig;
1058
1059                         calc_brushdata_symm(a, i);
1060                         do_brush_action(sd, cache, a);
1061                 }
1062         }
1063
1064         a->symm = orig;
1065 }
1066
1067 void add_face_normal(vec3f *norm, MVert *mvert, const MFace* face, float *fn)
1068 {
1069         vec3f c= {mvert[face->v1].co[0],mvert[face->v1].co[1],mvert[face->v1].co[2]};
1070         vec3f b= {mvert[face->v2].co[0],mvert[face->v2].co[1],mvert[face->v2].co[2]};
1071         vec3f a= {mvert[face->v3].co[0],mvert[face->v3].co[1],mvert[face->v3].co[2]};
1072         vec3f s1, s2;
1073         float final[3];
1074
1075         VecSubf(&s1.x,&a.x,&b.x);
1076         VecSubf(&s2.x,&c.x,&b.x);
1077
1078         final[0] = s1.y * s2.z - s1.z * s2.y;
1079         final[1] = s1.z * s2.x - s1.x * s2.z;
1080         final[2] = s1.x * s2.y - s1.y * s2.x;
1081
1082         if(fn)
1083                 VecCopyf(fn, final);
1084
1085         norm->x+= final[0];
1086         norm->y+= final[1];
1087         norm->z+= final[2];
1088 }
1089
1090 void update_damaged_vert(SculptSession *ss, ListBase *lb, BrushAction *a)
1091 {
1092         ActiveData *vert;
1093        
1094         for(vert= lb->first; vert; vert= vert->next) {
1095                 vec3f norm= {0,0,0};            
1096                 IndexNode *face= ss->vertex_users[vert->Index].first;
1097
1098                 while(face){
1099                         float *fn = NULL;
1100                         if(ss->face_normals)
1101                                 fn = &ss->face_normals[face->index*3];
1102                         add_face_normal(&norm, ss->mvert, &ss->mface[face->index], fn);
1103                         face= face->next;
1104                 }
1105                 Normalize(&norm.x);
1106                 
1107                 ss->mvert[vert->Index].no[0]=norm.x*32767;
1108                 ss->mvert[vert->Index].no[1]=norm.y*32767;
1109                 ss->mvert[vert->Index].no[2]=norm.z*32767;
1110         }
1111 }
1112
1113 void calc_damaged_verts(SculptSession *ss, BrushAction *a)
1114 {
1115         int i;
1116         
1117         for(i=0; i<8; ++i)
1118                 update_damaged_vert(ss, &a->grab_active_verts[i], a);
1119
1120         update_damaged_vert(ss, &ss->damaged_verts, a);
1121         BLI_freelistN(&ss->damaged_verts);
1122         ss->damaged_verts.first = ss->damaged_verts.last = NULL;
1123 }
1124
1125 void projverts_clear_inside(SculptSession *ss)
1126 {
1127         int i;
1128         for(i = 0; i < ss->totvert; ++i)
1129                 ss->projverts[i].inside = 0;
1130 }
1131
1132 void sculptmode_update_tex(SculptData *sd)
1133 {
1134         SculptSession *ss= sd->session;
1135         MTex *mtex = sd->mtex[sd->texact];
1136         TexResult texres = {0};
1137         float x, y, step=2.0/TC_SIZE, co[3];
1138         int hasrgb, ix, iy;
1139         
1140         /* Skip Default brush shape and non-textures */
1141         if(sd->texact == -1 || !sd->mtex[sd->texact]) return;
1142
1143         if(ss->texcache) {
1144                 MEM_freeN(ss->texcache);
1145                 ss->texcache= NULL;
1146         }
1147         
1148         ss->texcache_w = ss->texcache_h = TC_SIZE;
1149         ss->texcache = MEM_callocN(sizeof(int) * ss->texcache_w * ss->texcache_h, "Sculpt Texture cache");
1150         
1151         if(mtex && mtex->tex) {
1152                 BKE_image_get_ibuf(sd->mtex[sd->texact]->tex->ima, NULL);
1153                 
1154                 /*do normalized cannonical view coords for texture*/
1155                 for (y=-1.0, iy=0; iy<TC_SIZE; iy++, y += step) {
1156                         for (x=-1.0, ix=0; ix<TC_SIZE; ix++, x += step) {
1157                                 co[0]= x;
1158                                 co[1]= y;
1159                                 co[2]= 0.0f;
1160                                 
1161                                 /* This is copied from displace modifier code */
1162                                 hasrgb = multitex_ext(mtex->tex, co, NULL, NULL, 1, &texres);
1163                         
1164                                 /* if the texture gave an RGB value, we assume it didn't give a valid
1165                                  * intensity, so calculate one (formula from do_material_tex).
1166                                  * if the texture didn't give an RGB value, copy the intensity across
1167                                  */
1168                                 if(hasrgb & TEX_RGB)
1169                                         texres.tin = (0.35 * texres.tr + 0.45 *
1170                                                       texres.tg + 0.2 * texres.tb);
1171
1172                                 texres.tin = texres.tin * 255.0;
1173                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4] = (char)texres.tin;
1174                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+1] = (char)texres.tin;
1175                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+2] = (char)texres.tin;
1176                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+3] = (char)texres.tin;
1177                         }
1178                 }
1179         }
1180 }
1181
1182 /* pr_mouse is only used for the grab brush, can be NULL otherwise */
1183 static void init_brushaction(SculptData *sd, BrushAction *a, short *mouse, short *pr_mouse)
1184 {
1185         SculptSession *ss = sd->session;
1186         Brush *b = sd->brush;
1187         Object *ob = NULL; /* XXX */
1188         const float mouse_depth = 0; // XXX: get_depth(mouse[0], mouse[1]);
1189         float brush_edge_loc[3], zero_loc[3], oldloc[3];
1190         ModifierData *md;
1191         int i;
1192         const int anchored = sd->brush->flag & BRUSH_ANCHORED;
1193         short orig_mouse[2], dx=0, dy=0;
1194         float size = brush_size(sd, b);
1195
1196         a->symm.index = 0;
1197
1198         if(a->firsttime) 
1199                 a->depth = mouse_depth;
1200         
1201         /* Convert the location and size of the brush to
1202            modelspace coords */
1203         if(a->firsttime || !anchored) {
1204                 unproject(ss, a->symm.center_3d, mouse[0], mouse[1], mouse_depth);
1205                 /*a->mouse[0] = mouse[0];
1206                   a->mouse[1] = mouse[1];*/
1207         }
1208  
1209         if(anchored) {
1210                 project(ss, a->symm.center_3d, orig_mouse);
1211                 dx = mouse[0] - orig_mouse[0];
1212                 dy = mouse[1] - orig_mouse[1];
1213         }
1214  
1215         if(anchored) {
1216                 unproject(ss, brush_edge_loc, mouse[0], mouse[1], a->depth);
1217                 a->anchored_rot = atan2(dy, dx);
1218         }
1219         else
1220                 unproject(ss, brush_edge_loc, mouse[0] + size, mouse[1], mouse_depth);
1221  
1222         //a->size_3d = VecLenf(a->symm.center_3d, brush_edge_loc);
1223
1224         a->prev_radius = a->radius;
1225
1226         if(anchored)
1227                 a->radius = sqrt(dx*dx + dy*dy);
1228         else
1229                 a->radius = size;
1230
1231         /* Set the pivot to allow the model to rotate around the center of the brush */
1232         /*XXX: if(get_depth(mouse[0],mouse[1]) < 1.0)
1233           VecCopyf(sd->pivot, a->symm.center_3d); */
1234
1235         /* Now project the Up, Right, and Out normals from view to model coords */
1236         unproject(ss, zero_loc, 0, 0, 0);
1237         unproject(ss, a->symm.up, 0, -1, 0);
1238         unproject(ss, a->symm.right, 1, 0, 0);
1239         unproject(ss, a->symm.out, 0, 0, -1);
1240         VecSubf(a->symm.up, a->symm.up, zero_loc);
1241         VecSubf(a->symm.right, a->symm.right, zero_loc);
1242         VecSubf(a->symm.out, a->symm.out, zero_loc);
1243         Normalize(a->symm.up);
1244         Normalize(a->symm.right);
1245         Normalize(a->symm.out);
1246         
1247
1248
1249         if(b->sculpt_tool == SCULPT_TOOL_GRAB) {
1250                 float gcenter[3];
1251
1252                 /* Find the delta */
1253                 unproject(ss, gcenter, mouse[0], mouse[1], a->depth);
1254                 unproject(ss, oldloc, pr_mouse[0], pr_mouse[1], a->depth);
1255                 VecSubf(a->symm.grab_delta, gcenter, oldloc);
1256         }
1257         else if(b->sculpt_tool == SCULPT_TOOL_LAYER) {
1258                 if(!a->layer_disps)
1259                         a->layer_disps= MEM_callocN(sizeof(float)*ss->totvert,"Layer disps");
1260         }
1261
1262         if(b->sculpt_tool == SCULPT_TOOL_LAYER || anchored) {
1263                 unsigned i;
1264  
1265                 if(!a->mesh_store) {
1266                         a->mesh_store= MEM_mallocN(sizeof(vec3f) * ss->totvert, "Sculpt mesh store");
1267                         for(i = 0; i < ss->totvert; ++i)
1268                                 VecCopyf(&a->mesh_store[i].x, ss->mvert[i].co);
1269                 }
1270
1271                 if(anchored && a->layer_disps)
1272                         memset(a->layer_disps, 0, sizeof(float) * ss->totvert);
1273
1274                 if(anchored && !a->orig_norms) {
1275                         a->orig_norms= MEM_mallocN(sizeof(short) * 3 * ss->totvert, "Sculpt orig norm");
1276                         for(i = 0; i < ss->totvert; ++i) {
1277                                 a->orig_norms[i][0] = ss->mvert[i].no[0];
1278                                 a->orig_norms[i][1] = ss->mvert[i].no[1];
1279                                 a->orig_norms[i][2] = ss->mvert[i].no[2];
1280                         }
1281                 }
1282         }
1283 }
1284
1285 /* XXX: Used anywhere?
1286 void sculptmode_set_strength(const int delta)
1287 {
1288         int val = sculptmode_brush()->strength + delta;
1289         if(val < 1) val = 1;
1290         if(val > 100) val = 100;
1291         sculptmode_brush()->strength= val;
1292 }*/
1293
1294 /* XXX: haven't brought in the radial control files, not sure where to put them. Note that all the paint modes should have access to radial control! */
1295 #if 0
1296 static void sculpt_radialcontrol_callback(const int mode, const int val)
1297 {
1298         SculptSession *ss = sculpt_session();
1299         BrushData *br = sculptmode_brush();
1300
1301         if(mode == RADIALCONTROL_SIZE)
1302                 br->size = val;
1303         else if(mode == RADIALCONTROL_STRENGTH)
1304                 br->strength = val;
1305         else if(mode == RADIALCONTROL_ROTATION)
1306                 set_tex_angle(val);
1307
1308         ss->radialcontrol = NULL;
1309 }
1310
1311 /* Returns GL handle to brush texture */
1312 static GLuint sculpt_radialcontrol_calctex()
1313 {
1314         SculptData *sd= sculpt_data();
1315         SculptSession *ss= sculpt_session();
1316         int i, j;
1317         const int tsz = TC_SIZE;
1318         float *texdata= MEM_mallocN(sizeof(float)*tsz*tsz, "Brush preview");
1319         GLuint tex;
1320
1321         if(sd->texrept!=SCULPTREPT_3D)
1322                 sculptmode_update_tex();
1323         for(i=0; i<tsz; ++i)
1324                 for(j=0; j<tsz; ++j) {
1325                         float magn= sqrt(pow(i-tsz/2,2)+pow(j-tsz/2,2));
1326                         if(sd->texfade)
1327                                 texdata[i*tsz+j]= curve_strength(magn,tsz/2);
1328                         else
1329                                 texdata[i*tsz+j]= magn < tsz/2 ? 1 : 0;
1330                 }
1331         if(sd->texact != -1 && ss->texcache) {
1332                 for(i=0; i<tsz; ++i)
1333                         for(j=0; j<tsz; ++j) {
1334                                 const int col= ss->texcache[i*tsz+j];
1335                                 texdata[i*tsz+j]*= (((char*)&col)[0]+((char*)&col)[1]+((char*)&col)[2])/3.0f/255.0f;
1336                         }
1337         }
1338                 
1339         glGenTextures(1, &tex);
1340         glBindTexture(GL_TEXTURE_2D, tex);
1341         glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, tsz, tsz, 0, GL_ALPHA, GL_FLOAT, texdata);
1342         MEM_freeN(texdata);
1343
1344         return tex;
1345 }
1346
1347 void sculpt_radialcontrol_start(int mode)
1348 {
1349         SculptData *sd = sculpt_data();
1350         SculptSession *ss = sculpt_session();
1351         BrushData *br = sculptmode_brush();
1352         int orig=1, max=100;
1353
1354         if(mode == RADIALCONTROL_SIZE) {
1355                 orig = br->size;
1356                 max = 200;
1357         }
1358         else if(mode == RADIALCONTROL_STRENGTH) {
1359                 orig = br->strength;
1360                 max = 100;
1361         }
1362         else if(mode == RADIALCONTROL_ROTATION) {
1363                 if(sd->texact!=-1 && sd->mtex[sd->texact]) {
1364                         orig = sculpt_tex_angle();
1365                         max = 360;
1366                 }
1367                 else
1368                         mode = RADIALCONTROL_NONE;
1369         }
1370
1371         if(mode != RADIALCONTROL_NONE) {
1372                 ss->radialcontrol= radialcontrol_start(mode, sculpt_radialcontrol_callback, orig, max,
1373                                                        sculpt_radialcontrol_calctex());
1374         }
1375 }
1376 #endif
1377
1378 /* XXX: drawing code to go elsewhere!
1379 void sculpt_paint_brush(char clear)
1380 {
1381         if(sculpt_data()->flags & SCULPT_SCULPT_TOOL_DRAW) {
1382                 static short mvalo[2];
1383                 short mval[2];
1384                 const short rad= sculptmode_brush()->size;
1385
1386                 getmouseco_areawin(mval);
1387                 
1388                 persp(PERSP_WIN);
1389                 if(clear)
1390                         fdrawXORcirc(mval[0], mval[1], rad);
1391                 else
1392                         draw_sel_circle(mval, mvalo, rad, rad, 0);
1393                 
1394                 mvalo[0]= mval[0];
1395                 mvalo[1]= mval[1];
1396         }
1397 }
1398 */
1399
1400 void sculptmode_selectbrush_menu(void)
1401 {
1402         /* XXX: I guess menus belong elsewhere too?
1403
1404         SculptData *sd= sculpt_data();
1405         int val;
1406         
1407         pupmenu_set_active(sd->brush_type);
1408         
1409         val= pupmenu("Select Brush%t|Draw|Smooth|Pinch|Inflate|Grab|Layer|Flatten");
1410
1411         if(val>0) {
1412                 sd->brush_type= val;
1413
1414                 BIF_undo_push("Brush type");
1415                 
1416                 allqueue(REDRAWVIEW3D, 1);
1417                 allqueue(REDRAWBUTSEDIT, 1);
1418         }*/
1419 }
1420
1421 void sculptmode_update_all_projverts(SculptSession *ss)
1422 {
1423         unsigned i;
1424
1425         if(!ss->projverts)
1426                 ss->projverts = MEM_mallocN(sizeof(ProjVert)*ss->totvert,"ProjVerts");
1427
1428         for(i=0; i<ss->totvert; ++i) {
1429                 project(ss, ss->vertexcosnos ? &ss->vertexcosnos[i * 6] : ss->mvert[i].co, ss->projverts[i].co);
1430                 ss->projverts[i].inside= 0;
1431         }
1432 }
1433
1434 /* Checks whether full update mode (slower) needs to be used to work with modifiers */
1435 char sculpt_modifiers_active(Object *ob)
1436 {
1437         ModifierData *md;
1438         
1439         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1440                 if(md->mode & eModifierMode_Realtime && md->type != eModifierType_Multires)
1441                         return 1;
1442         }
1443         
1444         return 0;
1445 }
1446
1447 /* Sculpt mode handles multires differently from regular meshes, but only if
1448    it's the last modifier on the stack and it is not on the first level */
1449 struct MultiresModifierData *sculpt_multires_active(Object *ob)
1450 {
1451         ModifierData *md;
1452         
1453         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1454                 if(md->type == eModifierType_Multires && !md->next) {
1455                         MultiresModifierData *mmd = (MultiresModifierData*)md;
1456                         if(mmd->lvl != 1)
1457                                 return mmd;
1458                 }
1459         }
1460
1461         return NULL;
1462 }
1463
1464 static void sculpt_update_mesh_elements(SculptSession *ss, Object *ob)
1465 {
1466         if(sculpt_multires_active(ob)) {
1467                 DerivedMesh *dm = mesh_get_derived_final(NULL, ob, CD_MASK_BAREMESH); /* XXX scene=? */
1468                 ss->multires = 1;
1469                 ss->totvert = dm->getNumVerts(dm);
1470                 ss->totface = dm->getNumFaces(dm);
1471                 ss->mvert = dm->getVertDataArray(dm, CD_MVERT);
1472                 ss->mface = dm->getFaceDataArray(dm, CD_MFACE);
1473                 ss->face_normals = dm->getFaceDataArray(dm, CD_NORMAL);
1474         }
1475         else {
1476                 Mesh *me = get_mesh(ob);
1477                 ss->multires = 0;
1478                 ss->totvert = me->totvert;
1479                 ss->totface = me->totface;
1480                 ss->mvert = me->mvert;
1481                 ss->mface = me->mface;
1482                 ss->face_normals = NULL;
1483         }
1484 }
1485
1486 /* XXX: lots of drawing code (partial redraw), has to go elsewhere */
1487 #if 0
1488 void sculptmode_draw_wires(SculptSession *ss, int only_damaged)
1489 {
1490         Mesh *me = get_mesh(OBACT);
1491         int i;
1492
1493         bglPolygonOffset(1.0);
1494         glDepthMask(0);
1495         BIF_ThemeColor((OBACT==OBACT)?TH_ACTIVE:TH_SELECT);
1496
1497         for(i=0; i<me->totedge; i++) {
1498                 MEdge *med= &me->medge[i];
1499
1500                 if((!only_damaged || (ss->projverts[med->v1].inside || ss->projverts[med->v2].inside)) &&
1501                    (med->flag & ME_EDGEDRAW)) {
1502                         glDrawElements(GL_LINES, 2, GL_UNSIGNED_INT, &med->v1);
1503                 }
1504         }
1505
1506         glDepthMask(1);
1507         bglPolygonOffset(0.0);
1508 }
1509
1510 void sculptmode_draw_mesh(int only_damaged) 
1511 {
1512         int i, j, dt, drawCurrentMat = 1, matnr= -1;
1513         SculptSession *ss = sculpt_session();
1514
1515         sculpt_update_mesh_elements(ss, OBACT);
1516
1517         persp(PERSP_VIEW);
1518         mymultmatrix(OBACT->obmat);
1519         glEnable(GL_DEPTH_TEST);
1520         glEnable(GL_LIGHTING);
1521         /* XXX: GPU_set_object_materials(G.scene, OBACT, 0, NULL); */
1522         glEnable(GL_CULL_FACE);
1523
1524         glShadeModel(GL_SMOOTH);
1525
1526         glVertexPointer(3, GL_FLOAT, sizeof(MVert), &ss->mvert[0].co);
1527         glNormalPointer(GL_SHORT, sizeof(MVert), &ss->mvert[0].no);
1528
1529         dt= MIN2(G.vd->drawtype, OBACT->dt);
1530         if(dt==OB_WIRE)
1531                 glColorMask(0,0,0,0);
1532
1533         for(i=0; i<ss->totface; ++i) {
1534                 MFace *f= &ss->mface[i];
1535                 char inside= 0;
1536                 int new_matnr= f->mat_nr + 1;
1537                 
1538                 if(new_matnr != matnr)
1539                         drawCurrentMat= GPU_enable_material(matnr = new_matnr, NULL);
1540                 
1541                 /* If only_damaged!=0, only draw faces that are partially
1542                    inside the area(s) modified by the brush */
1543                 if(only_damaged) {
1544                         for(j=0; j<(f->v4?4:3); ++j) {
1545                                 if(ss->projverts[*((&f->v1)+j)].inside) {
1546                                         inside= 1;
1547                                         break;
1548                                 }
1549                         }
1550                 }
1551                 else
1552                         inside= 1;
1553                         
1554                 if(inside && drawCurrentMat)
1555                         glDrawElements(f->v4?GL_QUADS:GL_TRIANGLES, f->v4?4:3, GL_UNSIGNED_INT, &f->v1);
1556         }
1557
1558         glDisable(GL_CULL_FACE);
1559         glDisable(GL_LIGHTING);
1560         glColorMask(1,1,1,1);
1561
1562         if(dt==OB_WIRE || (OBACT->dtx & OB_DRAWWIRE))
1563                 sculptmode_draw_wires(ss, only_damaged);
1564
1565         glDisable(GL_DEPTH_TEST);
1566 }
1567 #endif
1568
1569
1570 static void sculpt_undo_push(SculptData *sd)
1571 {
1572         /*XXX: switch(sd->brush->sculpt_tool) {
1573         case SCULPT_TOOL_DRAW:
1574                 BIF_undo_push("Draw Brush"); break;
1575         case SCULPT_TOOL_SMOOTH:
1576                 BIF_undo_push("Smooth Brush"); break;
1577         case SCULPT_TOOL_PINCH:
1578                 BIF_undo_push("Pinch Brush"); break;
1579         case SCULPT_TOOL_INFLATE:
1580                 BIF_undo_push("Inflate Brush"); break;
1581         case SCULPT_TOOL_GRAB:
1582                 BIF_undo_push("Grab Brush"); break;
1583         case SCULPT_TOOL_LAYER:
1584                 BIF_undo_push("Layer Brush"); break;
1585         case SCULPT_TOOL_FLATTEN:
1586                 BIF_undo_push("Flatten Brush"); break;
1587         default:
1588                 BIF_undo_push("Sculpting"); break;
1589                 }*/
1590 }
1591
1592 void sculptmode_correct_state(SculptData *sd)
1593 {
1594         if(!sd->session)
1595                 sculpt_init_session(sd);
1596
1597         glEnableClientState(GL_VERTEX_ARRAY);
1598         glEnableClientState(GL_NORMAL_ARRAY);
1599         
1600         if(!sd->session->vertex_users) calc_vertex_users(sd->session);
1601 }
1602
1603 /**** Operator for applying a stroke (various attributes including mouse path)
1604       using the current brush. ****/
1605 static int sculpt_brush_stroke_poll(bContext *C)
1606 {
1607         // XXX: More to check for, of course
1608
1609         return G.f & G_SCULPTMODE;
1610 }
1611
1612 /* This is temporary, matrices should be data in operator for exec */
1613 static int sculpt_load_mats(bContext *C, bglMats *mats)
1614 {
1615         View3D *v3d = ((View3D*)CTX_wm_area(C)->spacedata.first);
1616         ARegion *ar = CTX_wm_region(C);
1617         Object *ob= CTX_data_active_object(C);
1618         float cpy[4][4];
1619         int i, j;
1620
1621         view3d_operator_needs_opengl(C);
1622
1623         Mat4MulMat4(cpy, v3d->viewmat, ob->obmat);
1624
1625         for(i = 0; i < 4; ++i) {
1626                 for(j = 0; j < 4; ++j) {
1627                         mats->projection[i*4+j] = v3d->winmat[i][j];
1628                         mats->modelview[i*4+j] = cpy[i][j];
1629                 }
1630         }
1631
1632         mats->viewport[0] = ar->winrct.xmin;
1633         mats->viewport[1] = ar->winrct.ymin;
1634         mats->viewport[2] = ar->winx;
1635         mats->viewport[3] = ar->winy;   
1636 }
1637
1638 /* Initialize the stroke cache invariants from operator properties */
1639 static int sculpt_update_cache_invariants(StrokeCache *cache, wmOperator *op, Object *ob)
1640 {
1641
1642         memset(cache, 0, sizeof(StrokeCache));
1643
1644         cache->radius = RNA_float_get(op->ptr, "radius");
1645         RNA_float_get_array(op->ptr, "scale", cache->scale);
1646         cache->flag = RNA_int_get(op->ptr, "flag");
1647         RNA_float_get_array(op->ptr, "clip_tolerance", cache->clip_tolerance);
1648         RNA_int_get_array(op->ptr, "mouse", cache->mouse);
1649 }
1650
1651 /* Initialize the stroke cache variants from operator properties */
1652 static int sculpt_update_cache_variants(StrokeCache *cache, PointerRNA *ptr)
1653 {
1654         cache->flip = RNA_boolean_get(ptr, "flip");
1655 }
1656
1657 /* Initialize stroke operator properties */
1658 static int sculpt_brush_stroke_init(bContext *C, wmOperator *op, wmEvent *event, SculptSession *ss)
1659 {
1660         SculptData *sd = &CTX_data_scene(C)->sculptdata;
1661         Object *ob= CTX_data_active_object(C);
1662         ModifierData *md;
1663         float brush_center[3], brush_edge[3];
1664         float depth = get_depth(C, event->x, event->y);
1665         float size = brush_size(sd, sd->brush);
1666         float scale[3], clip_tolerance[3] = {0,0,0};
1667         int mouse[2], flag = 0;
1668
1669         unproject(ss, brush_center, event->x, event->y, depth);
1670         unproject(ss, brush_edge, event->x + size, event->y, depth);
1671
1672         RNA_float_set(op->ptr, "radius", VecLenf(brush_center, brush_edge));
1673
1674         /* Set scaling adjustment */
1675         scale[0] = 1.0f / ob->size[0];
1676         scale[1] = 1.0f / ob->size[1];
1677         scale[2] = 1.0f / ob->size[2];
1678         RNA_float_set_array(op->ptr, "scale", scale);
1679
1680         /* Initialize mirror modifier clipping */
1681         for(md= ob->modifiers.first; md; md= md->next) {
1682                 if(md->type==eModifierType_Mirror && (md->mode & eModifierMode_Realtime)) {
1683                         const MirrorModifierData *mmd = (MirrorModifierData*) md;
1684                         
1685                         /* Mark each axis that needs clipping along with its tolerance */
1686                         if(mmd->flag & MOD_MIR_CLIPPING) {
1687                                 flag |= CLIP_X << mmd->axis;
1688                                 if(mmd->tolerance > clip_tolerance[mmd->axis])
1689                                         clip_tolerance[mmd->axis] = mmd->tolerance;
1690                         }
1691                 }
1692         }
1693         RNA_int_set(op->ptr, "flag", flag);
1694         RNA_float_set_array(op->ptr, "clip_tolerance", clip_tolerance);
1695
1696         mouse[0] = event->x;
1697         mouse[1] = event->y;
1698         RNA_int_set_array(op->ptr, "mouse", mouse);
1699
1700         sculpt_update_cache_invariants(ss->cache, op, ob);
1701 }
1702
1703 static int sculpt_brush_stroke_invoke(bContext *C, wmOperator *op, wmEvent *event)
1704 {
1705         SculptData *sd = &CTX_data_scene(C)->sculptdata;
1706         Object *ob= CTX_data_active_object(C);
1707         Mesh *me = get_mesh(ob);
1708         ARegion *ar = CTX_wm_region(C);
1709
1710         // XXX: temporary, much of sculptsession data should be in rna properties
1711         sd->session = MEM_callocN(sizeof(SculptSession), "test sculpt session");
1712         sd->session->mvert = me->mvert;
1713         sd->session->totvert = me->totvert;
1714         sd->session->mats = MEM_callocN(sizeof(bglMats), "test sculpt mats");
1715         sd->session->cache = MEM_callocN(sizeof(StrokeCache), "stroke cache");
1716         
1717         // XXX: temporary matrix stuff
1718         sculpt_load_mats(C, sd->session->mats);
1719
1720         sculptmode_update_all_projverts(sd->session);
1721
1722         sculpt_brush_stroke_init(C, op, event, sd->session);
1723
1724         /* add modal handler */
1725         WM_event_add_modal_handler(C, &CTX_wm_window(C)->handlers, op);
1726         
1727         return OPERATOR_RUNNING_MODAL;
1728 }
1729
1730 /* Temporary, most of brush action will become rna properties */
1731 static void sculpt_action_init(BrushAction *a)
1732 {
1733         memset(a, 0, sizeof(BrushAction));
1734 }
1735
1736 static int sculpt_brush_stroke_modal(bContext *C, wmOperator *op, wmEvent *event)
1737 {
1738         PointerRNA itemptr;
1739         SculptData *sd = &CTX_data_scene(C)->sculptdata;
1740         BrushAction a;
1741         Object *ob= CTX_data_active_object(C);
1742         ARegion *ar = CTX_wm_region(C);
1743
1744         sculpt_action_init(&a);
1745         unproject(sd->session, a.symm.center_3d, event->x, event->y, get_depth(C, event->x, event->y));
1746
1747         /* Add to stroke */
1748         RNA_collection_add(op->ptr, "stroke", &itemptr);
1749         RNA_float_set_array(&itemptr, "location", a.symm.center_3d);
1750         RNA_boolean_set(&itemptr, "flip", event->shift);
1751         sculpt_update_cache_variants(sd->session->cache, &itemptr);
1752
1753         do_symmetrical_brush_actions(&CTX_data_scene(C)->sculptdata, sd->session->cache, &a, NULL, NULL);
1754         //calc_damaged_verts(sd->session, &a);
1755         BLI_freelistN(&sd->session->damaged_verts);
1756
1757         DAG_object_flush_update(CTX_data_scene(C), ob, OB_RECALC_DATA);
1758         ED_region_tag_redraw(ar);
1759
1760         /* Finished */
1761         if(event->type == LEFTMOUSE && event->val == 0) {
1762                 View3D *v3d = ((View3D*)CTX_wm_area(C)->spacedata.first);
1763                 if(v3d->depths)
1764                         v3d->depths->damaged= 1;
1765
1766                 return OPERATOR_FINISHED;
1767         }
1768
1769         return OPERATOR_RUNNING_MODAL;
1770 }
1771
1772 static int sculpt_brush_stroke_exec(bContext *C, wmOperator *op)
1773 {
1774         BrushAction a;
1775         Object *ob= CTX_data_active_object(C);
1776         ARegion *ar = CTX_wm_region(C);
1777         SculptData *sd = &CTX_data_scene(C)->sculptdata;
1778
1779         sculpt_update_cache_invariants(sd->session->cache, op, ob);
1780
1781         RNA_BEGIN(op->ptr, itemptr, "stroke") {
1782                 float loc[3];
1783
1784                 sculpt_action_init(&a);         
1785                 RNA_float_get_array(&itemptr, "location", a.symm.center_3d);
1786                 sculpt_update_cache_variants(sd->session->cache, &itemptr);
1787
1788                 do_symmetrical_brush_actions(sd, sd->session->cache, &a, NULL, NULL);
1789                 BLI_freelistN(&sd->session->damaged_verts);
1790         }
1791         RNA_END;
1792
1793         DAG_object_flush_update(CTX_data_scene(C), ob, OB_RECALC_DATA);
1794         ED_region_tag_redraw(ar);
1795
1796         return OPERATOR_FINISHED;
1797 }
1798
1799 void SCULPT_OT_brush_stroke(wmOperatorType *ot)
1800 {
1801         PropertyRNA *prop;
1802         float vec3f_def[] = {0,0,0};
1803         int vec2i_def[] = {0,0};
1804
1805         ot->flag |= OPTYPE_REGISTER;
1806
1807         /* identifiers */
1808         ot->name= "Sculpt Mode";
1809         ot->idname= "SCULPT_OT_brush_stroke";
1810         
1811         /* api callbacks */
1812         ot->invoke= sculpt_brush_stroke_invoke;
1813         ot->modal= sculpt_brush_stroke_modal;
1814         ot->exec= sculpt_brush_stroke_exec;
1815         ot->poll= sculpt_brush_stroke_poll;
1816
1817         /* properties */
1818         prop= RNA_def_property(ot->srna, "stroke", PROP_COLLECTION, PROP_NONE);
1819         RNA_def_property_struct_runtime(prop, &RNA_OperatorStrokeElement);
1820
1821         /* Brush radius measured in object space, projected from the brush setting in pixels */
1822         prop= RNA_def_property(ot->srna, "radius", PROP_FLOAT, PROP_NONE);
1823
1824         /* If the object has a scaling factor, brushes also need to be scaled
1825            to work as expected. */
1826         prop= RNA_def_property(ot->srna, "scale", PROP_FLOAT, PROP_VECTOR);
1827         RNA_def_property_array(prop, 3);
1828         RNA_def_property_float_array_default(prop, vec3f_def);
1829
1830         prop= RNA_def_property(ot->srna, "flag", PROP_INT, PROP_NONE);
1831
1832         prop= RNA_def_property(ot->srna, "clip_tolerance", PROP_FLOAT, PROP_VECTOR);
1833         RNA_def_property_array(prop, 3);
1834         RNA_def_property_float_array_default(prop, vec3f_def);
1835
1836         /* The initial 2D location of the mouse */
1837         prop= RNA_def_property(ot->srna, "mouse", PROP_INT, PROP_VECTOR);
1838         RNA_def_property_array(prop, 2);
1839         RNA_def_property_int_array_default(prop, vec2i_def);
1840 }
1841
1842 /**** Toggle operator for turning sculpt mode on or off ****/
1843
1844 static int sculpt_toggle_mode(bContext *C, wmOperator *op)
1845 {
1846         if(G.f & G_SCULPTMODE) {
1847                 /* Leave sculptmode */
1848                 G.f &= ~G_SCULPTMODE;
1849         }
1850         else {
1851                 /* Enter sculptmode */
1852
1853                 G.f |= G_SCULPTMODE;
1854
1855                 /* Needed for testing, if there's no brush then create one */
1856                 CTX_data_scene(C)->sculptdata.brush = add_brush("test brush");
1857         }
1858
1859         return OPERATOR_FINISHED;
1860 }
1861
1862 void SCULPT_OT_toggle_mode(wmOperatorType *ot)
1863 {
1864         /* identifiers */
1865         ot->name= "Sculpt Mode";
1866         ot->idname= "SCULPT_OT_toggle_mode";
1867         
1868         /* api callbacks */
1869         ot->exec= sculpt_toggle_mode;
1870         ot->poll= ED_operator_object_active;
1871         
1872 }
1873
1874 void ED_operatortypes_sculpt()
1875 {
1876         WM_operatortype_append(SCULPT_OT_brush_stroke);
1877         WM_operatortype_append(SCULPT_OT_toggle_mode);
1878 }
1879
1880 void sculpt(SculptData *sd)
1881 {
1882         SculptSession *ss= sd->session;
1883         Object *ob= NULL; /*XXX */
1884         Mesh *me;
1885         MultiresModifierData *mmd = NULL;
1886         /* lastSigMouse is for the rake, to store the last place the mouse movement was significant */
1887         short mouse[2], mvalo[2], lastSigMouse[2],firsttime=1, mousebut;
1888         short modifier_calculations= 0;
1889         BrushAction *a = MEM_callocN(sizeof(BrushAction), "brush action");
1890         short spacing= 32000;
1891         int scissor_box[4];
1892         float offsetRot;
1893         int smooth_stroke = 0, i;
1894         int anchored, rake = 0 /* XXX: rake = ? */;
1895
1896         /* XXX: checking that sculpting is allowed
1897         if(!(G.f & G_SCULPTMODE) || G.obedit || !ob || ob->id.lib || !get_mesh(ob) || (get_mesh(ob)->totface == 0))
1898                 return;
1899         if(!(ob->lay & G.vd->lay))
1900                 error("Active object is not in this layer");
1901         if(ob_get_keyblock(ob)) {
1902                 if(!(ob->shapeflag & OB_SHAPE_LOCK)) {
1903                         error("Cannot sculpt on unlocked shape key");
1904                         return;
1905                 }
1906         }*/
1907         
1908         if(!ss) {
1909                 sculpt_init_session(sd);
1910                 ss= sd->session;
1911         }
1912
1913         anchored = sd->brush->flag & BRUSH_ANCHORED;
1914         smooth_stroke = (sd->flags & SCULPT_INPUT_SMOOTH) && (sd->brush->sculpt_tool != SCULPT_TOOL_GRAB) && !anchored;
1915
1916         if(smooth_stroke)
1917                 sculpt_stroke_new(256);
1918
1919         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
1920         ss->damaged_verts.first = ss->damaged_verts.last = NULL;
1921         ss->vertexcosnos = NULL;
1922
1923         mmd = sculpt_multires_active(ob);
1924         sculpt_update_mesh_elements(ss, ob);
1925
1926         /* Check that vertex users are up-to-date */
1927         if(ob != active_ob || !ss->vertex_users || ss->vertex_users_size != ss->totvert) {
1928                 sculpt_vertexusers_free(ss);
1929                 calc_vertex_users(ss);
1930                 if(ss->projverts)
1931                         MEM_freeN(ss->projverts);
1932                 ss->projverts = NULL;
1933                 active_ob= ob;
1934         }
1935                 
1936         glEnableClientState(GL_VERTEX_ARRAY);
1937         glEnableClientState(GL_NORMAL_ARRAY);
1938
1939         /*XXX:
1940         persp(PERSP_VIEW);
1941         getmouseco_areawin(mvalo);*/
1942
1943         /* Init texture
1944            FIXME: Shouldn't be doing this every time! */
1945         if(sd->texrept!=SCULPTREPT_3D)
1946                 sculptmode_update_tex(sd);
1947
1948         /*XXX: getmouseco_areawin(mouse); */
1949         mvalo[0]= mouse[0];
1950         mvalo[1]= mouse[1];
1951         lastSigMouse[0]=mouse[0];
1952         lastSigMouse[1]=mouse[1];
1953         mousebut = 0; /* XXX: L_MOUSE; */
1954
1955         /* If modifier_calculations is true, then extra time must be spent
1956            updating the mesh. This takes a *lot* longer, so it's worth
1957            skipping if the modifier stack is empty. */
1958         modifier_calculations= sculpt_modifiers_active(ob);
1959
1960         init_sculptmatrices();
1961
1962         if(modifier_calculations)
1963                 ss->vertexcosnos= mesh_get_mapped_verts_nors(NULL, ob); /* XXX: scene = ? */
1964         sculptmode_update_all_projverts(ss);
1965
1966         /* Capture original copy */
1967         if(sd->flags & SCULPT_DRAW_FAST)
1968                 glAccum(GL_LOAD, 1);
1969
1970         /* Get original scissor box */
1971         glGetIntegerv(GL_SCISSOR_BOX, scissor_box);
1972         
1973         /* For raking, get the original angle*/
1974         offsetRot=sculpt_tex_angle(sd);
1975
1976         me = get_mesh(ob);
1977
1978         while (/*XXX:get_mbut() & mousebut*/0) {
1979                 /* XXX: getmouseco_areawin(mouse); */
1980                 /* If rake, and the mouse has moved over 10 pixels (euclidean) (prevents jitter) then get the new angle */
1981                 if (rake && (pow(lastSigMouse[0]-mouse[0],2)+pow(lastSigMouse[1]-mouse[1],2))>100){
1982                         /*Nasty looking, but just orig + new angle really*/
1983                         set_tex_angle(sd, offsetRot+180.+to_deg(atan2((float)(mouse[1]-lastSigMouse[1]),(float)(mouse[0]-lastSigMouse[0]))));
1984                         lastSigMouse[0]=mouse[0];
1985                         lastSigMouse[1]=mouse[1];
1986                 }
1987                 
1988                 if(firsttime || mouse[0]!=mvalo[0] || mouse[1]!=mvalo[1] ||
1989                    sd->brush->flag & BRUSH_AIRBRUSH) {
1990                         a->firsttime = firsttime;
1991                         firsttime= 0;
1992
1993                         if(smooth_stroke)
1994                                 sculpt_stroke_add_point(ss->stroke, mouse[0], mouse[1]);
1995
1996                         spacing+= sqrt(pow(mvalo[0]-mouse[0],2)+pow(mvalo[1]-mouse[1],2));
1997
1998                         if(modifier_calculations && !ss->vertexcosnos)
1999                                 ss->vertexcosnos= mesh_get_mapped_verts_nors(NULL, ob); /*XXX scene = ? */
2000
2001                         if(sd->brush->sculpt_tool != SCULPT_TOOL_GRAB) {
2002                                 if(anchored) {
2003                                         /* Restore the mesh before continuing with anchored stroke */
2004                                         if(a->mesh_store) {
2005                                                 for(i = 0; i < ss->totvert; ++i) {
2006                                                         VecCopyf(ss->mvert[i].co, &a->mesh_store[i].x);
2007                                                         ss->mvert[i].no[0] = a->orig_norms[i][0];
2008                                                         ss->mvert[i].no[1] = a->orig_norms[i][1];
2009                                                         ss->mvert[i].no[2] = a->orig_norms[i][2];
2010                                                 }
2011                                         }
2012                                         
2013                                         //do_symmetrical_brush_actions(sd, a, mouse, NULL);
2014                                 }
2015                                 else {
2016                                         if(smooth_stroke) {
2017                                                 sculpt_stroke_apply(sd, ss->stroke, a);
2018                                         }
2019                                         else if(sd->spacing==0 || spacing>sd->spacing) {
2020                                                 //do_symmetrical_brush_actions(sd, a, mouse, NULL);
2021                                                 spacing= 0;
2022                                         }
2023                                 }
2024                         }
2025                         else {
2026                                 //do_symmetrical_brush_actions(sd, a, mouse, mvalo);
2027                                 unproject(ss, sd->pivot, mouse[0], mouse[1], a->depth);
2028                         }
2029
2030                         if((!ss->multires && modifier_calculations) || ob_get_keyblock(ob))
2031                                 ;/* XXX: DAG_object_flush_update(G.scene, ob, OB_RECALC_DATA); */
2032
2033                         if(modifier_calculations || sd->brush->sculpt_tool == SCULPT_TOOL_GRAB || !(sd->flags & SCULPT_DRAW_FAST)) {
2034                                 calc_damaged_verts(ss, a);
2035                                 /*XXX: scrarea_do_windraw(curarea);
2036                                 screen_swapbuffers(); */
2037                         } else { /* Optimized drawing */
2038                                 calc_damaged_verts(ss, a);
2039
2040                                 /* Draw the stored image to the screen */
2041                                 glAccum(GL_RETURN, 1);
2042
2043                                 sculpt_clear_damaged_areas(ss);
2044                                 
2045                                 /* Draw all the polygons that are inside the modified area(s) */
2046                                 glScissor(scissor_box[0], scissor_box[1], scissor_box[2], scissor_box[3]);
2047                                 /* XXX: sculptmode_draw_mesh(1); */
2048                                 glAccum(GL_LOAD, 1);
2049
2050                                 projverts_clear_inside(ss);
2051
2052                                 /* XXX: persp(PERSP_WIN); */
2053                                 glDisable(GL_DEPTH_TEST);
2054                                 
2055                                 /* Draw cursor */
2056                                 if(sd->flags & SCULPT_TOOL_DRAW)
2057                                         fdrawXORcirc((float)mouse[0],(float)mouse[1],sd->brush->size);
2058                                 /* XXX: if(smooth_stroke)
2059                                    sculpt_stroke_draw();
2060                                 
2061                                 myswapbuffers(); */
2062                         }
2063
2064                         BLI_freelistN(&ss->damaged_rects);
2065                         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
2066         
2067                         mvalo[0]= mouse[0];
2068                         mvalo[1]= mouse[1];
2069
2070                         if(ss->vertexcosnos) {
2071                                 MEM_freeN(ss->vertexcosnos);
2072                                 ss->vertexcosnos= NULL;
2073                         }
2074
2075                 }
2076                 else ;/*XXX:BIF_wait_for_statechange();*/
2077         }
2078
2079         /* Set the rotation of the brush back to what it was before any rake */
2080         set_tex_angle(sd, offsetRot);
2081         
2082         if(smooth_stroke) {
2083                 sculpt_stroke_apply_all(sd, ss->stroke, a);
2084                 calc_damaged_verts(ss, a);
2085                 BLI_freelistN(&ss->damaged_rects);
2086         }
2087
2088         if(a->layer_disps) MEM_freeN(a->layer_disps);
2089         if(a->mesh_store) MEM_freeN(a->mesh_store);
2090         if(a->orig_norms) MEM_freeN(a->orig_norms);
2091         for(i=0; i<8; ++i)
2092                 BLI_freelistN(&a->grab_active_verts[i]);
2093         MEM_freeN(a);
2094         sculpt_stroke_free(ss->stroke);
2095         ss->stroke = NULL;
2096
2097         if(mmd) {
2098                 if(mmd->undo_verts && mmd->undo_verts != ss->mvert)
2099                         MEM_freeN(mmd->undo_verts);
2100                 
2101                 mmd->undo_verts = ss->mvert;
2102                 mmd->undo_verts_tot = ss->totvert;
2103         }
2104
2105         sculpt_undo_push(sd);
2106
2107         /* XXX: if(G.vd->depths) G.vd->depths->damaged= 1;
2108            allqueue(REDRAWVIEW3D, 0); */
2109 }
2110
2111 /* Partial Mesh Visibility */
2112
2113 /* XXX: Partial vis. always was a mess, have to figure something out */
2114 #if 0
2115 /* mode: 0=hide outside selection, 1=hide inside selection */
2116 static void sculptmode_do_pmv(Object *ob, rcti *hb_2d, int mode)
2117 {
2118         Mesh *me= get_mesh(ob);
2119         float hidebox[6][3];
2120         vec3f plane_normals[4];
2121         float plane_ds[4];
2122         unsigned i, j;
2123         unsigned ndx_show, ndx_hide;
2124         MVert *nve;
2125         unsigned face_cnt_show= 0, face_ndx_show= 0;
2126         unsigned edge_cnt_show= 0, edge_ndx_show= 0;
2127         unsigned *old_map= NULL;
2128         const unsigned SHOW= 0, HIDE=1;
2129
2130         /* Convert hide box from 2D to 3D */
2131         unproject(hidebox[0], hb_2d->xmin, hb_2d->ymax, 1);
2132         unproject(hidebox[1], hb_2d->xmax, hb_2d->ymax, 1);
2133         unproject(hidebox[2], hb_2d->xmax, hb_2d->ymin, 1);
2134         unproject(hidebox[3], hb_2d->xmin, hb_2d->ymin, 1);
2135         unproject(hidebox[4], hb_2d->xmin, hb_2d->ymax, 0);
2136         unproject(hidebox[5], hb_2d->xmax, hb_2d->ymin, 0);
2137         
2138         /* Calculate normals for each side of hide box */
2139         CalcNormFloat(hidebox[0], hidebox[1], hidebox[4], &plane_normals[0].x);
2140         CalcNormFloat(hidebox[1], hidebox[2], hidebox[5], &plane_normals[1].x);
2141         CalcNormFloat(hidebox[2], hidebox[3], hidebox[5], &plane_normals[2].x);
2142         CalcNormFloat(hidebox[3], hidebox[0], hidebox[4], &plane_normals[3].x);
2143         
2144         /* Calculate D for each side of hide box */
2145         for(i= 0; i<4; ++i)
2146                 plane_ds[i]= hidebox[i][0]*plane_normals[i].x + hidebox[i][1]*plane_normals[i].y +
2147                         hidebox[i][2]*plane_normals[i].z;
2148         
2149         /* Add partial visibility to mesh */
2150         if(!me->pv) {
2151                 me->pv= MEM_callocN(sizeof(PartialVisibility),"PartialVisibility");
2152         } else {
2153                 old_map= MEM_callocN(sizeof(unsigned)*me->pv->totvert,"PMV oldmap");
2154                 for(i=0; i<me->pv->totvert; ++i) {
2155                         old_map[i]= me->pv->vert_map[i]<me->totvert?0:1;
2156                 }
2157                 mesh_pmv_revert(ob, me);
2158         }
2159         
2160         /* Kill sculpt data */
2161         active_ob= NULL;
2162         
2163         /* Initalize map with which verts are to be hidden */
2164         me->pv->vert_map= MEM_mallocN(sizeof(unsigned)*me->totvert, "PMV vertmap");
2165         me->pv->totvert= me->totvert;
2166         me->totvert= 0;
2167         for(i=0; i<me->pv->totvert; ++i) {
2168                 me->pv->vert_map[i]= mode ? HIDE:SHOW;
2169                 for(j=0; j<4; ++j) {
2170                         if(me->mvert[i].co[0] * plane_normals[j].x +
2171                            me->mvert[i].co[1] * plane_normals[j].y +
2172                            me->mvert[i].co[2] * plane_normals[j].z < plane_ds[j] ) {
2173                                 me->pv->vert_map[i]= mode ? SHOW:HIDE; /* Vert is outside the hide box */
2174                                 break;
2175                         }
2176                 }
2177                 if(old_map && old_map[i]) me->pv->vert_map[i]= 1;
2178                 if(!me->pv->vert_map[i]) ++me->totvert;
2179
2180         }
2181         if(old_map) MEM_freeN(old_map);
2182
2183         /* Find out how many faces to show */
2184         for(i=0; i<me->totface; ++i) {
2185                 if(!me->pv->vert_map[me->mface[i].v1] &&
2186                    !me->pv->vert_map[me->mface[i].v2] &&
2187                    !me->pv->vert_map[me->mface[i].v3]) {
2188                         if(me->mface[i].v4) {
2189                                 if(!me->pv->vert_map[me->mface[i].v4])
2190                                         ++face_cnt_show;
2191                         }
2192                         else ++face_cnt_show;
2193                 }
2194         }
2195         /* Find out how many edges to show */
2196         for(i=0; i<me->totedge; ++i) {
2197                 if(!me->pv->vert_map[me->medge[i].v1] &&
2198                    !me->pv->vert_map[me->medge[i].v2])
2199                         ++edge_cnt_show;
2200         }
2201
2202         /* Create new vert array and reset each vert's map with map[old]=new index */
2203         nve= MEM_mallocN(sizeof(MVert)*me->pv->totvert, "PMV verts");
2204         ndx_show= 0; ndx_hide= me->totvert;
2205         for(i=0; i<me->pv->totvert; ++i) {
2206                 if(me->pv->vert_map[i]) {
2207                         me->pv->vert_map[i]= ndx_hide;
2208                         nve[me->pv->vert_map[i]]= me->mvert[i];
2209                         ++ndx_hide;
2210                 } else {
2211                         me->pv->vert_map[i]= ndx_show;
2212                         nve[me->pv->vert_map[i]]= me->mvert[i];
2213                         ++ndx_show;
2214                 }
2215         }
2216         CustomData_free_layer_active(&me->vdata, CD_MVERT, me->pv->totvert);
2217         me->mvert= CustomData_add_layer(&me->vdata, CD_MVERT, CD_ASSIGN, nve, me->totvert);
2218
2219         /* Create new face array */
2220         me->pv->old_faces= me->mface;
2221         me->pv->totface= me->totface;
2222         me->mface= MEM_mallocN(sizeof(MFace)*face_cnt_show, "PMV faces");
2223         for(i=0; i<me->totface; ++i) {
2224                 MFace *pr_f= &me->pv->old_faces[i];
2225                 char show= 0;
2226
2227                 if(me->pv->vert_map[pr_f->v1] < me->totvert &&
2228                    me->pv->vert_map[pr_f->v2] < me->totvert &&
2229                    me->pv->vert_map[pr_f->v3] < me->totvert) {
2230                         if(pr_f->v4) {
2231                                 if(me->pv->vert_map[pr_f->v4] < me->totvert)
2232                                         show= 1;
2233                         }
2234                         else show= 1;
2235                 }
2236
2237                 if(show) {
2238                         MFace *cr_f= &me->mface[face_ndx_show];
2239                         *cr_f= *pr_f;
2240                         cr_f->v1= me->pv->vert_map[pr_f->v1];
2241                         cr_f->v2= me->pv->vert_map[pr_f->v2];
2242                         cr_f->v3= me->pv->vert_map[pr_f->v3];
2243                         cr_f->v4= pr_f->v4 ? me->pv->vert_map[pr_f->v4] : 0;
2244                         test_index_face(cr_f,NULL,0,pr_f->v4?4:3);
2245                         ++face_ndx_show;
2246                 }
2247         }
2248         me->totface= face_cnt_show;
2249         CustomData_set_layer(&me->fdata, CD_MFACE, me->mface);
2250
2251         /* Create new edge array */
2252         me->pv->old_edges= me->medge;
2253         me->pv->totedge= me->totedge;
2254         me->medge= MEM_mallocN(sizeof(MEdge)*edge_cnt_show, "PMV edges");
2255         me->pv->edge_map= MEM_mallocN(sizeof(int)*me->pv->totedge,"PMV edgemap");
2256         for(i=0; i<me->totedge; ++i) {
2257                 if(me->pv->vert_map[me->pv->old_edges[i].v1] < me->totvert &&
2258                    me->pv->vert_map[me->pv->old_edges[i].v2] < me->totvert) {
2259                         MEdge *cr_e= &me->medge[edge_ndx_show];
2260                         me->pv->edge_map[i]= edge_ndx_show;
2261                         *cr_e= me->pv->old_edges[i];
2262                         cr_e->v1= me->pv->vert_map[me->pv->old_edges[i].v1];
2263                         cr_e->v2= me->pv->vert_map[me->pv->old_edges[i].v2];
2264                         ++edge_ndx_show;
2265                 }
2266                 else me->pv->edge_map[i]= -1;
2267         }
2268         me->totedge= edge_cnt_show;
2269         CustomData_set_layer(&me->edata, CD_MEDGE, me->medge);
2270
2271         /* XXX: DAG_object_flush_update(G.scene, OBACT, OB_RECALC_DATA); */
2272 }
2273
2274 static rcti sculptmode_pmv_box()
2275 {
2276         /*XXX:  short down[2], mouse[2];
2277         rcti ret;
2278
2279         getmouseco_areawin(down);
2280
2281         while((get_mbut()&L_MOUSE) || (get_mbut()&R_MOUSE)) {
2282                 getmouseco_areawin(mouse);
2283
2284                 scrarea_do_windraw(curarea);
2285
2286                 persp(PERSP_WIN);
2287                 glLineWidth(2);
2288                 setlinestyle(2);
2289                 sdrawXORline(down[0],down[1],mouse[0],down[1]);
2290                 sdrawXORline(mouse[0],down[1],mouse[0],mouse[1]);
2291                 sdrawXORline(mouse[0],mouse[1],down[0],mouse[1]);
2292                 sdrawXORline(down[0],mouse[1],down[0],down[1]);
2293                 setlinestyle(0);
2294                 glLineWidth(1);
2295                 persp(PERSP_VIEW);
2296
2297                 screen_swapbuffers();
2298                 backdrawview3d(0);
2299         }
2300
2301         ret.xmin= down[0]<mouse[0]?down[0]:mouse[0];
2302         ret.ymin= down[1]<mouse[1]?down[1]:mouse[1];
2303         ret.xmax= down[0]>mouse[0]?down[0]:mouse[0];
2304         ret.ymax= down[1]>mouse[1]?down[1]:mouse[1];
2305         return ret;*/
2306 }
2307
2308 void sculptmode_pmv(int mode)
2309 {
2310         Object *ob= NULL; /*XXX: OBACT; */
2311         rcti hb_2d;
2312         
2313         if(ob_get_key(ob)) {
2314                 error("Cannot hide mesh with shape keys enabled");
2315                 return;
2316         }
2317         
2318         hb_2d= sculptmode_pmv_box(); /* Get 2D hide box */
2319         
2320         sculptmode_correct_state();
2321
2322         waitcursor(1);
2323
2324         if(hb_2d.xmax-hb_2d.xmin > 3 && hb_2d.ymax-hb_2d.ymin > 3) {
2325                 init_sculptmatrices();
2326
2327                 sculptmode_do_pmv(ob,&hb_2d,mode);
2328         }
2329         else mesh_pmv_off(ob, get_mesh(ob));
2330
2331         /*XXX: scrarea_do_windraw(curarea); */
2332
2333         BIF_undo_push("Partial mesh hide");
2334
2335         waitcursor(0);
2336 }
2337 #endif