31d5ae4610ddfbb3f38fb24d7c3585aac3165b88
[blender-staging.git] / source / blender / src / sculptmode.c
1 /*
2  * $Id$
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software  Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2006 by Nicholas Bishop
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  *
29  * Implements the Sculpt Mode tools
30  *
31  * BDR_sculptmode.h
32  *
33  */
34
35 #include "GHOST_Types.h"
36
37 #include "MEM_guardedalloc.h"
38
39 #include "BLI_arithb.h"
40 #include "BLI_blenlib.h"
41 #include "BLI_dynstr.h"
42
43 #include "DNA_armature_types.h"
44 #include "DNA_brush_types.h"
45 #include "DNA_image_types.h"
46 #include "DNA_key_types.h"
47 #include "DNA_mesh_types.h"
48 #include "DNA_meshdata_types.h"
49 #include "DNA_modifier_types.h"
50 #include "DNA_object_types.h"
51 #include "DNA_screen_types.h"
52 #include "DNA_scene_types.h"
53 #include "DNA_texture_types.h"
54 #include "DNA_view3d_types.h"
55 #include "DNA_userdef_types.h"
56 #include "DNA_color_types.h"
57
58 #include "BKE_customdata.h"
59 #include "BKE_DerivedMesh.h"
60 #include "BKE_depsgraph.h"
61 #include "BKE_global.h"
62 #include "BKE_image.h"
63 #include "BKE_key.h"
64 #include "BKE_library.h"
65 #include "BKE_main.h"
66 #include "BKE_mesh.h"
67 #include "BKE_modifier.h"
68 #include "BKE_sculpt.h"
69 #include "BKE_texture.h"
70 #include "BKE_utildefines.h"
71 #include "BKE_colortools.h"
72
73 #include "BIF_editkey.h"
74 #include "BIF_editview.h"
75 #include "BIF_glutil.h"
76 #include "BIF_gl.h"
77 #include "BIF_interface.h"
78 #include "BIF_mywindow.h"
79 #include "BIF_radialcontrol.h"
80 #include "BIF_resources.h"
81 #include "BIF_screen.h"
82 #include "BIF_space.h"
83 #include "BIF_toolbox.h"
84
85 #include "BDR_sculptmode.h"
86
87 #include "BSE_drawview.h"
88 #include "BSE_edit.h"
89 #include "BSE_view.h"
90
91 #include "IMB_imbuf_types.h"
92
93 #include "blendef.h"
94 #include "multires.h"
95 #include "mydevice.h"
96
97 #include "RE_render_ext.h"
98 #include "RE_shader_ext.h" /*for multitex_ext*/
99
100 #include "GPU_draw.h"
101
102 #include <math.h>
103 #include <stdlib.h>
104 #include <string.h>
105
106 /* Number of vertices to average in order to determine the flatten distance */
107 #define FLATTEN_SAMPLE_SIZE 10
108
109 /* Texture cache size */
110 #define TC_SIZE 256
111
112 /* ===== STRUCTS =====
113  *
114  */
115
116 /* Used by vertex_users to store face indices in a list */
117 typedef struct IndexNode {
118         struct IndexNode* next,* prev;
119         int Index;
120 } IndexNode;
121
122
123 /* ActiveData stores an Index into the mvert array of Mesh, plus Fade, which
124    stores how far the vertex is from the brush center, scaled to the range [0,1]. */
125 typedef struct ActiveData {
126         struct ActiveData *next, *prev;
127         unsigned int Index;
128         float Fade;
129         float dist;
130 } ActiveData;
131
132 typedef struct BrushActionSymm {
133         float center_3d[3];
134         char index;
135
136         float up[3], right[3], out[3];
137
138         /* Grab brush */
139         float grab_delta[3];
140 } BrushActionSymm;
141
142 typedef struct BrushAction {
143         BrushActionSymm symm;
144
145         char firsttime;
146
147         /* Some brushes need access to original mesh vertices */
148         vec3f *mesh_store;
149         short (*orig_norms)[3];
150
151         short mouse[2];
152         float size_3d;
153
154         float prev_radius;
155         float radius;
156
157         float *layer_disps;
158         char flip;
159
160         char clip[3];
161         float cliptol[3];
162
163         float anchored_rot;
164
165         /* Grab brush */
166         ListBase grab_active_verts[8];
167         float depth;
168
169         /* Adjust brush strength along each axis
170            to adjust for object scaling */
171         float scale[3];
172 } BrushAction;
173
174 typedef struct RectNode {
175         struct RectNode *next, *prev;
176         rcti r;
177 } RectNode;
178
179 /* Used to store to 2D screen coordinates of each vertex in the mesh. */
180 typedef struct ProjVert {
181         short co[2];
182         
183         /* Used to mark whether a vertex is inside a rough bounding box
184            containing the brush. */
185         char inside;
186 } ProjVert;
187
188 static Object *active_ob= NULL;
189
190 SculptData *sculpt_data(void)
191 {
192         return &G.scene->sculptdata;
193 }
194
195 void sculpt_init_session(void);
196 void init_brushaction(BrushAction *a, short *, short *);
197 void sculpt_undo_push(const short);
198
199 SculptSession *sculpt_session(void)
200 {
201         if(!sculpt_data()->session)
202                 sculpt_init_session();
203         return sculpt_data()->session;
204 }
205
206 /* ===== MEMORY =====
207  * 
208  * Allocate/initialize/free data
209  */
210
211 void sculpt_init_session(void)
212 {
213         if(sculpt_data()->session)
214                 sculptsession_free(G.scene);
215         sculpt_data()->session= MEM_callocN(sizeof(SculptSession), "SculptSession");
216 }
217
218 /* vertex_users is an array of Lists that store all the faces that use a
219    particular vertex. vertex_users is in the same order as mesh.mvert */
220 void calc_vertex_users()
221 {
222         SculptSession *ss= sculpt_session();
223         int i,j;
224         IndexNode *node= NULL;
225         Mesh *me= get_mesh(OBACT);
226
227         sculpt_vertexusers_free(ss);
228         
229         /* For efficiency, use vertex_users_mem as a memory pool (may be larger
230            than necessary if mesh has triangles, but only one alloc is needed.) */
231         ss->vertex_users= MEM_callocN(sizeof(ListBase) * me->totvert, "vertex_users");
232         ss->vertex_users_size= me->totvert;
233         ss->vertex_users_mem= MEM_callocN(sizeof(IndexNode)*me->totface*4, "vertex_users_mem");
234         node= ss->vertex_users_mem;
235
236         /* Find the users */
237         for(i=0; i<me->totface; ++i){
238                 for(j=0; j<(me->mface[i].v4?4:3); ++j, ++node) {
239                         node->Index=i;
240                         BLI_addtail(&ss->vertex_users[((unsigned int*)(&me->mface[i]))[j]], node);
241                 }
242         }
243 }
244
245 /* ===== INTERFACE =====
246  */
247
248 void sculptmode_rem_tex(void *junk0,void *junk1)
249 {
250         MTex *mtex= G.scene->sculptdata.mtex[G.scene->sculptdata.texact];
251         if(mtex) {
252                 SculptSession *ss= sculpt_session();
253                 if(mtex->tex) mtex->tex->id.us--;
254                 MEM_freeN(mtex);
255                 G.scene->sculptdata.mtex[G.scene->sculptdata.texact]= NULL;
256                 /* Clear brush preview */
257                 if(ss->texcache) {
258                         MEM_freeN(ss->texcache);
259                         ss->texcache= NULL;
260                 }
261                 BIF_undo_push("Unlink brush texture");
262                 allqueue(REDRAWBUTSEDIT, 0);
263                 allqueue(REDRAWOOPS, 0);
264         }
265 }
266
267 /* ===== OPENGL =====
268  *
269  * Simple functions to get data from the GL
270  */
271
272 /* Store the modelview and projection matrices and viewport. */
273 void init_sculptmatrices()
274 {
275         SculptSession *ss= sculpt_session();
276
277         glMatrixMode(GL_MODELVIEW);
278         glPushMatrix();
279         glMultMatrixf(OBACT->obmat);
280
281         if(!ss->mats)
282                 ss->mats = MEM_callocN(sizeof(bglMats), "sculpt bglmats");
283         bgl_get_mats(ss->mats);
284         
285         glPopMatrix();
286
287 }
288
289 /* Uses window coordinates (x,y) to find the depth in the GL depth buffer. If
290    available, G.vd->depths is used so that the brush doesn't sculpt on top of
291    itself (G.vd->depths is only updated at the end of a brush stroke.) */
292 float get_depth(short x, short y)
293 {
294         float depth;
295
296         if(x<0 || y<0) return 1;
297         if(x>=curarea->winx || y>=curarea->winy) return 1;
298
299         if(G.vd->depths && x<G.vd->depths->w && y<G.vd->depths->h)
300                 return G.vd->depths->depths[y*G.vd->depths->w+x];
301         
302         x+= curarea->winrct.xmin;
303         y+= curarea->winrct.ymin;
304         
305         glReadPixels(x, y, 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &depth);
306
307         return depth;
308 }
309
310 /* Uses window coordinates (x,y) and depth component z to find a point in
311    modelspace */
312 void unproject(float out[3], const short x, const short y, const float z)
313 {
314         SculptSession *ss= sculpt_session();
315         double ux, uy, uz;
316
317         gluUnProject(x,y,z, ss->mats->modelview, ss->mats->projection,
318                      (GLint *)ss->mats->viewport, &ux, &uy, &uz );
319         out[0] = ux;
320         out[1] = uy;
321         out[2] = uz;
322 }
323
324 /* Convert a point in model coordinates to 2D screen coordinates. */
325 static void projectf(const float v[3], float p[2])
326 {
327         SculptSession *ss= sculpt_session();
328         double ux, uy, uz;
329
330         gluProject(v[0],v[1],v[2], ss->mats->modelview, ss->mats->projection,
331                    (GLint *)ss->mats->viewport, &ux, &uy, &uz);
332         p[0]= ux;
333         p[1]= uy;
334 }
335
336 static void project(const float v[3], short p[2])
337 {
338         float f[2];
339         projectf(v, f);
340
341         p[0]= f[0];
342         p[1]= f[1];
343 }
344
345 /* ===== Sculpting =====
346  *
347  */
348
349 /* Return modified brush size. Uses current tablet pressure (if available) to
350    shrink the brush. Skipped for grab brush because only the first mouse down
351    size is used, which is small if the user has just touched the pen to the
352    tablet */
353 char brush_size()
354 {
355         const BrushData *b= sculptmode_brush();
356         float size= b->size;
357         float pressure= get_pressure();
358         short activedevice= get_activedevice();
359         
360         if(sculpt_data()->brush_type!=GRAB_BRUSH) {
361                 const float size_factor= G.scene->sculptdata.tablet_size / 10.0f;
362                 if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
363                         size*= G.scene->sculptdata.tablet_size==0?1:
364                                (1-size_factor) + pressure*size_factor;
365         }
366         
367         return size;
368 }
369
370 /* Return modified brush strength. Includes the direction of the brush, positive
371    values pull vertices, negative values push. Uses tablet pressure and a
372    special multiplier found experimentally to scale the strength factor. */
373 float brush_strength(BrushAction *a)
374 {
375         const BrushData* b= sculptmode_brush();
376         float dir= b->dir==1 ? 1 : -1;
377         float pressure= 1;
378         short activedevice= get_activedevice();
379         float flip= a->flip ? -1:1;
380         float anchored = b->flag & SCULPT_BRUSH_ANCHORED ? 25 : 1;
381
382         const float strength_factor= G.scene->sculptdata.tablet_strength / 10.0f;
383         if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
384                 pressure= G.scene->sculptdata.tablet_strength==0?1:
385                                   (1-strength_factor) + get_pressure()*strength_factor;
386         
387         /* Flip direction for eraser */
388         if(activedevice==DEV_ERASER)
389                 dir= -dir;
390
391         switch(G.scene->sculptdata.brush_type){
392         case DRAW_BRUSH:
393         case LAYER_BRUSH:
394                 return b->strength / 5000.0f * dir * pressure * flip * anchored * G.vd->grid;
395         case SMOOTH_BRUSH:
396                 return b->strength / 50.0f * pressure * anchored;
397         case PINCH_BRUSH:
398                 return b->strength / 1000.0f * dir * pressure * flip * anchored;
399         case GRAB_BRUSH:
400                 return 1;
401         case INFLATE_BRUSH:
402                 return b->strength / 5000.0f * dir * pressure * flip * anchored;
403         case FLATTEN_BRUSH:
404                 return b->strength / 500.0f * pressure * anchored;
405         default:
406                 return 0;
407         }
408 }
409
410 /* For clipping against a mirror modifier */
411 void sculpt_clip(const BrushAction *a, float *co, const float val[3])
412 {
413         char i;
414         for(i=0; i<3; ++i) {
415                 if(a->clip[i] && (fabs(co[i]) <= a->cliptol[i]))
416                         co[i]= 0.0f;
417                 else
418                         co[i]= val[i];
419         }               
420 }
421
422 void sculpt_axislock(float *co)
423 {
424         SculptData *sd = sculpt_data();
425         if (sd->axislock == AXISLOCK_X+AXISLOCK_Y+AXISLOCK_Z) return;
426         if (G.vd->twmode == V3D_MANIP_LOCAL) {
427                 float mat[3][3], imat[3][3];
428                 Mat3CpyMat4(mat, OBACT->obmat);
429                 Mat3Inv(imat, mat);
430                 Mat3MulVecfl(mat, co);
431                 if (sd->axislock & AXISLOCK_X) co[0] = 0.0;
432                 if (sd->axislock & AXISLOCK_Y) co[1] = 0.0;
433                 if (sd->axislock & AXISLOCK_Z) co[2] = 0.0;             
434                 Mat3MulVecfl(imat, co);
435         } else {
436                 if (sd->axislock & AXISLOCK_X) co[0] = 0.0;
437                 if (sd->axislock & AXISLOCK_Y) co[1] = 0.0;
438                 if (sd->axislock & AXISLOCK_Z) co[2] = 0.0;             
439         }
440 }
441
442 static void add_norm_if(float view_vec[3], float out[3], float out_flip[3], const short no[3])
443 {
444         float fno[3] = {no[0], no[1], no[2]};
445
446         Normalize(fno);
447
448         if((Inpf(view_vec, fno)) > 0) {
449                 VecAddf(out, out, fno);
450         } else {
451                 VecAddf(out_flip, out_flip, fno); /* out_flip is used when out is {0,0,0} */
452         }
453 }
454
455 /* Currently only for the draw brush; finds average normal for all active
456    vertices */
457 void calc_area_normal(float out[3], const BrushAction *a, const float *outdir, const ListBase* active_verts)
458 {
459         Mesh *me = get_mesh(OBACT);
460         ActiveData *node = active_verts->first;
461         SculptData *sd = sculpt_data();
462         const int view = sd->brush_type==DRAW_BRUSH ? sculptmode_brush()->view : 0;
463         float out_flip[3];
464         
465         out[0]=out[1]=out[2] = out_flip[0]=out_flip[1]=out_flip[2] = 0;
466
467         if(sculptmode_brush()->flag & SCULPT_BRUSH_ANCHORED) {
468                 for(; node; node = node->next)
469                         add_norm_if(((BrushAction*)a)->symm.out, out, out_flip, a->orig_norms[node->Index]);
470         }
471         else {
472                 for(; node; node = node->next)
473                         add_norm_if(((BrushAction*)a)->symm.out, out, out_flip, me->mvert[node->Index].no);
474         }
475
476         if (out[0]==0.0 && out[1]==0.0 && out[2]==0.0) {
477                 VECCOPY(out, out_flip);
478         }
479         
480         Normalize(out);
481
482         if(outdir) {
483                 out[0] = outdir[0] * view + out[0] * (10-view);
484                 out[1] = outdir[1] * view + out[1] * (10-view);
485                 out[2] = outdir[2] * view + out[2] * (10-view);
486         }
487         
488         Normalize(out);
489 }
490
491 void do_draw_brush(const BrushAction *a, const ListBase* active_verts)
492 {
493         Mesh *me= get_mesh(OBACT);
494         float area_normal[3];
495         ActiveData *node= active_verts->first;
496
497         calc_area_normal(area_normal, a, a->symm.out, active_verts);
498         
499         sculpt_axislock(area_normal);
500         
501         while(node){
502                 float *co= me->mvert[node->Index].co;
503                 
504                 const float val[3]= {co[0]+area_normal[0]*node->Fade*a->scale[0],
505                                      co[1]+area_normal[1]*node->Fade*a->scale[1],
506                                      co[2]+area_normal[2]*node->Fade*a->scale[2]};
507                                      
508                 sculpt_clip(a, co, val);
509                 
510                 node= node->next;
511         }
512 }
513
514 /* For the smooth brush, uses the neighboring vertices around vert to calculate
515    a smoothed location for vert. Skips corner vertices (used by only one
516    polygon.) */
517 vec3f neighbor_average(const int vert)
518 {
519         SculptSession *ss= sculpt_session();
520         Mesh *me= get_mesh(OBACT);
521         int i, skip= -1, total=0;
522         IndexNode *node= ss->vertex_users[vert].first;
523         vec3f avg= {0,0,0};
524         char ncount= BLI_countlist(&ss->vertex_users[vert]);
525         MFace *f;
526                 
527         /* Don't modify corner vertices */
528         if(ncount==1) {
529                 VecCopyf(&avg.x, me->mvert[vert].co);
530                 return avg;
531         }
532
533         while(node){
534                 f= &me->mface[node->Index];
535                 
536                 if(f->v4) {
537                         skip= (f->v1==vert?2:
538                                f->v2==vert?3:
539                                f->v3==vert?0:
540                                f->v4==vert?1:-1);
541                 }
542
543                 for(i=0; i<(f->v4?4:3); ++i) {
544                         if(i != skip && (ncount!=2 || BLI_countlist(&ss->vertex_users[(&f->v1)[i]]) <= 2)) {
545                                 VecAddf(&avg.x,&avg.x,me->mvert[(&f->v1)[i]].co);
546                                 ++total;
547                         }
548                 }
549
550                 node= node->next;
551         }
552
553         if(total>0) {
554                 avg.x/= total;
555                 avg.y/= total;
556                 avg.z/= total;
557         }
558         else
559                 VecCopyf(&avg.x, me->mvert[vert].co);
560
561         return avg;
562 }
563
564 void do_smooth_brush(const BrushAction *a, const ListBase* active_verts)
565 {
566         ActiveData *node= active_verts->first;
567         Mesh *me= get_mesh(OBACT);
568
569         while(node){
570                 float *co= me->mvert[node->Index].co;
571                 const vec3f avg= neighbor_average(node->Index);
572                 const float val[3]= {co[0]+(avg.x-co[0])*node->Fade,
573                                      co[1]+(avg.y-co[1])*node->Fade,
574                                      co[2]+(avg.z-co[2])*node->Fade};
575                 sculpt_clip(a, co, val);
576                 node= node->next;
577         }
578 }
579
580 void do_pinch_brush(const BrushAction *a, const ListBase* active_verts)
581 {
582         Mesh *me= get_mesh(OBACT);
583         ActiveData *node= active_verts->first;
584
585         while(node) {
586                 float *co= me->mvert[node->Index].co;
587                 const float val[3]= {co[0]+(a->symm.center_3d[0]-co[0])*node->Fade,
588                                      co[1]+(a->symm.center_3d[1]-co[1])*node->Fade,
589                                      co[2]+(a->symm.center_3d[2]-co[2])*node->Fade};
590                 sculpt_clip(a, co, val);
591                 node= node->next;
592         }
593 }
594
595 void do_grab_brush(BrushAction *a)
596 {
597         Mesh *me= get_mesh(OBACT);
598         ActiveData *node= a->grab_active_verts[a->symm.index].first;
599         float add[3];
600         float grab_delta[3];
601         
602         VecCopyf(grab_delta, a->symm.grab_delta);
603         sculpt_axislock(grab_delta);
604         
605         while(node) {
606                 float *co= me->mvert[node->Index].co;
607                 
608                 VecCopyf(add, grab_delta);
609                 VecMulf(add, node->Fade);
610                 VecAddf(add, add, co);
611                 sculpt_clip(a, co, add);
612
613                 node= node->next;
614         }
615         
616 }
617
618 void do_layer_brush(BrushAction *a, const ListBase *active_verts)
619 {
620         Mesh *me= get_mesh(OBACT);
621         float area_normal[3];
622         ActiveData *node= active_verts->first;
623         const float bstr= brush_strength(a);
624
625         calc_area_normal(area_normal, a, NULL, active_verts);
626
627         while(node){
628                 float *disp= &a->layer_disps[node->Index];
629                 
630                 if((bstr > 0 && *disp < bstr) ||
631                   (bstr < 0 && *disp > bstr)) {
632                         float *co= me->mvert[node->Index].co;
633                         
634                         *disp+= node->Fade;
635
636                         if(bstr < 0) {
637                                 if(*disp < bstr)
638                                         *disp = bstr;
639                         } else {
640                                 if(*disp > bstr)
641                                         *disp = bstr;
642                         }
643
644                         {
645                                 const float val[3]= {a->mesh_store[node->Index].x+area_normal[0] * *disp*a->scale[0],
646                                                      a->mesh_store[node->Index].y+area_normal[1] * *disp*a->scale[1],
647                                                      a->mesh_store[node->Index].z+area_normal[2] * *disp*a->scale[2]};
648                                 sculpt_clip(a, co, val);
649                         }
650                 }
651
652                 node= node->next;
653         }
654 }
655
656 void do_inflate_brush(const BrushAction *a, const ListBase *active_verts)
657 {
658         ActiveData *node= active_verts->first;
659         float add[3];
660         Mesh *me= get_mesh(OBACT);
661         
662         while(node) {
663                 float *co= me->mvert[node->Index].co;
664                 short *no= me->mvert[node->Index].no;
665
666                 add[0]= no[0]/ 32767.0f;
667                 add[1]= no[1]/ 32767.0f;
668                 add[2]= no[2]/ 32767.0f;
669                 VecMulf(add, node->Fade);
670                 add[0]*= a->scale[0];
671                 add[1]*= a->scale[1];
672                 add[2]*= a->scale[2];
673                 VecAddf(add, add, co);
674                 
675                 sculpt_clip(a, co, add);
676
677                 node= node->next;
678         }
679 }
680
681 void calc_flatten_center(Mesh *me, ActiveData *node, float co[3])
682 {
683         ActiveData *outer[FLATTEN_SAMPLE_SIZE];
684         int i;
685         
686         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
687                 outer[i] = node;
688                 
689         for(; node; node = node->next) {
690                 for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i) {
691                         if(node->dist > outer[i]->dist) {
692                                 outer[i] = node;
693                                 break;
694                         }
695                 }
696         }
697         
698         co[0] = co[1] = co[2] = 0.0f;
699         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
700                 VecAddf(co, co, me->mvert[outer[i]->Index].co);
701         VecMulf(co, 1.0f / FLATTEN_SAMPLE_SIZE);
702 }
703
704 void do_flatten_brush(const BrushAction *a, const ListBase *active_verts)
705 {
706         Mesh *me= get_mesh(OBACT);
707         ActiveData *node= active_verts->first;
708         /* area_normal and cntr define the plane towards which vertices are squashed */
709         float area_normal[3];
710         float cntr[3];
711
712         calc_area_normal(area_normal, a, a->symm.out, active_verts);
713         calc_flatten_center(me, node, cntr);
714
715         while(node){
716                 float *co= me->mvert[node->Index].co;
717                 float p1[3], sub1[3], sub2[3], intr[3], val[3];
718                 
719                 /* Find the intersection between squash-plane and vertex (along the area normal) */
720                 VecSubf(p1, co, area_normal);
721                 VecSubf(sub1, cntr, p1);
722                 VecSubf(sub2, co, p1);
723                 VecSubf(intr, co, p1);
724                 VecMulf(intr, Inpf(area_normal, sub1) / Inpf(area_normal, sub2));
725                 VecAddf(intr, intr, p1);
726                 
727                 VecSubf(val, intr, co);
728                 VecMulf(val, node->Fade);
729                 VecAddf(val, val, co);
730                 
731                 sculpt_clip(a, co, val);
732                 
733                 node= node->next;
734         }
735 }
736
737 /* Uses the brush curve control to find a strength value between 0 and 1 */
738 float curve_strength(float p, const float len)
739 {
740         if(p > len) p= len;
741         return curvemapping_evaluateF(G.scene->sculptdata.cumap, 0, p/len);
742 }
743
744 /* Uses symm to selectively flip any axis of a coordinate. */
745 void flip_coord(float co[3], const char symm)
746 {
747         if(symm & SYMM_X)
748                 co[0]= -co[0];
749         if(symm & SYMM_Y)
750                 co[1]= -co[1];
751         if(symm & SYMM_Z)
752                 co[2]= -co[2];
753 }
754
755 /* Use the warpfac field in MTex to store a rotation value for sculpt textures. Value is in degrees */
756 float sculpt_tex_angle(void)
757 {
758         SculptData *sd= sculpt_data();
759         if(sd->texact!=-1 && sd->mtex[sd->texact])
760                 return sd->mtex[sd->texact]->warpfac;
761         return 0;
762 }
763
764 void set_tex_angle(const float f)
765 {
766         SculptData *sd = sculpt_data();
767         if(sd->texact != -1 && sd->mtex[sd->texact])
768                 sd->mtex[sd->texact]->warpfac = f;
769 }
770         
771 float to_rad(const float deg)
772 {
773         return deg * (M_PI/180.0f);
774 }
775
776 float to_deg(const float rad)
777 {
778         return rad * (180.0f/M_PI);
779 }
780
781 /* Get a pixel from the texcache at (px, py) */
782 static unsigned char get_texcache_pixel(const SculptSession *ss, int px, int py)
783 {
784         unsigned *p;
785         p = ss->texcache + py * ss->texcache_w + px;
786         return ((unsigned char*)(p))[0];
787 }
788
789 static float get_texcache_pixel_bilinear(const SculptSession *ss, float u, float v)
790 {
791         int x, y, x2, y2;
792         const int tc_max = TC_SIZE - 1;
793         float urat, vrat, uopp;
794
795         if(u < 0) u = 0;
796         else if(u >= TC_SIZE) u = tc_max;
797         if(v < 0) v = 0;
798         else if(v >= TC_SIZE) v = tc_max;
799
800         x = floor(u);
801         y = floor(v);
802         x2 = x + 1;
803         y2 = y + 1;
804
805         if(x2 > TC_SIZE) x2 = tc_max;
806         if(y2 > TC_SIZE) y2 = tc_max;
807         
808         urat = u - x;
809         vrat = v - y;
810         uopp = 1 - urat;
811                 
812         return ((get_texcache_pixel(ss, x, y) * uopp +
813                  get_texcache_pixel(ss, x2, y) * urat) * (1 - vrat) + 
814                 (get_texcache_pixel(ss, x, y2) * uopp +
815                  get_texcache_pixel(ss, x2, y2) * urat) * vrat) / 255.0;
816 }
817
818 /* Return a multiplier for brush strength on a particular vertex. */
819 float tex_strength(BrushAction *a, float *point, const float len,const unsigned vindex)
820 {
821         SculptData *sd= sculpt_data();
822         SculptSession *ss= sculpt_session();
823         float avg= 1;
824
825         if(sd->texact==-1 || !sd->mtex[sd->texact])
826                 avg= 1;
827         else if(sd->texrept==SCULPTREPT_3D) {
828                 /* Get strength by feeding the vertex location directly
829                    into a texture */
830                 float jnk;
831                 const float factor= 0.01;
832                 MTex mtex;
833                 memset(&mtex,0,sizeof(MTex));
834                 mtex.tex= sd->mtex[sd->texact]->tex;
835                 mtex.projx= 1;
836                 mtex.projy= 2;
837                 mtex.projz= 3;
838                 VecCopyf(mtex.size, sd->mtex[sd->texact]->size);
839                 VecMulf(mtex.size, factor);
840                 if(!sd->texsep)
841                         mtex.size[1]= mtex.size[2]= mtex.size[0];
842                 
843                 externtex(&mtex,point,&avg,&jnk,&jnk,&jnk,&jnk);
844         }
845         else if(ss->texcache) {
846                 const float bsize= a->radius * 2;
847                 const float rot= to_rad(sculpt_tex_angle()) + a->anchored_rot;
848                 int px, py;
849                 float flip[3], point_2d[2];
850
851                 /* If the active area is being applied for symmetry, flip it
852                    across the symmetry axis in order to project it. This insures
853                    that the brush texture will be oriented correctly. */
854                 VecCopyf(flip, point);
855                 flip_coord(flip, a->symm.index);
856                 projectf(flip, point_2d);
857
858                 /* For Tile and Drag modes, get the 2D screen coordinates of the
859                    and scale them up or down to the texture size. */
860                 if(sd->texrept==SCULPTREPT_TILE) {
861                         const int sx= (const int)sd->mtex[sd->texact]->size[0];
862                         const int sy= (const int)sd->texsep ? sd->mtex[sd->texact]->size[1] : sx;
863                         
864                         float fx= point_2d[0];
865                         float fy= point_2d[1];
866                         
867                         float angle= atan2(fy, fx) - rot;
868                         float len= sqrtf(fx*fx + fy*fy);
869                         
870                         if(rot<0.001 && rot>-0.001) {
871                                 px= point_2d[0];
872                                 py= point_2d[1];
873                         } else {
874                                 px= len * cos(angle) + 2000;
875                                 py= len * sin(angle) + 2000;
876                         }
877                         if(sx != 1)
878                                 px %= sx-1;
879                         if(sy != 1)
880                                 py %= sy-1;
881                         avg= get_texcache_pixel_bilinear(ss, TC_SIZE*px/sx, TC_SIZE*py/sy);
882                 } else {
883                         float fx= (point_2d[0] - a->mouse[0]) / bsize;
884                         float fy= (point_2d[1] - a->mouse[1]) / bsize;
885
886                         float angle= atan2(fy, fx) - rot;
887                         float len= sqrtf(fx*fx + fy*fy);
888                         
889                         fx = len * cos(angle) + 0.5;
890                         fy = len * sin(angle) + 0.5;
891
892                         avg= get_texcache_pixel_bilinear(ss, fx * TC_SIZE, fy * TC_SIZE);
893                 }
894         }
895
896         if(sd->texfade)
897                 avg*= curve_strength(len, a->size_3d); /* Smooth curve */
898
899         return avg;
900 }
901
902 /* Mark area around the brush as damaged. projverts are marked if they are
903    inside the area and the damaged rectangle in 2D screen coordinates is 
904    added to damaged_rects. */
905 void sculpt_add_damaged_rect(BrushAction *a)
906 {
907         short p[2];
908         RectNode *rn= MEM_mallocN(sizeof(RectNode),"RectNode");
909         Mesh *me= get_mesh(OBACT);
910         SculptSession *ss = sculpt_session();
911         const float radius = a->radius > a->prev_radius ? a->radius : a->prev_radius;
912         unsigned i;
913
914         /* Find center */
915         project(a->symm.center_3d, p);
916         rn->r.xmin= p[0] - radius;
917         rn->r.ymin= p[1] - radius;
918         rn->r.xmax= p[0] + radius;
919         rn->r.ymax= p[1] + radius;
920
921         BLI_addtail(&sculpt_session()->damaged_rects, rn);
922
923         /* Update insides */
924         for(i=0; i<me->totvert; ++i) {
925                 if(!ss->projverts[i].inside) {
926                         if(ss->projverts[i].co[0] > rn->r.xmin && ss->projverts[i].co[1] > rn->r.ymin &&
927                            ss->projverts[i].co[0] < rn->r.xmax && ss->projverts[i].co[1] < rn->r.ymax) {
928                                 ss->projverts[i].inside= 1;
929                         }
930                 }
931         }
932 }
933
934 /* Clears the depth buffer in each modified area. */
935 void sculpt_clear_damaged_areas(SculptSession *ss)
936 {
937         RectNode *rn= NULL;
938
939         for(rn = ss->damaged_rects.first; rn; rn = rn->next) {
940                 rcti clp = rn->r;
941                 rcti *win = &curarea->winrct;
942                 
943                 clp.xmin += win->xmin;
944                 clp.xmax += win->xmin;
945                 clp.ymin += win->ymin;
946                 clp.ymax += win->ymin;
947                 
948                 if(clp.xmin < win->xmax && clp.xmax > win->xmin &&
949                    clp.ymin < win->ymax && clp.ymax > win->ymin) {
950                         if(clp.xmin < win->xmin) clp.xmin = win->xmin;
951                         if(clp.ymin < win->ymin) clp.ymin = win->ymin;
952                         if(clp.xmax > win->xmax) clp.xmax = win->xmax;
953                         if(clp.ymax > win->ymax) clp.ymax = win->ymax;
954
955                         glScissor(clp.xmin + 1, clp.ymin + 1,
956                                   clp.xmax - clp.xmin - 2,
957                                   clp.ymax - clp.ymin - 2);
958                 }
959                 
960                 glClear(GL_DEPTH_BUFFER_BIT);
961         }
962 }
963
964 void do_brush_action(BrushAction *a)
965 {
966         int i;
967         float av_dist;
968         ListBase active_verts={0,0};
969         ActiveData *adata= 0;
970         float *vert;
971         Mesh *me= get_mesh(OBACT);
972         const float bstrength= brush_strength(a);
973         KeyBlock *keyblock= ob_get_keyblock(OBACT);
974         SculptData *sd = sculpt_data();
975         SculptSession *ss = sculpt_session();
976
977         sculpt_add_damaged_rect(a);
978
979         /* Build a list of all vertices that are potentially within the brush's
980            area of influence. Only do this once for the grab brush. */
981         if((sd->brush_type != GRAB_BRUSH) || a->firsttime) {
982                 for(i=0; i<me->totvert; ++i) {
983                         /* Projverts.inside provides a rough bounding box */
984                         if(ss->projverts[i].inside) {
985                                 vert= ss->vertexcosnos ? &ss->vertexcosnos[i*6] : me->mvert[i].co;
986                                 av_dist= VecLenf(a->symm.center_3d, vert);
987                                 if(av_dist < a->size_3d) {
988                                         adata= (ActiveData*)MEM_mallocN(sizeof(ActiveData), "ActiveData");
989
990                                         adata->Index = i;
991                                         /* Fade is used to store the final strength at which the brush
992                                            should modify a particular vertex. */
993                                         adata->Fade= tex_strength(a, vert, av_dist, i) * bstrength;
994                                         adata->dist = av_dist;
995
996                                         if(sd->brush_type == GRAB_BRUSH && a->firsttime)
997                                                 BLI_addtail(&a->grab_active_verts[a->symm.index], adata);
998                                         else
999                                                 BLI_addtail(&active_verts, adata);
1000                                 }
1001                         }
1002                 }
1003         }
1004
1005         /* Only act if some verts are inside the brush area */
1006         if(active_verts.first || (sd->brush_type == GRAB_BRUSH && a->grab_active_verts[a->symm.index].first)) {
1007                 /* Apply one type of brush action */
1008                 switch(G.scene->sculptdata.brush_type){
1009                 case DRAW_BRUSH:
1010                         do_draw_brush(a, &active_verts);
1011                         break;
1012                 case SMOOTH_BRUSH:
1013                         do_smooth_brush(a, &active_verts);
1014                         break;
1015                 case PINCH_BRUSH:
1016                         do_pinch_brush(a, &active_verts);
1017                         break;
1018                 case INFLATE_BRUSH:
1019                         do_inflate_brush(a, &active_verts);
1020                         break;
1021                 case GRAB_BRUSH:
1022                         do_grab_brush(a);
1023                         break;
1024                 case LAYER_BRUSH:
1025                         do_layer_brush(a, &active_verts);
1026                         break;
1027                 case FLATTEN_BRUSH:
1028                         do_flatten_brush(a, &active_verts);
1029                         break;
1030                 }
1031         
1032                 /* Copy the modified vertices from mesh to the active key */
1033                 if(keyblock) {
1034                         float *co= keyblock->data;
1035                         if(co) {
1036                                 if(sd->brush_type == GRAB_BRUSH)
1037                                         adata = a->grab_active_verts[a->symm.index].first;
1038                                 else
1039                                         adata = active_verts.first;
1040
1041                                 for(; adata; adata= adata->next)
1042                                         if(adata->Index < keyblock->totelem)
1043                                                 VecCopyf(&co[adata->Index*3], me->mvert[adata->Index].co);
1044                         }
1045                 }
1046
1047                 if(ss->vertexcosnos)
1048                         BLI_freelistN(&active_verts);
1049                 else {
1050                         if(sd->brush_type != GRAB_BRUSH)
1051                                 addlisttolist(&ss->damaged_verts, &active_verts);
1052                 }
1053         }
1054 }
1055
1056 /* Flip all the editdata across the axis/axes specified by symm. Used to
1057    calculate multiple modifications to the mesh when symmetry is enabled. */
1058 void calc_brushdata_symm(BrushAction *a, const char symm)
1059 {
1060         flip_coord(a->symm.center_3d, symm);
1061         flip_coord(a->symm.up, symm);
1062         flip_coord(a->symm.right, symm);
1063         flip_coord(a->symm.out, symm);
1064         
1065         a->symm.index= symm;
1066
1067         flip_coord(a->symm.grab_delta, symm);
1068 }
1069
1070 void do_symmetrical_brush_actions(BrushAction *a, short co[2], short pr_co[2])
1071 {
1072         const char symm = sculpt_data()->symm;
1073         BrushActionSymm orig;
1074         int i;
1075
1076         init_brushaction(a, co, pr_co);
1077         orig = a->symm;
1078         do_brush_action(a);
1079
1080         for(i = 1; i <= symm; ++i) {
1081                 if(symm & i && (symm != 5 || i != 3) && (symm != 6 || (i != 3 && i != 5))) {
1082                         // Restore the original symmetry data
1083                         a->symm = orig;
1084
1085                         calc_brushdata_symm(a, i);
1086                         do_brush_action(a);
1087                 }
1088         }
1089
1090         a->symm = orig;
1091 }
1092
1093 void add_face_normal(vec3f *norm, const MFace* face)
1094 {
1095         Mesh *me= get_mesh(OBACT);
1096
1097         vec3f c= {me->mvert[face->v1].co[0],me->mvert[face->v1].co[1],me->mvert[face->v1].co[2]};
1098         vec3f b= {me->mvert[face->v2].co[0],me->mvert[face->v2].co[1],me->mvert[face->v2].co[2]};
1099         vec3f a= {me->mvert[face->v3].co[0],me->mvert[face->v3].co[1],me->mvert[face->v3].co[2]};
1100         vec3f s1, s2;
1101
1102         VecSubf(&s1.x,&a.x,&b.x);
1103         VecSubf(&s2.x,&c.x,&b.x);
1104
1105         norm->x+= s1.y * s2.z - s1.z * s2.y;
1106         norm->y+= s1.z * s2.x - s1.x * s2.z;
1107         norm->z+= s1.x * s2.y - s1.y * s2.x;
1108 }
1109
1110 void update_damaged_vert(Mesh *me, ListBase *lb)
1111 {
1112         ActiveData *vert;
1113        
1114         for(vert= lb->first; vert; vert= vert->next) {
1115                 vec3f norm= {0,0,0};            
1116                 IndexNode *face= sculpt_session()->vertex_users[vert->Index].first;
1117
1118                 while(face){
1119                         add_face_normal(&norm,&me->mface[face->Index]);
1120                         face= face->next;
1121                 }
1122                 Normalize(&norm.x);
1123                 
1124                 me->mvert[vert->Index].no[0]=norm.x*32767;
1125                 me->mvert[vert->Index].no[1]=norm.y*32767;
1126                 me->mvert[vert->Index].no[2]=norm.z*32767;
1127         }
1128 }
1129
1130 void calc_damaged_verts(ListBase *damaged_verts, BrushAction *a)
1131 {
1132         Mesh *me= get_mesh(OBACT);
1133         int i;
1134         
1135         for(i=0; i<8; ++i)
1136                 update_damaged_vert(me, &a->grab_active_verts[i]);
1137
1138         update_damaged_vert(me, damaged_verts);
1139         BLI_freelistN(damaged_verts);
1140         damaged_verts->first = damaged_verts->last = NULL;
1141 }
1142
1143 void projverts_clear_inside(SculptSession *ss)
1144 {
1145         Mesh *me = get_mesh(OBACT);
1146         if(me) {
1147                 int i;
1148                 for(i = 0; i < me->totvert; ++i)
1149                         ss->projverts[i].inside = 0;
1150         }
1151 }
1152
1153 BrushData *sculptmode_brush(void)
1154 {
1155         SculptData *sd= &G.scene->sculptdata;
1156
1157         BrushData *bd = 
1158                 (sd->brush_type==DRAW_BRUSH ? &sd->drawbrush :
1159                  sd->brush_type==SMOOTH_BRUSH ? &sd->smoothbrush :
1160                  sd->brush_type==PINCH_BRUSH ? &sd->pinchbrush :
1161                  sd->brush_type==INFLATE_BRUSH ? &sd->inflatebrush :
1162                  sd->brush_type==GRAB_BRUSH ? &sd->grabbrush :
1163                  sd->brush_type==LAYER_BRUSH ? &sd->layerbrush :
1164                  sd->brush_type==FLATTEN_BRUSH ? &sd->flattenbrush : NULL);
1165
1166         if(!bd) {
1167                 sculptdata_init(G.scene);
1168                 bd = &sd->drawbrush;
1169         }
1170
1171         return bd;
1172 }
1173
1174 void sculptmode_update_tex()
1175 {
1176         SculptData *sd= sculpt_data();
1177         SculptSession *ss= sculpt_session();
1178         MTex *mtex = sd->mtex[sd->texact];
1179         TexResult texres = {0};
1180         float x, y, step=2.0/TC_SIZE, co[3];
1181         int hasrgb, ix, iy;
1182         
1183         /* Skip Default brush shape and non-textures */
1184         if(sd->texact == -1 || !sd->mtex[sd->texact]) return;
1185
1186         if(ss->texcache) {
1187                 MEM_freeN(ss->texcache);
1188                 ss->texcache= NULL;
1189         }
1190         
1191         ss->texcache_w = ss->texcache_h = TC_SIZE;
1192         ss->texcache = MEM_callocN(sizeof(int) * ss->texcache_w * ss->texcache_h, "Sculpt Texture cache");
1193         
1194         if(mtex && mtex->tex) {
1195                 BKE_image_get_ibuf(sd->mtex[sd->texact]->tex->ima, NULL);
1196                 
1197                 /*do normalized cannonical view coords for texture*/
1198                 for (y=-1.0, iy=0; iy<TC_SIZE; iy++, y += step) {
1199                         for (x=-1.0, ix=0; ix<TC_SIZE; ix++, x += step) {
1200                                 co[0]= x;
1201                                 co[1]= y;
1202                                 co[2]= 0.0f;
1203                                 
1204                                 /* This is copied from displace modifier code */
1205                                 hasrgb = multitex_ext(mtex->tex, co, NULL, NULL, 1, &texres);
1206                         
1207                                 /* if the texture gave an RGB value, we assume it didn't give a valid
1208                                  * intensity, so calculate one (formula from do_material_tex).
1209                                  * if the texture didn't give an RGB value, copy the intensity across
1210                                  */
1211                                 if(hasrgb & TEX_RGB)
1212                                         texres.tin = (0.35 * texres.tr + 0.45 *
1213                                                       texres.tg + 0.2 * texres.tb);
1214
1215                                 texres.tin = texres.tin * 255.0;
1216                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4] = (char)texres.tin;
1217                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+1] = (char)texres.tin;
1218                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+2] = (char)texres.tin;
1219                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+3] = (char)texres.tin;
1220                         }
1221                 }
1222         }
1223 }
1224
1225 /* pr_mouse is only used for the grab brush, can be NULL otherwise */
1226 void init_brushaction(BrushAction *a, short *mouse, short *pr_mouse)
1227 {
1228         SculptData *sd = sculpt_data();
1229         const float mouse_depth = get_depth(mouse[0], mouse[1]);
1230         float brush_edge_loc[3], zero_loc[3], oldloc[3];
1231         ModifierData *md;
1232         int i;
1233         const char flip = (get_qual() == LR_SHIFTKEY);
1234         const char anchored = sculptmode_brush()->flag & SCULPT_BRUSH_ANCHORED;
1235         short orig_mouse[2], dx=0, dy=0;
1236
1237         a->flip = flip;
1238         a->symm.index = 0;
1239
1240         if(a->firsttime) 
1241                 a->depth = mouse_depth;
1242         
1243         /* Convert the location and size of the brush to
1244            modelspace coords */
1245         if(a->firsttime || !anchored) {
1246                 unproject(a->symm.center_3d, mouse[0], mouse[1], mouse_depth);
1247                 a->mouse[0] = mouse[0];
1248                 a->mouse[1] = mouse[1];
1249         }
1250  
1251         if(anchored) {
1252                 project(a->symm.center_3d, orig_mouse);
1253                 dx = mouse[0] - orig_mouse[0];
1254                 dy = mouse[1] - orig_mouse[1];
1255         }
1256  
1257         if(anchored) {
1258                 unproject(brush_edge_loc, mouse[0], mouse[1], a->depth);
1259                 a->anchored_rot = atan2(dy, dx);
1260         }
1261         else
1262                 unproject(brush_edge_loc, mouse[0] + brush_size(), mouse[1], mouse_depth);
1263  
1264         a->size_3d = VecLenf(a->symm.center_3d, brush_edge_loc);
1265
1266         a->prev_radius = a->radius;
1267
1268         if(anchored)
1269                 a->radius = sqrt(dx*dx + dy*dy);
1270         else
1271                 a->radius = brush_size();
1272
1273         /* Set the pivot to allow the model to rotate around the center of the brush */
1274         if(get_depth(mouse[0],mouse[1]) < 1.0)
1275                 VecCopyf(sd->pivot, a->symm.center_3d);
1276
1277         /* Now project the Up, Right, and Out normals from view to model coords */
1278         unproject(zero_loc, 0, 0, 0);
1279         unproject(a->symm.up, 0, -1, 0);
1280         unproject(a->symm.right, 1, 0, 0);
1281         unproject(a->symm.out, 0, 0, -1);
1282         VecSubf(a->symm.up, a->symm.up, zero_loc);
1283         VecSubf(a->symm.right, a->symm.right, zero_loc);
1284         VecSubf(a->symm.out, a->symm.out, zero_loc);
1285         Normalize(a->symm.up);
1286         Normalize(a->symm.right);
1287         Normalize(a->symm.out);
1288         
1289         /* Initialize mirror modifier clipping */
1290         for(i=0; i<3; ++i) {
1291                 a->clip[i]= 0;
1292                 a->cliptol[i]= 0;
1293         }
1294         for(md= OBACT->modifiers.first; md; md= md->next) {
1295                 if(md->type==eModifierType_Mirror && (md->mode & eModifierMode_Realtime)) {
1296                         const MirrorModifierData *mmd = (MirrorModifierData*) md;
1297                         
1298                         if(mmd->flag & MOD_MIR_CLIPPING) {
1299                                 a->clip[mmd->axis]= 1;
1300                                 if(mmd->tolerance > a->cliptol[mmd->axis])
1301                                         a->cliptol[mmd->axis] = mmd->tolerance;
1302                         }
1303                 }
1304         }
1305
1306         if(sd->brush_type == GRAB_BRUSH) {
1307                 float gcenter[3];
1308
1309                 /* Find the delta */
1310                 unproject(gcenter, mouse[0], mouse[1], a->depth);
1311                 unproject(oldloc, pr_mouse[0], pr_mouse[1], a->depth);
1312                 VecSubf(a->symm.grab_delta, gcenter, oldloc);
1313         }
1314         else if(sd->brush_type == LAYER_BRUSH) {
1315                 Mesh *me= get_mesh(OBACT);
1316
1317                 if(!a->layer_disps)
1318                         a->layer_disps= MEM_callocN(sizeof(float)*me->totvert,"Layer disps");
1319         }
1320
1321         if(sd->brush_type == LAYER_BRUSH || anchored) {
1322                 Mesh *me= get_mesh(OBACT);
1323                 unsigned i;
1324  
1325                 if(!a->mesh_store) {
1326                         a->mesh_store= MEM_mallocN(sizeof(vec3f) * me->totvert, "Sculpt mesh store");
1327                         for(i = 0; i < me->totvert; ++i)
1328                                 VecCopyf(&a->mesh_store[i].x, me->mvert[i].co);
1329                 }
1330
1331                 if(anchored && a->layer_disps)
1332                         memset(a->layer_disps, 0, sizeof(float) * me->totvert);
1333
1334                 if(anchored && !a->orig_norms) {
1335                         a->orig_norms= MEM_mallocN(sizeof(short) * 3 * me->totvert, "Sculpt orig norm");
1336                         for(i = 0; i < me->totvert; ++i) {
1337                                 a->orig_norms[i][0] = me->mvert[i].no[0];
1338                                 a->orig_norms[i][1] = me->mvert[i].no[1];
1339                                 a->orig_norms[i][2] = me->mvert[i].no[2];
1340                         }
1341                 }
1342         }
1343 }
1344 void sculptmode_set_strength(const int delta)
1345 {
1346         int val = sculptmode_brush()->strength + delta;
1347         if(val < 1) val = 1;
1348         if(val > 100) val = 100;
1349         sculptmode_brush()->strength= val;
1350 }
1351
1352 static void sculpt_radialcontrol_callback(const int mode, const int val)
1353 {
1354         SculptSession *ss = sculpt_session();
1355         BrushData *br = sculptmode_brush();
1356
1357         if(mode == RADIALCONTROL_SIZE)
1358                 br->size = val;
1359         else if(mode == RADIALCONTROL_STRENGTH)
1360                 br->strength = val;
1361         else if(mode == RADIALCONTROL_ROTATION)
1362                 set_tex_angle(val);
1363
1364         ss->radialcontrol = NULL;
1365 }
1366
1367 /* Returns GL handle to brush texture */
1368 static GLuint sculpt_radialcontrol_calctex()
1369 {
1370         SculptData *sd= sculpt_data();
1371         SculptSession *ss= sculpt_session();
1372         int i, j;
1373         const int tsz = TC_SIZE;
1374         float *texdata= MEM_mallocN(sizeof(float)*tsz*tsz, "Brush preview");
1375         GLuint tex;
1376
1377         if(sd->texrept!=SCULPTREPT_3D)
1378                 sculptmode_update_tex();
1379         for(i=0; i<tsz; ++i)
1380                 for(j=0; j<tsz; ++j) {
1381                         float magn= sqrt(pow(i-tsz/2,2)+pow(j-tsz/2,2));
1382                         if(sd->texfade)
1383                                 texdata[i*tsz+j]= curve_strength(magn,tsz/2);
1384                         else
1385                                 texdata[i*tsz+j]= magn < tsz/2 ? 1 : 0;
1386                 }
1387         if(sd->texact != -1 && ss->texcache) {
1388                 for(i=0; i<tsz; ++i)
1389                         for(j=0; j<tsz; ++j) {
1390                                 const int col= ss->texcache[i*tsz+j];
1391                                 texdata[i*tsz+j]*= (((char*)&col)[0]+((char*)&col)[1]+((char*)&col)[2])/3.0f/255.0f;
1392                         }
1393         }
1394                 
1395         glGenTextures(1, &tex);
1396         glBindTexture(GL_TEXTURE_2D, tex);
1397         glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, tsz, tsz, 0, GL_ALPHA, GL_FLOAT, texdata);
1398         MEM_freeN(texdata);
1399
1400         return tex;
1401 }
1402
1403 void sculpt_radialcontrol_start(int mode)
1404 {
1405         SculptData *sd = sculpt_data();
1406         SculptSession *ss = sculpt_session();
1407         BrushData *br = sculptmode_brush();
1408         int orig=1, max=100;
1409
1410         if(mode == RADIALCONTROL_SIZE) {
1411                 orig = br->size;
1412                 max = 200;
1413         }
1414         else if(mode == RADIALCONTROL_STRENGTH) {
1415                 orig = br->strength;
1416                 max = 100;
1417         }
1418         else if(mode == RADIALCONTROL_ROTATION) {
1419                 if(sd->texact!=-1 && sd->mtex[sd->texact]) {
1420                         orig = sculpt_tex_angle();
1421                         max = 360;
1422                 }
1423                 else
1424                         mode = RADIALCONTROL_NONE;
1425         }
1426
1427         if(mode != RADIALCONTROL_NONE) {
1428                 ss->radialcontrol= radialcontrol_start(mode, sculpt_radialcontrol_callback, orig, max,
1429                                                        sculpt_radialcontrol_calctex());
1430         }
1431 }
1432
1433 void sculpt_paint_brush(char clear)
1434 {
1435         if(sculpt_data()->flags & SCULPT_DRAW_BRUSH) {
1436                 static short mvalo[2];
1437                 short mval[2];
1438                 const short rad= sculptmode_brush()->size;
1439
1440                 getmouseco_areawin(mval);
1441                 
1442                 persp(PERSP_WIN);
1443                 if(clear)
1444                         fdrawXORcirc(mval[0], mval[1], rad);
1445                 else
1446                         draw_sel_circle(mval, mvalo, rad, rad, 0);
1447                 
1448                 mvalo[0]= mval[0];
1449                 mvalo[1]= mval[1];
1450         }
1451 }
1452
1453 void sculptmode_selectbrush_menu(void)
1454 {
1455         SculptData *sd= sculpt_data();
1456         int val;
1457         
1458         pupmenu_set_active(sd->brush_type);
1459         
1460         val= pupmenu("Select Brush%t|Draw|Smooth|Pinch|Inflate|Grab|Layer|Flatten");
1461
1462         if(val>0) {
1463                 sd->brush_type= val;
1464
1465                 BIF_undo_push("Brush type");
1466                 
1467                 allqueue(REDRAWVIEW3D, 1);
1468                 allqueue(REDRAWBUTSEDIT, 1);
1469         }
1470 }
1471
1472 void sculptmode_update_all_projverts(float *vertcosnos)
1473 {
1474         SculptSession *ss = sculpt_session();
1475         Mesh *me= get_mesh(OBACT);
1476         unsigned i;
1477
1478         if(!ss->projverts)
1479                 ss->projverts = MEM_mallocN(sizeof(ProjVert)*me->totvert,"ProjVerts");
1480
1481         for(i=0; i<me->totvert; ++i) {
1482                 project(vertcosnos ? &vertcosnos[i * 6] : me->mvert[i].co, ss->projverts[i].co);
1483                 ss->projverts[i].inside= 0;
1484         }
1485 }
1486
1487 void sculptmode_draw_wires(SculptSession *ss, int only_damaged, Mesh *me)
1488 {
1489         int i;
1490
1491         bglPolygonOffset(1.0);
1492         glDepthMask(0);
1493         BIF_ThemeColor((OBACT==OBACT)?TH_ACTIVE:TH_SELECT);
1494
1495         for(i=0; i<me->totedge; i++) {
1496                 MEdge *med= &me->medge[i];
1497
1498                 if((!only_damaged || (ss->projverts[med->v1].inside || ss->projverts[med->v2].inside)) &&
1499                    (med->flag & ME_EDGEDRAW)) {
1500                         glDrawElements(GL_LINES, 2, GL_UNSIGNED_INT, &med->v1);
1501                 }
1502         }
1503
1504         glDepthMask(1);
1505         bglPolygonOffset(0.0);
1506 }
1507
1508 void sculptmode_draw_mesh(int only_damaged) 
1509 {
1510         Mesh *me= get_mesh(OBACT);
1511         int i, j, dt, drawCurrentMat = 1, matnr= -1;
1512         SculptSession *ss = sculpt_session();
1513
1514         persp(PERSP_VIEW);
1515         mymultmatrix(OBACT->obmat);
1516         glEnable(GL_DEPTH_TEST);
1517         glEnable(GL_LIGHTING);
1518         GPU_set_object_materials(G.scene, OBACT, 0, NULL);
1519         glEnable(GL_CULL_FACE);
1520
1521         glShadeModel(GL_SMOOTH);
1522
1523         glVertexPointer(3, GL_FLOAT, sizeof(MVert), &me->mvert[0].co);
1524         glNormalPointer(GL_SHORT, sizeof(MVert), &me->mvert[0].no);
1525
1526         dt= MIN2(G.vd->drawtype, OBACT->dt);
1527         if(dt==OB_WIRE)
1528                 glColorMask(0,0,0,0);
1529
1530         for(i=0; i<me->totface; ++i) {
1531                 MFace *f= &me->mface[i];
1532                 char inside= 0;
1533                 int new_matnr= f->mat_nr + 1;
1534                 
1535                 if(new_matnr != matnr)
1536                         drawCurrentMat= GPU_enable_material(matnr = new_matnr, NULL);
1537                 
1538                 /* If only_damaged!=0, only draw faces that are partially
1539                    inside the area(s) modified by the brush */
1540                 if(only_damaged) {
1541                         for(j=0; j<(f->v4?4:3); ++j) {
1542                                 if(ss->projverts[*((&f->v1)+j)].inside) {
1543                                         inside= 1;
1544                                         break;
1545                                 }
1546                         }
1547                 }
1548                 else
1549                         inside= 1;
1550                         
1551                 if(inside && drawCurrentMat)
1552                         glDrawElements(f->v4?GL_QUADS:GL_TRIANGLES, f->v4?4:3, GL_UNSIGNED_INT, &f->v1);
1553         }
1554
1555         glDisable(GL_CULL_FACE);
1556         glDisable(GL_LIGHTING);
1557         glColorMask(1,1,1,1);
1558
1559         if(dt==OB_WIRE || (OBACT->dtx & OB_DRAWWIRE))
1560                 sculptmode_draw_wires(ss, only_damaged, me);
1561
1562         glDisable(GL_DEPTH_TEST);
1563 }
1564
1565 void sculptmode_correct_state(void)
1566 {
1567         if(!sculpt_session())
1568                 sculpt_init_session();
1569
1570         glEnableClientState(GL_VERTEX_ARRAY);
1571         glEnableClientState(GL_NORMAL_ARRAY);
1572         
1573         if(!sculpt_session()->vertex_users) calc_vertex_users();
1574 }
1575
1576 /* Checks whether full update mode (slower) needs to be used to work with modifiers */
1577 char sculpt_modifiers_active(Object *ob)
1578 {
1579         ModifierData *md;
1580         
1581         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1582                 if(md->mode & eModifierMode_Realtime)
1583                         return 1;
1584         }
1585         
1586         return 0;
1587 }
1588
1589 void sculpt(void)
1590 {
1591         SculptData *sd= sculpt_data();
1592         SculptSession *ss= sculpt_session();
1593         Object *ob= OBACT;
1594         /* lastSigMouse is for the rake, to store the last place the mouse movement was significant */
1595         short mouse[2], mvalo[2], lastSigMouse[2],firsttime=1, mousebut;
1596         short modifier_calculations= 0;
1597         BrushAction *a = MEM_callocN(sizeof(BrushAction), "brush action");
1598         short spacing= 32000;
1599         int scissor_box[4];
1600         float offsetRot;
1601         int smooth_stroke = 0, i;
1602         int anchored;
1603
1604         if(!(G.f & G_SCULPTMODE) || G.obedit || !ob || ob->id.lib || !get_mesh(ob) || (get_mesh(ob)->totface == 0))
1605                 return;
1606         if(!(ob->lay & G.vd->lay))
1607                 error("Active object is not in this layer");
1608         if(ob_get_keyblock(ob)) {
1609                 if(!(ob->shapeflag & OB_SHAPE_LOCK)) {
1610                         error("Cannot sculpt on unlocked shape key");
1611                         return;
1612                 }
1613         }
1614         
1615         if(!ss) {
1616                 sculpt_init_session();
1617                 ss= sd->session;
1618         }
1619
1620         anchored = sculptmode_brush()->flag & SCULPT_BRUSH_ANCHORED;
1621         smooth_stroke = (sd->flags & SCULPT_INPUT_SMOOTH) && (sd->brush_type != GRAB_BRUSH) && !anchored;
1622
1623         if(smooth_stroke)
1624                 sculpt_stroke_new(256);
1625
1626         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
1627         ss->damaged_verts.first = ss->damaged_verts.last = NULL;
1628         ss->vertexcosnos = NULL;
1629
1630         /* Check that vertex users are up-to-date */
1631         if(ob != active_ob || !ss->vertex_users || ss->vertex_users_size != get_mesh(ob)->totvert) {
1632                 sculpt_vertexusers_free(ss);
1633                 calc_vertex_users();
1634                 if(ss->projverts)
1635                         MEM_freeN(ss->projverts);
1636                 ss->projverts = NULL;
1637                 active_ob= ob;
1638         }
1639                 
1640         glEnableClientState(GL_VERTEX_ARRAY);
1641         glEnableClientState(GL_NORMAL_ARRAY);
1642
1643         persp(PERSP_VIEW);
1644         
1645         getmouseco_areawin(mvalo);
1646
1647         /* Init texture
1648            FIXME: Shouldn't be doing this every time! */
1649         if(sd->texrept!=SCULPTREPT_3D)
1650                 sculptmode_update_tex();
1651
1652         getmouseco_areawin(mouse);
1653         mvalo[0]= mouse[0];
1654         mvalo[1]= mouse[1];
1655         lastSigMouse[0]=mouse[0];
1656         lastSigMouse[1]=mouse[1];
1657         mousebut = L_MOUSE;
1658
1659         /* If modifier_calculations is true, then extra time must be spent
1660            updating the mesh. This takes a *lot* longer, so it's worth
1661            skipping if the modifier stack is empty. */
1662         modifier_calculations= sculpt_modifiers_active(ob);
1663
1664         init_sculptmatrices();
1665
1666         if(modifier_calculations)
1667                 ss->vertexcosnos= mesh_get_mapped_verts_nors(ob);
1668         sculptmode_update_all_projverts(ss->vertexcosnos);
1669
1670         /* Set scaling adjustment */
1671         a->scale[0]= 1.0f / ob->size[0];
1672         a->scale[1]= 1.0f / ob->size[1];
1673         a->scale[2]= 1.0f / ob->size[2];
1674
1675         /* Capture original copy */
1676         if(sd->flags & SCULPT_DRAW_FAST)
1677                 glAccum(GL_LOAD, 1);
1678
1679         /* Get original scissor box */
1680         glGetIntegerv(GL_SCISSOR_BOX, scissor_box);
1681         
1682         /* For raking, get the original angle*/
1683         offsetRot=sculpt_tex_angle();
1684         
1685         while (get_mbut() & mousebut) {
1686                 getmouseco_areawin(mouse);
1687                 /* If rake, and the mouse has moved over 10 pixels (euclidean) (prevents jitter) then get the new angle */
1688                 if (sd->rake && (pow(lastSigMouse[0]-mouse[0],2)+pow(lastSigMouse[1]-mouse[1],2))>100){
1689                         /*Nasty looking, but just orig + new angle really*/
1690                         set_tex_angle(offsetRot+180.+to_deg(atan2((float)(mouse[1]-lastSigMouse[1]),(float)(mouse[0]-lastSigMouse[0]))));
1691                         lastSigMouse[0]=mouse[0];
1692                         lastSigMouse[1]=mouse[1];
1693                 }
1694                 
1695                 if(firsttime || mouse[0]!=mvalo[0] || mouse[1]!=mvalo[1] ||
1696                    sculptmode_brush()->flag & SCULPT_BRUSH_AIRBRUSH) {
1697                         a->firsttime = firsttime;
1698                         firsttime= 0;
1699
1700                         if(smooth_stroke)
1701                                 sculpt_stroke_add_point(mouse[0], mouse[1]);
1702
1703                         spacing+= sqrt(pow(mvalo[0]-mouse[0],2)+pow(mvalo[1]-mouse[1],2));
1704
1705                         if(modifier_calculations && !ss->vertexcosnos)
1706                                 ss->vertexcosnos= mesh_get_mapped_verts_nors(ob);
1707
1708                         if(G.scene->sculptdata.brush_type != GRAB_BRUSH) {
1709                                 if(anchored) {
1710                                         Mesh *me = get_mesh(ob);
1711                                         
1712                                         /* Restore the mesh before continuing with anchored stroke */
1713                                         if(a->mesh_store) {
1714                                                 for(i = 0; i < me->totvert; ++i) {
1715                                                         VecCopyf(me->mvert[i].co, &a->mesh_store[i].x);
1716                                                         me->mvert[i].no[0] = a->orig_norms[i][0];
1717                                                         me->mvert[i].no[1] = a->orig_norms[i][1];
1718                                                         me->mvert[i].no[2] = a->orig_norms[i][2];
1719                                                 }
1720                                         }
1721                                         
1722                                         do_symmetrical_brush_actions(a, mouse, NULL);
1723                                 }
1724                                 else {
1725                                         if(smooth_stroke) {
1726                                                 sculpt_stroke_apply(a);
1727                                         }
1728                                         else if(sd->spacing==0 || spacing>sd->spacing) {
1729                                                 do_symmetrical_brush_actions(a, mouse, NULL);
1730                                                 spacing= 0;
1731                                         }
1732                                 }
1733                         }
1734                         else {
1735                                 do_symmetrical_brush_actions(a, mouse, mvalo);
1736                                 unproject(sd->pivot, mouse[0], mouse[1], a->depth);
1737                         }
1738
1739                         if(modifier_calculations || ob_get_keyblock(ob))
1740                                 DAG_object_flush_update(G.scene, ob, OB_RECALC_DATA);
1741
1742                         if(modifier_calculations || sd->brush_type == GRAB_BRUSH || !(sd->flags & SCULPT_DRAW_FAST)) {
1743                                 calc_damaged_verts(&ss->damaged_verts, a);
1744                                 scrarea_do_windraw(curarea);
1745                                 screen_swapbuffers();
1746                         } else { /* Optimized drawing */
1747                                 calc_damaged_verts(&ss->damaged_verts, a);
1748
1749                                 /* Draw the stored image to the screen */
1750                                 glAccum(GL_RETURN, 1);
1751
1752                                 sculpt_clear_damaged_areas(ss);
1753                                 
1754                                 /* Draw all the polygons that are inside the modified area(s) */
1755                                 glScissor(scissor_box[0], scissor_box[1], scissor_box[2], scissor_box[3]);
1756                                 sculptmode_draw_mesh(1);
1757                                 glAccum(GL_LOAD, 1);
1758
1759                                 projverts_clear_inside(ss);
1760
1761                                 persp(PERSP_WIN);
1762                                 glDisable(GL_DEPTH_TEST);
1763                                 
1764                                 /* Draw cursor */
1765                                 if(sculpt_data()->flags & SCULPT_DRAW_BRUSH)
1766                                         fdrawXORcirc((float)mouse[0],(float)mouse[1],sculptmode_brush()->size);
1767                                 if(smooth_stroke)
1768                                         sculpt_stroke_draw();
1769                                 
1770                                 myswapbuffers();
1771                         }
1772
1773                         BLI_freelistN(&ss->damaged_rects);
1774                         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
1775         
1776                         mvalo[0]= mouse[0];
1777                         mvalo[1]= mouse[1];
1778
1779                         if(ss->vertexcosnos) {
1780                                 MEM_freeN(ss->vertexcosnos);
1781                                 ss->vertexcosnos= NULL;
1782                         }
1783
1784                 }
1785                 else BIF_wait_for_statechange();
1786         }
1787
1788         /* Set the rotation of the brush back to what it was before any rake */
1789         set_tex_angle(offsetRot);
1790         
1791         if(smooth_stroke) {
1792                 sculpt_stroke_apply_all(a);
1793                 calc_damaged_verts(&ss->damaged_verts, a);
1794                 BLI_freelistN(&ss->damaged_rects);
1795         }
1796
1797         if(a->layer_disps) MEM_freeN(a->layer_disps);
1798         if(a->mesh_store) MEM_freeN(a->mesh_store);
1799         if(a->orig_norms) MEM_freeN(a->orig_norms);
1800         for(i=0; i<8; ++i)
1801                 BLI_freelistN(&a->grab_active_verts[i]);
1802         MEM_freeN(a);
1803         sculpt_stroke_free();
1804
1805         sculpt_undo_push(G.scene->sculptdata.brush_type);
1806
1807         if(G.vd->depths) G.vd->depths->damaged= 1;
1808         
1809         allqueue(REDRAWVIEW3D, 0);
1810 }
1811
1812 void sculpt_undo_push(const short brush_type)
1813 {
1814         switch(brush_type) {
1815         case DRAW_BRUSH:
1816                 BIF_undo_push("Draw Brush"); break;
1817         case SMOOTH_BRUSH:
1818                 BIF_undo_push("Smooth Brush"); break;
1819         case PINCH_BRUSH:
1820                 BIF_undo_push("Pinch Brush"); break;
1821         case INFLATE_BRUSH:
1822                 BIF_undo_push("Inflate Brush"); break;
1823         case GRAB_BRUSH:
1824                 BIF_undo_push("Grab Brush"); break;
1825         case LAYER_BRUSH:
1826                 BIF_undo_push("Layer Brush"); break;
1827         case FLATTEN_BRUSH:
1828                 BIF_undo_push("Flatten Brush"); break;
1829         default:
1830                 BIF_undo_push("Sculpting"); break;
1831         }
1832 }
1833
1834 void set_sculptmode(void)
1835 {
1836         if(G.f & G_SCULPTMODE) {
1837                 Mesh *me= get_mesh(OBACT);
1838                 
1839                 G.f &= ~G_SCULPTMODE;
1840
1841                 sculptsession_free(G.scene);
1842                 if(me && me->pv) 
1843                         mesh_pmv_off(OBACT, me);
1844         } 
1845         else {
1846                 G.f |= G_SCULPTMODE;
1847
1848                 /* Called here to sanity-check the brush */
1849                 sculptmode_brush();
1850
1851                 sculpt_init_session();
1852                 
1853                 glEnableClientState(GL_VERTEX_ARRAY);
1854                 glEnableClientState(GL_NORMAL_ARRAY);
1855         }
1856         
1857         active_ob= NULL;
1858
1859         allqueue(REDRAWVIEW3D, 1);
1860         allqueue(REDRAWBUTSEDIT, 0);
1861 }
1862
1863 /* Partial Mesh Visibility */
1864
1865 /* mode: 0=hide outside selection, 1=hide inside selection */
1866 static void sculptmode_do_pmv(Object *ob, rcti *hb_2d, int mode)
1867 {
1868         Mesh *me= get_mesh(ob);
1869         float hidebox[6][3];
1870         vec3f plane_normals[4];
1871         float plane_ds[4];
1872         unsigned i, j;
1873         unsigned ndx_show, ndx_hide;
1874         MVert *nve;
1875         unsigned face_cnt_show= 0, face_ndx_show= 0;
1876         unsigned edge_cnt_show= 0, edge_ndx_show= 0;
1877         unsigned *old_map= NULL;
1878         const unsigned SHOW= 0, HIDE=1;
1879
1880         /* Convert hide box from 2D to 3D */
1881         unproject(hidebox[0], hb_2d->xmin, hb_2d->ymax, 1);
1882         unproject(hidebox[1], hb_2d->xmax, hb_2d->ymax, 1);
1883         unproject(hidebox[2], hb_2d->xmax, hb_2d->ymin, 1);
1884         unproject(hidebox[3], hb_2d->xmin, hb_2d->ymin, 1);
1885         unproject(hidebox[4], hb_2d->xmin, hb_2d->ymax, 0);
1886         unproject(hidebox[5], hb_2d->xmax, hb_2d->ymin, 0);
1887         
1888         /* Calculate normals for each side of hide box */
1889         CalcNormFloat(hidebox[0], hidebox[1], hidebox[4], &plane_normals[0].x);
1890         CalcNormFloat(hidebox[1], hidebox[2], hidebox[5], &plane_normals[1].x);
1891         CalcNormFloat(hidebox[2], hidebox[3], hidebox[5], &plane_normals[2].x);
1892         CalcNormFloat(hidebox[3], hidebox[0], hidebox[4], &plane_normals[3].x);
1893         
1894         /* Calculate D for each side of hide box */
1895         for(i= 0; i<4; ++i)
1896                 plane_ds[i]= hidebox[i][0]*plane_normals[i].x + hidebox[i][1]*plane_normals[i].y +
1897                         hidebox[i][2]*plane_normals[i].z;
1898         
1899         /* Add partial visibility to mesh */
1900         if(!me->pv) {
1901                 me->pv= MEM_callocN(sizeof(PartialVisibility),"PartialVisibility");
1902         } else {
1903                 old_map= MEM_callocN(sizeof(unsigned)*me->pv->totvert,"PMV oldmap");
1904                 for(i=0; i<me->pv->totvert; ++i) {
1905                         old_map[i]= me->pv->vert_map[i]<me->totvert?0:1;
1906                 }
1907                 mesh_pmv_revert(ob, me);
1908         }
1909         
1910         /* Kill sculpt data */
1911         active_ob= NULL;
1912         
1913         /* Initalize map with which verts are to be hidden */
1914         me->pv->vert_map= MEM_mallocN(sizeof(unsigned)*me->totvert, "PMV vertmap");
1915         me->pv->totvert= me->totvert;
1916         me->totvert= 0;
1917         for(i=0; i<me->pv->totvert; ++i) {
1918                 me->pv->vert_map[i]= mode ? HIDE:SHOW;
1919                 for(j=0; j<4; ++j) {
1920                         if(me->mvert[i].co[0] * plane_normals[j].x +
1921                            me->mvert[i].co[1] * plane_normals[j].y +
1922                            me->mvert[i].co[2] * plane_normals[j].z < plane_ds[j] ) {
1923                                 me->pv->vert_map[i]= mode ? SHOW:HIDE; /* Vert is outside the hide box */
1924                                 break;
1925                         }
1926                 }
1927                 if(old_map && old_map[i]) me->pv->vert_map[i]= 1;
1928                 if(!me->pv->vert_map[i]) ++me->totvert;
1929
1930         }
1931         if(old_map) MEM_freeN(old_map);
1932
1933         /* Find out how many faces to show */
1934         for(i=0; i<me->totface; ++i) {
1935                 if(!me->pv->vert_map[me->mface[i].v1] &&
1936                    !me->pv->vert_map[me->mface[i].v2] &&
1937                    !me->pv->vert_map[me->mface[i].v3]) {
1938                         if(me->mface[i].v4) {
1939                                 if(!me->pv->vert_map[me->mface[i].v4])
1940                                         ++face_cnt_show;
1941                         }
1942                         else ++face_cnt_show;
1943                 }
1944         }
1945         /* Find out how many edges to show */
1946         for(i=0; i<me->totedge; ++i) {
1947                 if(!me->pv->vert_map[me->medge[i].v1] &&
1948                    !me->pv->vert_map[me->medge[i].v2])
1949                         ++edge_cnt_show;
1950         }
1951
1952         /* Create new vert array and reset each vert's map with map[old]=new index */
1953         nve= MEM_mallocN(sizeof(MVert)*me->pv->totvert, "PMV verts");
1954         ndx_show= 0; ndx_hide= me->totvert;
1955         for(i=0; i<me->pv->totvert; ++i) {
1956                 if(me->pv->vert_map[i]) {
1957                         me->pv->vert_map[i]= ndx_hide;
1958                         nve[me->pv->vert_map[i]]= me->mvert[i];
1959                         ++ndx_hide;
1960                 } else {
1961                         me->pv->vert_map[i]= ndx_show;
1962                         nve[me->pv->vert_map[i]]= me->mvert[i];
1963                         ++ndx_show;
1964                 }
1965         }
1966         CustomData_free_layer_active(&me->vdata, CD_MVERT, me->pv->totvert);
1967         me->mvert= CustomData_add_layer(&me->vdata, CD_MVERT, CD_ASSIGN, nve, me->totvert);
1968
1969         /* Create new face array */
1970         me->pv->old_faces= me->mface;
1971         me->pv->totface= me->totface;
1972         me->mface= MEM_mallocN(sizeof(MFace)*face_cnt_show, "PMV faces");
1973         for(i=0; i<me->totface; ++i) {
1974                 MFace *pr_f= &me->pv->old_faces[i];
1975                 char show= 0;
1976
1977                 if(me->pv->vert_map[pr_f->v1] < me->totvert &&
1978                    me->pv->vert_map[pr_f->v2] < me->totvert &&
1979                    me->pv->vert_map[pr_f->v3] < me->totvert) {
1980                         if(pr_f->v4) {
1981                                 if(me->pv->vert_map[pr_f->v4] < me->totvert)
1982                                         show= 1;
1983                         }
1984                         else show= 1;
1985                 }
1986
1987                 if(show) {
1988                         MFace *cr_f= &me->mface[face_ndx_show];
1989                         *cr_f= *pr_f;
1990                         cr_f->v1= me->pv->vert_map[pr_f->v1];
1991                         cr_f->v2= me->pv->vert_map[pr_f->v2];
1992                         cr_f->v3= me->pv->vert_map[pr_f->v3];
1993                         cr_f->v4= pr_f->v4 ? me->pv->vert_map[pr_f->v4] : 0;
1994                         test_index_face(cr_f,NULL,0,pr_f->v4?4:3);
1995                         ++face_ndx_show;
1996                 }
1997         }
1998         me->totface= face_cnt_show;
1999         CustomData_set_layer(&me->fdata, CD_MFACE, me->mface);
2000
2001         /* Create new edge array */
2002         me->pv->old_edges= me->medge;
2003         me->pv->totedge= me->totedge;
2004         me->medge= MEM_mallocN(sizeof(MEdge)*edge_cnt_show, "PMV edges");
2005         me->pv->edge_map= MEM_mallocN(sizeof(int)*me->pv->totedge,"PMV edgemap");
2006         for(i=0; i<me->totedge; ++i) {
2007                 if(me->pv->vert_map[me->pv->old_edges[i].v1] < me->totvert &&
2008                    me->pv->vert_map[me->pv->old_edges[i].v2] < me->totvert) {
2009                         MEdge *cr_e= &me->medge[edge_ndx_show];
2010                         me->pv->edge_map[i]= edge_ndx_show;
2011                         *cr_e= me->pv->old_edges[i];
2012                         cr_e->v1= me->pv->vert_map[me->pv->old_edges[i].v1];
2013                         cr_e->v2= me->pv->vert_map[me->pv->old_edges[i].v2];
2014                         ++edge_ndx_show;
2015                 }
2016                 else me->pv->edge_map[i]= -1;
2017         }
2018         me->totedge= edge_cnt_show;
2019         CustomData_set_layer(&me->edata, CD_MEDGE, me->medge);
2020
2021         DAG_object_flush_update(G.scene, OBACT, OB_RECALC_DATA);
2022 }
2023
2024 static rcti sculptmode_pmv_box()
2025 {
2026         short down[2], mouse[2];
2027         rcti ret;
2028
2029         getmouseco_areawin(down);
2030
2031         while((get_mbut()&L_MOUSE) || (get_mbut()&R_MOUSE)) {
2032                 getmouseco_areawin(mouse);
2033
2034                 scrarea_do_windraw(curarea);
2035
2036                 persp(PERSP_WIN);
2037                 glLineWidth(2);
2038                 setlinestyle(2);
2039                 sdrawXORline(down[0],down[1],mouse[0],down[1]);
2040                 sdrawXORline(mouse[0],down[1],mouse[0],mouse[1]);
2041                 sdrawXORline(mouse[0],mouse[1],down[0],mouse[1]);
2042                 sdrawXORline(down[0],mouse[1],down[0],down[1]);
2043                 setlinestyle(0);
2044                 glLineWidth(1);
2045                 persp(PERSP_VIEW);
2046
2047                 screen_swapbuffers();
2048                 backdrawview3d(0);
2049         }
2050
2051         ret.xmin= down[0]<mouse[0]?down[0]:mouse[0];
2052         ret.ymin= down[1]<mouse[1]?down[1]:mouse[1];
2053         ret.xmax= down[0]>mouse[0]?down[0]:mouse[0];
2054         ret.ymax= down[1]>mouse[1]?down[1]:mouse[1];
2055         return ret;
2056 }
2057
2058 void sculptmode_pmv(int mode)
2059 {
2060         Object *ob= OBACT;
2061         rcti hb_2d;
2062         
2063         if(ob_get_key(ob)) {
2064                 error("Cannot hide mesh with shape keys enabled");
2065                 return;
2066         }
2067         
2068         hb_2d= sculptmode_pmv_box(); /* Get 2D hide box */
2069         
2070         sculptmode_correct_state();
2071
2072         waitcursor(1);
2073
2074         if(hb_2d.xmax-hb_2d.xmin > 3 && hb_2d.ymax-hb_2d.ymin > 3) {
2075                 init_sculptmatrices();
2076
2077                 sculptmode_do_pmv(ob,&hb_2d,mode);
2078         }
2079         else mesh_pmv_off(ob, get_mesh(ob));
2080
2081         scrarea_do_windraw(curarea);
2082
2083         BIF_undo_push("Partial mesh hide");
2084
2085         waitcursor(0);
2086 }