source/blender/src/sculptmode.c

   1 /*
   2  * $Id$
   3  *
   4  * ***** BEGIN GPL LICENSE BLOCK *****
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version 2
   9  * of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software  Foundation,
  18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19  *
  20  * The Original Code is Copyright (C) 2006 by Nicholas Bishop
  21  * All rights reserved.
  22  *
  23  * The Original Code is: all of this file.
  24  *
  25  * Contributor(s): none yet.
  26  *
  27  * ***** END GPL LICENSE BLOCK *****
  28  *
  29  * Implements the Sculpt Mode tools
  30  *
  31  * BDR_sculptmode.h
  32  *
  33  */
  34
  35 #include "GHOST_Types.h"
  36
  37 #include "MEM_guardedalloc.h"
  38
  39 #include "BLI_arithb.h"
  40 #include "BLI_blenlib.h"
  41 #include "BLI_dynstr.h"
  42
  43 #include "DNA_armature_types.h"
  44 #include "DNA_brush_types.h"
  45 #include "DNA_image_types.h"
  46 #include "DNA_key_types.h"
  47 #include "DNA_mesh_types.h"
  48 #include "DNA_meshdata_types.h"
  49 #include "DNA_modifier_types.h"
  50 #include "DNA_object_types.h"
  51 #include "DNA_screen_types.h"
  52 #include "DNA_scene_types.h"
  53 #include "DNA_texture_types.h"
  54 #include "DNA_view3d_types.h"
  55 #include "DNA_userdef_types.h"
  56
  57 #include "BKE_customdata.h"
  58 #include "BKE_DerivedMesh.h"
  59 #include "BKE_depsgraph.h"
  60 #include "BKE_global.h"
  61 #include "BKE_image.h"
  62 #include "BKE_key.h"
  63 #include "BKE_library.h"
  64 #include "BKE_main.h"
  65 #include "BKE_mesh.h"
  66 #include "BKE_modifier.h"
  67 #include "BKE_texture.h"
  68 #include "BKE_utildefines.h"
  69
  70 #include "BIF_editkey.h"
  71 #include "BIF_editview.h"
  72 #include "BIF_glutil.h"
  73 #include "BIF_gl.h"
  74 #include "BIF_interface.h"
  75 #include "BIF_mywindow.h"
  76 #include "BIF_resources.h"
  77 #include "BIF_screen.h"
  78 #include "BIF_space.h"
  79 #include "BIF_toolbox.h"
  80
  81 #include "BDR_drawobject.h"
  82 #include "BDR_sculptmode.h"
  83
  84 #include "BSE_drawview.h"
  85 #include "BSE_edit.h"
  86 #include "BSE_view.h"
  87
  88 #include "IMB_imbuf_types.h"
  89
  90 #include "blendef.h"
  91 #include "multires.h"
  92 #include "mydevice.h"
  93
  94 #include "RE_render_ext.h"
  95 #include "RE_shader_ext.h" /*for multitex_ext*/
  96
  97 #include <math.h>
  98 #include <stdlib.h>
  99 #include <string.h>
 100
 101 /* Number of vertices to average in order to determine the flatten distance */
 102 #define FLATTEN_SAMPLE_SIZE 10
 103
 104 /* ===== STRUCTS =====
 105  *
 106  */
 107
 108 /* Used by vertex_users to store face indices in a list */
 109 typedef struct IndexNode {
 110         struct IndexNode* next,* prev;
 111         int Index;
 112 } IndexNode;
 113
 114
 115 /* ActiveData stores an Index into the mvert array of Mesh, plus Fade, which
 116    stores how far the vertex is from the brush center, scaled to the range [0,1]. */
 117 typedef struct ActiveData {
 118         struct ActiveData *next, *prev;
 119         unsigned int Index;
 120         float Fade;
 121         float dist;
 122 } ActiveData;
 123
 124 typedef struct GrabData {
 125         char firsttime;
 126         ListBase active_verts[8];
 127         unsigned char index;
 128         vec3f delta, delta_symm;
 129         float depth;
 130 } GrabData;
 131
 132 typedef struct EditData {
 133         vec3f center;
 134         float size;
 135         char flip;
 136         short mouse[2];
 137
 138         /* Adjust brush strength along each axis
 139            to adjust for object scaling */
 140         float scale[3];
 141
 142         /* View normals */
 143         vec3f up, right, out;
 144
 145         GrabData *grabdata;
 146         float *layer_disps;
 147         vec3f *layer_store;
 148
 149         char clip[3];
 150         float cliptol[3];
 151
 152         char symm;
 153 } EditData;
 154
 155 typedef struct RectNode {
 156         struct RectNode *next, *prev;
 157         rcti r;
 158 } RectNode;
 159
 160 /* Used to store to 2D screen coordinates of each vertex in the mesh. */
 161 typedef struct ProjVert {
 162         short co[2];
 163
 164         /* Used to mark whether a vertex is inside a rough bounding box
 165            containing the brush. */
 166         char inside;
 167 } ProjVert;
 168
 169 static ProjVert *projverts= NULL;
 170 static Object *active_ob= NULL;
 171
 172 SculptData *sculpt_data(void)
 173 {
 174         return &G.scene->sculptdata;
 175 }
 176
 177 void sculpt_init_session(void);
 178 void init_editdata(EditData *e, short *, short *);
 179
 180 SculptSession *sculpt_session(void)
 181 {
 182         if(!sculpt_data()->session)
 183                 sculpt_init_session();
 184         return sculpt_data()->session;
 185 }
 186
 187 /* ===== MEMORY =====
 188  *
 189  * Allocate/initialize/free data
 190  */
 191
 192 /* Initialize 'permanent' sculpt data that is saved with file kept after
 193    switching out of sculptmode. */
 194 void sculptmode_init(Scene *sce)
 195 {
 196         SculptData *sd;
 197
 198         if(!sce) {
 199                 error("Unable to initialize sculptmode: bad scene");
 200                 return;
 201         }
 202
 203         sd= &sce->sculptdata;
 204
 205         memset(sd, 0, sizeof(SculptData));
 206
 207         sd->drawbrush.size = sd->smoothbrush.size = sd->pinchbrush.size =
 208                 sd->inflatebrush.size = sd->grabbrush.size =
 209                 sd->layerbrush.size = sd->flattenbrush.size = 50;
 210         sd->drawbrush.strength = sd->smoothbrush.strength =
 211                 sd->pinchbrush.strength = sd->inflatebrush.strength =
 212                 sd->grabbrush.strength = sd->layerbrush.strength =
 213                 sd->flattenbrush.strength = 25;
 214         sd->drawbrush.dir = sd->pinchbrush.dir = sd->inflatebrush.dir = sd->layerbrush.dir= 1;
 215         sd->drawbrush.airbrush = sd->smoothbrush.airbrush =
 216                 sd->pinchbrush.airbrush = sd->inflatebrush.airbrush =
 217                 sd->layerbrush.airbrush = sd->flattenbrush.airbrush = 0;
 218         sd->drawbrush.view= 0;
 219         sd->brush_type= DRAW_BRUSH;
 220         sd->texact= -1;
 221         sd->texfade= 1;
 222         sd->averaging= 1;
 223         sd->texsep= 0;
 224         sd->texrept= SCULPTREPT_DRAG;
 225         sd->flags= SCULPT_DRAW_BRUSH;
 226         sd->tablet_size=3;
 227         sd->tablet_strength=10;
 228         sd->rake=0;
 229 }
 230
 231 void sculptmode_free_session(Scene *);
 232 void sculpt_init_session(void)
 233 {
 234         if(sculpt_data()->session)
 235                 sculptmode_free_session(G.scene);
 236         sculpt_data()->session= MEM_callocN(sizeof(SculptSession), "SculptSession");
 237 }
 238
 239 void sculptmode_free_vertexusers(SculptSession *ss)
 240 {
 241         if(ss && ss->vertex_users){
 242                 MEM_freeN(ss->vertex_users);
 243                 MEM_freeN(ss->vertex_users_mem);
 244                 ss->vertex_users= NULL;
 245                 ss->vertex_users_mem= NULL;
 246                 ss->vertex_users_size= 0;
 247         }
 248 }
 249
 250 void sculptmode_propset_end(SculptSession *ss, int);
 251 void sculptmode_free_session(Scene *sce)
 252 {
 253         SculptSession *ss= sce->sculptdata.session;
 254         if(ss) {
 255                 sculptmode_free_vertexusers(ss);
 256                 if(ss->texcache)
 257                         MEM_freeN(ss->texcache);
 258                 sculptmode_propset_end(ss, 1);
 259                 MEM_freeN(ss);
 260                 sce->sculptdata.session= NULL;
 261         }
 262 }
 263
 264 void sculptmode_free_all(Scene *sce)
 265 {
 266         SculptData *sd= &sce->sculptdata;
 267         int a;
 268
 269         sculptmode_free_session(sce);
 270
 271         for(a=0; a<MAX_MTEX; a++) {
 272                 MTex *mtex= sd->mtex[a];
 273                 if(mtex) {
 274                         if(mtex->tex) mtex->tex->id.us--;
 275                         MEM_freeN(mtex);
 276                 }
 277         }
 278 }
 279
 280 /* vertex_users is an array of Lists that store all the faces that use a
 281    particular vertex. vertex_users is in the same order as mesh.mvert */
 282 void calc_vertex_users()
 283 {
 284         SculptSession *ss= sculpt_session();
 285         int i,j;
 286         IndexNode *node= NULL;
 287         Mesh *me= get_mesh(OBACT);
 288
 289         sculptmode_free_vertexusers(ss);
 290
 291         /* For efficiency, use vertex_users_mem as a memory pool (may be larger
 292            than necessary if mesh has triangles, but only one alloc is needed.) */
 293         ss->vertex_users= MEM_callocN(sizeof(ListBase) * me->totvert, "vertex_users");
 294         ss->vertex_users_size= me->totvert;
 295         ss->vertex_users_mem= MEM_callocN(sizeof(IndexNode)*me->totface*4, "vertex_users_mem");
 296         node= ss->vertex_users_mem;
 297
 298         /* Find the users */
 299         for(i=0; i<me->totface; ++i){
 300                 for(j=0; j<(me->mface[i].v4?4:3); ++j, ++node) {
 301                         node->Index=i;
 302                         BLI_addtail(&ss->vertex_users[((unsigned int*)(&me->mface[i]))[j]], node);
 303                 }
 304         }
 305 }
 306
 307 /* ===== INTERFACE =====
 308  */
 309
 310 void sculptmode_rem_tex(void *junk0,void *junk1)
 311 {
 312         MTex *mtex= G.scene->sculptdata.mtex[G.scene->sculptdata.texact];
 313         if(mtex) {
 314                 SculptSession *ss= sculpt_session();
 315                 if(mtex->tex) mtex->tex->id.us--;
 316                 MEM_freeN(mtex);
 317                 G.scene->sculptdata.mtex[G.scene->sculptdata.texact]= NULL;
 318                 /* Clear brush preview */
 319                 if(ss->texcache) {
 320                         MEM_freeN(ss->texcache);
 321                         ss->texcache= NULL;
 322                 }
 323                 BIF_undo_push("Unlink brush texture");
 324                 allqueue(REDRAWBUTSEDIT, 0);
 325                 allqueue(REDRAWOOPS, 0);
 326         }
 327 }
 328
 329 /* ===== OPENGL =====
 330  *
 331  * Simple functions to get data from the GL
 332  */
 333
 334 /* Store the modelview and projection matrices and viewport. */
 335 void init_sculptmatrices()
 336 {
 337         SculptSession *ss= sculpt_session();
 338
 339         glMatrixMode(GL_MODELVIEW);
 340         glPushMatrix();
 341         glMultMatrixf(OBACT->obmat);
 342
 343         bgl_get_mats(&ss->mats);
 344
 345         glPopMatrix();
 346
 347 }
 348
 349 /* Uses window coordinates (x,y) to find the depth in the GL depth buffer. If
 350    available, G.vd->depths is used so that the brush doesn't sculpt on top of
 351    itself (G.vd->depths is only updated at the end of a brush stroke.) */
 352 float get_depth(short x, short y)
 353 {
 354         float depth;
 355
 356         if(x<0 || y<0) return 1;
 357         if(x>=curarea->winx || y>=curarea->winy) return 1;
 358
 359         if(G.vd->depths && x<G.vd->depths->w && y<G.vd->depths->h)
 360                 return G.vd->depths->depths[y*G.vd->depths->w+x];
 361
 362         x+= curarea->winrct.xmin;
 363         y+= curarea->winrct.ymin;
 364
 365         glReadPixels(x, y, 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &depth);
 366
 367         return depth;
 368 }
 369
 370 /* Uses window coordinates (x,y) and depth component z to find a point in
 371    modelspace */
 372 vec3f unproject(const short x, const short y, const float z)
 373 {
 374         SculptSession *ss= sculpt_session();
 375         double ux, uy, uz;
 376         vec3f p;
 377
 378         gluUnProject(x,y,z, ss->mats.modelview, ss->mats.projection,
 379                      (GLint *)ss->mats.viewport, &ux, &uy, &uz );
 380         p.x= ux;
 381         p.y= uy;
 382         p.z= uz;
 383         return p;
 384 }
 385
 386 /* Convert a point in model coordinates to 2D screen coordinates. */
 387 void project(const float v[3], short p[2])
 388 {
 389         SculptSession *ss= sculpt_session();
 390         double ux, uy, uz;
 391
 392         gluProject(v[0],v[1],v[2], ss->mats.modelview, ss->mats.projection,
 393                    (GLint *)ss->mats.viewport, &ux, &uy, &uz);
 394         p[0]= ux;
 395         p[1]= uy;
 396 }
 397
 398 /* ===== Sculpting =====
 399  *
 400  */
 401
 402 /* Return modified brush size. Uses current tablet pressure (if available) to
 403    shrink the brush. Skipped for grab brush because only the first mouse down
 404    size is used, which is small if the user has just touched the pen to the
 405    tablet */
 406 char brush_size()
 407 {
 408         const BrushData *b= sculptmode_brush();
 409         float size= b->size;
 410         float pressure= get_pressure();
 411         short activedevice= get_activedevice();
 412
 413         if(sculpt_data()->brush_type!=GRAB_BRUSH) {
 414                 const float size_factor= G.scene->sculptdata.tablet_size / 10.0f;
 415                 if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
 416                         size*= G.scene->sculptdata.tablet_size==0?1:
 417                                (1-size_factor) + pressure*size_factor;
 418         }
 419
 420         return size;
 421 }
 422
 423 /* Return modified brush strength. Includes the direction of the brush, positive
 424    values pull vertices, negative values push. Uses tablet pressure and a
 425    special multiplier found experimentally to scale the strength factor. */
 426 float brush_strength(EditData *e)
 427 {
 428         const BrushData* b= sculptmode_brush();
 429         float dir= b->dir==1 ? 1 : -1;
 430         float pressure= 1;
 431         short activedevice= get_activedevice();
 432         float flip= e->flip ? -1:1;
 433
 434         const float strength_factor= G.scene->sculptdata.tablet_strength / 10.0f;
 435         if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
 436                 pressure= G.scene->sculptdata.tablet_strength==0?1:
 437                                   (1-strength_factor) + get_pressure()*strength_factor;
 438
 439         /* Flip direction for eraser */
 440         if(activedevice==DEV_ERASER)
 441                 dir= -dir;
 442
 443         switch(G.scene->sculptdata.brush_type){
 444         case DRAW_BRUSH:
 445         case LAYER_BRUSH:
 446                 return b->strength / 5000.0f * dir * pressure * flip;
 447         case SMOOTH_BRUSH:
 448                 return b->strength / 50.0f * pressure;
 449         case PINCH_BRUSH:
 450                 return b->strength / 1000.0f * dir * pressure * flip;
 451         case GRAB_BRUSH:
 452                 return 1;
 453         case INFLATE_BRUSH:
 454                 return b->strength / 5000.0f * dir * pressure * flip;
 455         case FLATTEN_BRUSH:
 456                 return b->strength / 500.0f * pressure;
 457         default:
 458                 return 0;
 459         }
 460 }
 461
 462 /* For clipping against a mirror modifier */
 463 void sculpt_clip(const EditData *e, float *co, const float val[3])
 464 {
 465         char i;
 466         for(i=0; i<3; ++i) {
 467                 if(e->clip[i] && (fabs(co[i]) <= e->cliptol[i]))
 468                         co[i]= 0.0f;
 469                 else
 470                         co[i]= val[i];
 471         }
 472 }
 473
 474 /* Currently only for the draw brush; finds average normal for all active
 475    vertices */
 476 vec3f calc_area_normal(const vec3f *outdir, const ListBase* active_verts)
 477 {
 478         Mesh *me= get_mesh(OBACT);
 479         vec3f area_normal= {0,0,0};
 480         ActiveData *node= active_verts->first;
 481         const int view= sculpt_data()->brush_type==DRAW_BRUSH ? sculptmode_brush()->view : 0;
 482
 483         while(node){
 484                 area_normal.x+= me->mvert[node->Index].no[0];
 485                 area_normal.y+= me->mvert[node->Index].no[1];
 486                 area_normal.z+= me->mvert[node->Index].no[2];
 487                 node= node->next;
 488         }
 489         Normalize(&area_normal.x);
 490         if(outdir) {
 491                 area_normal.x= outdir->x * view + area_normal.x * (10-view);
 492                 area_normal.y= outdir->y * view + area_normal.y * (10-view);
 493                 area_normal.z= outdir->z * view + area_normal.z * (10-view);
 494         }
 495         Normalize(&area_normal.x);
 496         return area_normal;
 497 }
 498 void do_draw_brush(const EditData *e, const ListBase* active_verts)
 499 {
 500         Mesh *me= get_mesh(OBACT);
 501         const vec3f area_normal= calc_area_normal(&e->out, active_verts);
 502         ActiveData *node= active_verts->first;
 503
 504         while(node){
 505                 float *co= me->mvert[node->Index].co;
 506
 507                 const float val[3]= {co[0]+area_normal.x*node->Fade*e->scale[0],
 508                                      co[1]+area_normal.y*node->Fade*e->scale[1],
 509                                      co[2]+area_normal.z*node->Fade*e->scale[2]};
 510
 511                 sculpt_clip(e, co, val);
 512
 513                 node= node->next;
 514         }
 515 }
 516
 517 /* For the smooth brush, uses the neighboring vertices around vert to calculate
 518    a smoothed location for vert. Skips corner vertices (used by only one
 519    polygon.) */
 520 vec3f neighbor_average(const int vert)
 521 {
 522         SculptSession *ss= sculpt_session();
 523         Mesh *me= get_mesh(OBACT);
 524         int i, skip= -1, total=0;
 525         IndexNode *node= ss->vertex_users[vert].first;
 526         vec3f avg= {0,0,0};
 527         char ncount= BLI_countlist(&ss->vertex_users[vert]);
 528         MFace *f;
 529
 530         /* Don't modify corner vertices */
 531         if(ncount==1) {
 532                 VecCopyf(&avg.x, me->mvert[vert].co);
 533                 return avg;
 534         }
 535
 536         while(node){
 537                 f= &me->mface[node->Index];
 538
 539                 if(f->v4) {
 540                         skip= (f->v1==vert?2:
 541                                f->v2==vert?3:
 542                                f->v3==vert?0:
 543                                f->v4==vert?1:-1);
 544                 }
 545
 546                 for(i=0; i<(f->v4?4:3); ++i) {
 547                         if(i != skip && (ncount!=2 || BLI_countlist(&ss->vertex_users[(&f->v1)[i]]) <= 2)) {
 548                                 VecAddf(&avg.x,&avg.x,me->mvert[(&f->v1)[i]].co);
 549                                 ++total;
 550                         }
 551                 }
 552
 553                 node= node->next;
 554         }
 555
 556         if(total>0) {
 557                 avg.x/= total;
 558                 avg.y/= total;
 559                 avg.z/= total;
 560         }
 561         else
 562                 VecCopyf(&avg.x, me->mvert[vert].co);
 563
 564         return avg;
 565 }
 566
 567 void do_smooth_brush(const EditData *e, const ListBase* active_verts)
 568 {
 569         ActiveData *node= active_verts->first;
 570         Mesh *me= get_mesh(OBACT);
 571
 572         while(node){
 573                 float *co= me->mvert[node->Index].co;
 574                 const vec3f avg= neighbor_average(node->Index);
 575                 const float val[3]= {co[0]+(avg.x-co[0])*node->Fade,
 576                                      co[1]+(avg.y-co[1])*node->Fade,
 577                                      co[2]+(avg.z-co[2])*node->Fade};
 578                 sculpt_clip(e, co, val);
 579                 node= node->next;
 580         }
 581 }
 582
 583 void do_pinch_brush(const EditData *e, const ListBase* active_verts)
 584 {
 585         Mesh *me= get_mesh(OBACT);
 586         ActiveData *node= active_verts->first;
 587
 588         while(node) {
 589                 float *co= me->mvert[node->Index].co;
 590                 const float val[3]= {co[0]+(e->center.x-co[0])*node->Fade,
 591                                      co[1]+(e->center.y-co[1])*node->Fade,
 592                                      co[2]+(e->center.z-co[2])*node->Fade};
 593                 sculpt_clip(e, co, val);
 594                 node= node->next;
 595         }
 596 }
 597
 598 void do_grab_brush(EditData *e)
 599 {
 600         Mesh *me= get_mesh(OBACT);
 601         ActiveData *node= e->grabdata->active_verts[e->grabdata->index].first;
 602         float add[3];
 603
 604         while(node) {
 605                 float *co= me->mvert[node->Index].co;
 606
 607                 VecCopyf(add, &e->grabdata->delta_symm.x);
 608                 VecMulf(add, node->Fade);
 609                 VecAddf(add, add, co);
 610                 sculpt_clip(e, co, add);
 611
 612                 node= node->next;
 613         }
 614 }
 615
 616 void do_layer_brush(EditData *e, const ListBase *active_verts)
 617 {
 618         Mesh *me= get_mesh(OBACT);
 619         vec3f area_normal= calc_area_normal(NULL, active_verts);
 620         ActiveData *node= active_verts->first;
 621         const float bstr= brush_strength(e);
 622
 623         while(node){
 624                 float *disp= &e->layer_disps[node->Index];
 625
 626                 if((bstr > 0 && *disp < bstr) ||
 627                   (bstr < 0 && *disp > bstr)) {
 628                         float *co= me->mvert[node->Index].co;
 629
 630                         *disp+= node->Fade;
 631
 632                         if(bstr < 0) {
 633                                 if(*disp < bstr)
 634                                         *disp = bstr;
 635                         } else {
 636                                 if(*disp > bstr)
 637                                         *disp = bstr;
 638                         }
 639
 640                         {
 641                                 const float val[3]= {e->layer_store[node->Index].x+area_normal.x * *disp*e->scale[0],
 642                                                      e->layer_store[node->Index].y+area_normal.y * *disp*e->scale[1],
 643                                                      e->layer_store[node->Index].z+area_normal.z * *disp*e->scale[2]};
 644                                 sculpt_clip(e, co, val);
 645                         }
 646                 }
 647
 648                 node= node->next;
 649         }
 650 }
 651
 652 void do_inflate_brush(const EditData *e, const ListBase *active_verts)
 653 {
 654         ActiveData *node= active_verts->first;
 655         float add[3];
 656         Mesh *me= get_mesh(OBACT);
 657
 658         while(node) {
 659                 float *co= me->mvert[node->Index].co;
 660                 short *no= me->mvert[node->Index].no;
 661
 662                 add[0]= no[0]/ 32767.0f;
 663                 add[1]= no[1]/ 32767.0f;
 664                 add[2]= no[2]/ 32767.0f;
 665                 VecMulf(add, node->Fade);
 666                 add[0]*= e->scale[0];
 667                 add[1]*= e->scale[1];
 668                 add[2]*= e->scale[2];
 669                 VecAddf(add, add, co);
 670
 671                 sculpt_clip(e, co, add);
 672
 673                 node= node->next;
 674         }
 675 }
 676
 677 void calc_flatten_center(Mesh *me, ActiveData *node, const EditData *e, float co[3])
 678 {
 679         ActiveData *outer[FLATTEN_SAMPLE_SIZE];
 680         int i;
 681
 682         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
 683                 outer[i] = node;
 684
 685         for(; node; node = node->next) {
 686                 for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i) {
 687                         if(node->dist > outer[i]->dist) {
 688                                 outer[i] = node;
 689                                 break;
 690                         }
 691                 }
 692         }
 693
 694         co[0] = co[1] = co[2] = 0.0f;
 695         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
 696                 VecAddf(co, co, me->mvert[outer[i]->Index].co);
 697         VecMulf(co, 1.0f / FLATTEN_SAMPLE_SIZE);
 698 }
 699
 700 void do_flatten_brush(const EditData *e, const ListBase *active_verts)
 701 {
 702         Mesh *me= get_mesh(OBACT);
 703         ActiveData *node= active_verts->first;
 704         /* area_normal and cntr define the plane towards which vertices are squashed */
 705         vec3f area_normal= calc_area_normal(&e->out, active_verts);
 706         float cntr[3];
 707
 708         calc_flatten_center(me, node, e, cntr);
 709
 710         while(node){
 711                 float *co= me->mvert[node->Index].co;
 712                 float p1[3], sub1[3], sub2[3], intr[3], val[3];
 713
 714                 /* Find the intersection between squash-plane and vertex (along the area normal) */
 715                 VecSubf(p1, co, &area_normal.x);
 716                 VecSubf(sub1, cntr, p1);
 717                 VecSubf(sub2, co, p1);
 718                 VecSubf(intr, co, p1);
 719                 VecMulf(intr, Inpf(&area_normal.x, sub1) / Inpf(&area_normal.x, sub2));
 720                 VecAddf(intr, intr, p1);
 721
 722                 VecSubf(val, intr, co);
 723                 VecMulf(val, node->Fade);
 724                 VecAddf(val, val, co);
 725
 726                 sculpt_clip(e, co, val);
 727
 728                 node= node->next;
 729         }
 730 }
 731
 732 /* Creates a smooth curve for the brush shape. This is the cos(x) curve from
 733    [0,PI] scaled to [0,len]. The range is scaled to [0,1]. */
 734 float simple_strength(float p, const float len)
 735 {
 736         if(p > len) p= len;
 737         return 0.5f * (cos(M_PI*p/len) + 1);
 738 }
 739
 740 /* Uses symm to selectively flip any axis of a coordinate. */
 741 void flip_coord(float co[3], const char symm)
 742 {
 743         if(symm & SYMM_X)
 744                 co[0]= -co[0];
 745         if(symm & SYMM_Y)
 746                 co[1]= -co[1];
 747         if(symm & SYMM_Z)
 748                 co[2]= -co[2];
 749 }
 750
 751 /* Use the warpfac field in MTex to store a rotation value for sculpt textures. Value is in degrees */
 752 float tex_angle(void)
 753 {
 754         SculptData *sd= sculpt_data();
 755         if(sd->texact!=-1 && sd->mtex[sd->texact])
 756                 return sd->mtex[sd->texact]->warpfac;
 757         return 0;
 758 }
 759
 760 void set_tex_angle(const float f)
 761 {
 762         SculptData *sd = sculpt_data();
 763         if(sd->texact != -1 && sd->mtex[sd->texact])
 764                 sd->mtex[sd->texact]->warpfac = f;
 765 }
 766
 767 float to_rad(const float deg)
 768 {
 769         return deg * (M_PI/180.0f);
 770 }
 771
 772 float to_deg(const float rad)
 773 {
 774         return rad * (180.0f/M_PI);
 775 }
 776
 777 /* Get a pixel from the texcache at (px, py) */
 778 unsigned *get_texcache_pixel(const SculptSession *ss, int px, int py)
 779 {
 780         if(px < 0) px= 0;
 781         if(py < 0) py= 0;
 782         if(px > ss->texcache_w - 1) px= ss->texcache_w - 1;
 783         if(py > ss->texcache_h - 1) py= ss->texcache_h - 1;
 784         return ss->texcache + py * ss->texcache_w + px;
 785 }
 786
 787 /* Return a multiplier for brush strength on a particular vertex. */
 788 float tex_strength(EditData *e, float *point, const float len,const unsigned vindex)
 789 {
 790         SculptData *sd= sculpt_data();
 791         SculptSession *ss= sculpt_session();
 792         float avg= 1;
 793
 794         if(sd->texact==-1 || !sd->mtex[sd->texact])
 795                 avg= 1;
 796         else if(sd->texrept==SCULPTREPT_3D) {
 797                 /* Get strength by feeding the vertex location directly
 798                    into a texture */
 799                 float jnk;
 800                 const float factor= 0.01;
 801                 MTex mtex;
 802                 memset(&mtex,0,sizeof(MTex));
 803                 mtex.tex= sd->mtex[sd->texact]->tex;
 804                 mtex.projx= 1;
 805                 mtex.projy= 2;
 806                 mtex.projz= 3;
 807                 VecCopyf(mtex.size, sd->mtex[sd->texact]->size);
 808                 VecMulf(mtex.size, factor);
 809                 if(!sd->texsep)
 810                         mtex.size[1]= mtex.size[2]= mtex.size[0];
 811
 812                 externtex(&mtex,point,&avg,&jnk,&jnk,&jnk,&jnk);
 813         }
 814         else if(ss->texcache) {
 815                 const short bsize= sculptmode_brush()->size * 2;
 816                 const short half= sculptmode_brush()->size;
 817                 const float rot= to_rad(tex_angle());
 818                 const unsigned tcw = ss->texcache_w, tch = ss->texcache_h;
 819                 int px, py;
 820                 unsigned i, *p;
 821                 ProjVert pv;
 822
 823                 /* If the active area is being applied for symmetry, flip it
 824                    across the symmetry axis in order to project it. This insures
 825                    that the brush texture will be oriented correctly. */
 826                 if(!e->symm)
 827                         pv= projverts[vindex];
 828                 else {
 829                         float co[3];
 830                         VecCopyf(co, point);
 831                         flip_coord(co, e->symm);
 832                         project(co, pv.co);
 833                 }
 834
 835                 /* For Tile and Drag modes, get the 2D screen coordinates of the
 836                    and scale them up or down to the texture size. */
 837                 if(sd->texrept==SCULPTREPT_TILE) {
 838                         const int sx= (const int)sd->mtex[sd->texact]->size[0];
 839                         const int sy= (const int)sd->texsep ? sd->mtex[sd->texact]->size[1] : sx;
 840
 841                         float fx= pv.co[0];
 842                         float fy= pv.co[1];
 843
 844                         float angle= atan2(fy, fx) - rot;
 845                         float len= sqrtf(fx*fx + fy*fy);
 846
 847                         if(rot<0.001 && rot>-0.001) {
 848                                 px= pv.co[0];
 849                                 py= pv.co[1];
 850                         } else {
 851                                 px= len * cos(angle) + 2000;
 852                                 py= len * sin(angle) + 2000;
 853                         }
 854                         px %= sx-1;
 855                         py %= sy-1;
 856                         p= get_texcache_pixel(ss, tcw*px/sx, tch*py/sy);
 857                 } else {
 858                         float fx= (pv.co[0] - e->mouse[0] + half) * (tcw*1.0f/bsize) - tcw/2;
 859                         float fy= (pv.co[1] - e->mouse[1] + half) * (tch*1.0f/bsize) - tch/2;
 860
 861                         float angle= atan2(fy, fx) - rot;
 862                         float len= sqrtf(fx*fx + fy*fy);
 863
 864                         px= tcw/2 + len * cos(angle);
 865                         py= tch/2 + len * sin(angle);
 866
 867                         p= get_texcache_pixel(ss, px, py);
 868                 }
 869
 870                 avg= 0;
 871                 for(i=0; i<3; ++i)
 872                         avg+= ((unsigned char*)(p))[i] / 255.0f;
 873
 874                 avg/= 3;
 875         }
 876
 877         if(sd->texfade)
 878                 avg*= simple_strength(len,e->size); /* Smooth curve */
 879
 880         return avg;
 881 }
 882
 883 /* Mark area around the brush as damaged. projverts are marked if they are
 884    inside the area and the damaged rectangle in 2D screen coordinates is
 885    added to damaged_rects. */
 886 void sculptmode_add_damaged_rect(EditData *e)
 887 {
 888         short p[2];
 889         const float radius= brush_size();
 890         RectNode *rn= MEM_mallocN(sizeof(RectNode),"RectNode");
 891         Mesh *me= get_mesh(OBACT);
 892         unsigned i;
 893
 894         /* Find center */
 895         project(&e->center.x, p);
 896         rn->r.xmin= p[0]-radius;
 897         rn->r.ymin= p[1]-radius;
 898         rn->r.xmax= p[0]+radius;
 899         rn->r.ymax= p[1]+radius;
 900
 901         BLI_addtail(&sculpt_session()->damaged_rects, rn);
 902
 903         /* Update insides */
 904         for(i=0; i<me->totvert; ++i) {
 905                 if(!projverts[i].inside) {
 906                         if(projverts[i].co[0] > rn->r.xmin && projverts[i].co[1] > rn->r.ymin &&
 907                            projverts[i].co[0] < rn->r.xmax && projverts[i].co[1] < rn->r.ymax) {
 908                                 projverts[i].inside= 1;
 909                         }
 910                 }
 911         }
 912 }
 913
 914 void do_brush_action(EditData e)
 915 {
 916         int i;
 917         float av_dist;
 918         ListBase active_verts={0,0};
 919         ActiveData *adata= 0;
 920         float *vert;
 921         Mesh *me= get_mesh(OBACT);
 922         const float bstrength= brush_strength(&e);
 923         KeyBlock *keyblock= ob_get_keyblock(OBACT);
 924         SculptSession *ss = sculpt_session();
 925
 926         sculptmode_add_damaged_rect(&e);
 927
 928         /* Build a list of all vertices that are potentially within the brush's
 929            area of influence. Only do this once for the grab brush. */
 930         if(!e.grabdata || (e.grabdata && e.grabdata->firsttime)) {
 931                 for(i=0; i<me->totvert; ++i) {
 932                         /* Projverts.inside provides a rough bounding box */
 933                         if(projverts[i].inside) {
 934                                 vert= ss->vertexcosnos ? &ss->vertexcosnos[i*6] : me->mvert[i].co;
 935                                 av_dist= VecLenf(&e.center.x,vert);
 936                                 if(av_dist < e.size) {
 937                                         adata= (ActiveData*)MEM_mallocN(sizeof(ActiveData), "ActiveData");
 938                                         adata->Index = i;
 939                                         /* Fade is used to store the final strength at which the brush
 940                                            should modify a particular vertex. */
 941                                         adata->Fade= tex_strength(&e,vert,av_dist,i) * bstrength;
 942                                         adata->dist = av_dist;
 943                                         if(e.grabdata && e.grabdata->firsttime)
 944                                                 BLI_addtail(&e.grabdata->active_verts[e.grabdata->index], adata);
 945                                         else
 946                                                 BLI_addtail(&active_verts, adata);
 947                                 }
 948                         }
 949                 }
 950         }
 951
 952         /* Only act if some verts are inside the brush area */
 953         if(active_verts.first || (e.grabdata && e.grabdata->active_verts[e.grabdata->index].first)) {
 954                 /* Apply one type of brush action */
 955                 switch(G.scene->sculptdata.brush_type){
 956                 case DRAW_BRUSH:
 957                         do_draw_brush(&e, &active_verts);
 958                         break;
 959                 case SMOOTH_BRUSH:
 960                         do_smooth_brush(&e, &active_verts);
 961                         break;
 962                 case PINCH_BRUSH:
 963                         do_pinch_brush(&e, &active_verts);
 964                         break;
 965                 case INFLATE_BRUSH:
 966                         do_inflate_brush(&e, &active_verts);
 967                         break;
 968                 case GRAB_BRUSH:
 969                         do_grab_brush(&e);
 970                         break;
 971                 case LAYER_BRUSH:
 972                         do_layer_brush(&e, &active_verts);
 973                         break;
 974                 case FLATTEN_BRUSH:
 975                         do_flatten_brush(&e, &active_verts);
 976                         break;
 977                 }
 978
 979                 /* Copy the modified vertices from mesh to the active key */
 980                 if(keyblock) {
 981                         float *co= keyblock->data;
 982                         if(co) {
 983                                 adata = e.grabdata ? e.grabdata->active_verts[e.grabdata->index].first : active_verts.first;
 984                                 for(; adata; adata= adata->next)
 985                                         if(adata->Index < keyblock->totelem)
 986                                                 VecCopyf(&co[adata->Index*3], me->mvert[adata->Index].co);
 987                         }
 988                 }
 989
 990                 if(ss->vertexcosnos)
 991                         BLI_freelistN(&active_verts);
 992                 else {
 993                         if(!e.grabdata)
 994                                 addlisttolist(&ss->damaged_verts, &active_verts);
 995                 }
 996         }
 997 }
 998
 999 /* Flip all the editdata across the axis/axes specified by symm. Used to
1000    calculate multiple modifications to the mesh when symmetry is enabled. */
1001 EditData flip_editdata(EditData *e, const char symm)
1002 {
1003         EditData fe= *e;
1004         GrabData *gd= fe.grabdata;
1005
1006         flip_coord(&fe.center.x, symm);
1007         flip_coord(&fe.up.x, symm);
1008         flip_coord(&fe.right.x, symm);
1009         flip_coord(&fe.out.x, symm);
1010
1011         fe.symm= symm;
1012
1013         project(&e->center.x,fe.mouse);
1014
1015         if(gd) {
1016                 gd->index= symm;
1017                 gd->delta_symm= gd->delta;
1018                 flip_coord(&gd->delta_symm.x, symm);
1019         }
1020
1021         return fe;
1022 }
1023
1024 void do_symmetrical_brush_actions(EditData * e, short co[2], short pr_co[2])
1025 {
1026         const char symm= sculpt_data()->symm;
1027
1028         init_editdata(e, co, pr_co);
1029
1030         do_brush_action(flip_editdata(e, 0));
1031
1032         if(symm & SYMM_X)
1033                 do_brush_action(flip_editdata(e, SYMM_X));
1034         if(symm & SYMM_Y)
1035                 do_brush_action(flip_editdata(e, SYMM_Y));
1036         if(symm & SYMM_Z)
1037                 do_brush_action(flip_editdata(e, SYMM_Z));
1038         if(symm & SYMM_X && symm & SYMM_Y)
1039                 do_brush_action(flip_editdata(e, SYMM_X | SYMM_Y));
1040         if(symm & SYMM_X && symm & SYMM_Z)
1041                 do_brush_action(flip_editdata(e, SYMM_X | SYMM_Z));
1042         if(symm & SYMM_Y && symm & SYMM_Z)
1043                 do_brush_action(flip_editdata(e, SYMM_Y | SYMM_Z));
1044         if(symm & SYMM_X && symm & SYMM_Y && symm & SYMM_Z)
1045                 do_brush_action(flip_editdata(e, SYMM_X | SYMM_Y | SYMM_Z));
1046 }
1047
1048 void add_face_normal(vec3f *norm, const MFace* face)
1049 {
1050         Mesh *me= get_mesh(OBACT);
1051
1052         vec3f c= {me->mvert[face->v1].co[0],me->mvert[face->v1].co[1],me->mvert[face->v1].co[2]};
1053         vec3f b= {me->mvert[face->v2].co[0],me->mvert[face->v2].co[1],me->mvert[face->v2].co[2]};
1054         vec3f a= {me->mvert[face->v3].co[0],me->mvert[face->v3].co[1],me->mvert[face->v3].co[2]};
1055         vec3f s1, s2;
1056
1057         VecSubf(&s1.x,&a.x,&b.x);
1058         VecSubf(&s2.x,&c.x,&b.x);
1059
1060         norm->x+= s1.y * s2.z - s1.z * s2.y;
1061         norm->y+= s1.z * s2.x - s1.x * s2.z;
1062         norm->z+= s1.x * s2.y - s1.y * s2.x;
1063 }
1064
1065 void update_damaged_vert(Mesh *me, ListBase *lb)
1066 {
1067         ActiveData *vert;
1068
1069         for(vert= lb->first; vert; vert= vert->next) {
1070                 vec3f norm= {0,0,0};
1071                 IndexNode *face= sculpt_session()->vertex_users[vert->Index].first;
1072
1073                 while(face){
1074                         add_face_normal(&norm,&me->mface[face->Index]);
1075                         face= face->next;
1076                 }
1077                 Normalize(&norm.x);
1078
1079                 me->mvert[vert->Index].no[0]=norm.x*32767;
1080                 me->mvert[vert->Index].no[1]=norm.y*32767;
1081                 me->mvert[vert->Index].no[2]=norm.z*32767;
1082         }
1083 }
1084
1085 void calc_damaged_verts(ListBase *damaged_verts, GrabData *grabdata)
1086 {
1087         Mesh *me= get_mesh(OBACT);
1088
1089         if(grabdata) {
1090                 int i;
1091                 for(i=0; i<8; ++i)
1092                         update_damaged_vert(me,&grabdata->active_verts[i]);
1093         } else {
1094                 update_damaged_vert(me,damaged_verts);
1095                 BLI_freelistN(damaged_verts);
1096                 damaged_verts->first = damaged_verts->last = NULL;
1097         }
1098 }
1099
1100 void projverts_clear_inside()
1101 {
1102         Mesh *me = get_mesh(OBACT);
1103         if(me) {
1104                 int i;
1105                 for(i = 0; i < me->totvert; ++i)
1106                         projverts[i].inside = 0;
1107         }
1108 }
1109
1110 BrushData *sculptmode_brush(void)
1111 {
1112         SculptData *sd= &G.scene->sculptdata;
1113
1114         BrushData *bd =
1115                 (sd->brush_type==DRAW_BRUSH ? &sd->drawbrush :
1116                  sd->brush_type==SMOOTH_BRUSH ? &sd->smoothbrush :
1117                  sd->brush_type==PINCH_BRUSH ? &sd->pinchbrush :
1118                  sd->brush_type==INFLATE_BRUSH ? &sd->inflatebrush :
1119                  sd->brush_type==GRAB_BRUSH ? &sd->grabbrush :
1120                  sd->brush_type==LAYER_BRUSH ? &sd->layerbrush :
1121                  sd->brush_type==FLATTEN_BRUSH ? &sd->flattenbrush : NULL);
1122
1123         if(!bd) {
1124                 sculptmode_init(G.scene);
1125                 bd = &sd->drawbrush;
1126         }
1127
1128         return bd;
1129 }
1130
1131 void sculptmode_update_tex()
1132 {
1133         SculptData *sd= sculpt_data();
1134         SculptSession *ss= sculpt_session();
1135         MTex *mtex = sd->mtex[sd->texact];
1136         TexResult texres = {0};
1137         float x, y, step=2.0/128.0, co[3];
1138         int hasrgb, ix, iy;
1139
1140         /* Skip Default brush shape and non-textures */
1141         if(sd->texact == -1 || !sd->mtex[sd->texact]) return;
1142
1143         if(ss->texcache) {
1144                 MEM_freeN(ss->texcache);
1145                 ss->texcache= NULL;
1146         }
1147
1148         ss->texcache_w = ss->texcache_h = 128;
1149         ss->texcache = MEM_callocN(sizeof(int) * ss->texcache_w * ss->texcache_h, "Sculpt Texture cache");
1150
1151         if(mtex && mtex->tex) {
1152                 BKE_image_get_ibuf(sd->mtex[sd->texact]->tex->ima, NULL);
1153
1154                 /*do normalized cannonical view coords for texture*/
1155                 for (y=-1.0, iy=0; iy<128; iy++, y += step) {
1156                         for (x=-1.0, ix=0; ix<128; ix++, x += step) {
1157                                 co[0]= x;
1158                                 co[1]= y;
1159                                 co[2]= 0.0f;
1160
1161                                 /* This is copied from displace modifier code */
1162                                 hasrgb = multitex_ext(mtex->tex, co, NULL, NULL, 1, &texres);
1163
1164                                 /* if the texture gave an RGB value, we assume it didn't give a valid
1165                                  * intensity, so calculate one (formula from do_material_tex).
1166                                  * if the texture didn't give an RGB value, copy the intensity across
1167                                  */
1168                                 if(hasrgb & TEX_RGB)
1169                                         texres.tin = (0.35 * texres.tr + 0.45 *
1170                                                       texres.tg + 0.2 * texres.tb);
1171
1172                                 texres.tin = texres.tin * 255.0;
1173                                 ((char*)ss->texcache)[(iy*128+ix)*4] = (char)texres.tin;
1174                                 ((char*)ss->texcache)[(iy*128+ix)*4+1] = (char)texres.tin;
1175                                 ((char*)ss->texcache)[(iy*128+ix)*4+2] = (char)texres.tin;
1176                                 ((char*)ss->texcache)[(iy*128+ix)*4+3] = (char)texres.tin;
1177                         }
1178                 }
1179         }
1180 }
1181
1182 /* pr_mouse is only used for the grab brush, can be NULL otherwise */
1183 void init_editdata(EditData *e, short *mouse, short *pr_mouse)
1184 {
1185         SculptData *sd = sculpt_data();
1186         const float mouse_depth= get_depth(mouse[0],mouse[1]);
1187         vec3f brush_edge_loc, zero_loc, oldloc;
1188         ModifierData *md;
1189         int i;
1190         const char flip = (get_qual() == LR_SHIFTKEY);
1191
1192         e->flip= flip;
1193
1194         /* Convert the location and size of the brush to
1195            modelspace coords */
1196         e->center= unproject(mouse[0],mouse[1],mouse_depth);
1197         brush_edge_loc= unproject(mouse[0] +
1198                                   brush_size(),mouse[1],
1199                                   mouse_depth);
1200         e->size= VecLenf(&e->center.x,&brush_edge_loc.x);
1201
1202         /* Set the pivot to allow the model to rotate around the center of the brush */
1203         if(get_depth(mouse[0],mouse[1]) < 1.0)
1204                 sculpt_session()->pivot= e->center;
1205
1206         /* Now project the Up, Right, and Out normals from view to model coords */
1207         zero_loc= unproject(0, 0, 0);
1208         e->up= unproject(0, -1, 0);
1209         e->right= unproject(1, 0, 0);
1210         e->out= unproject(0, 0, -1);
1211         VecSubf(&e->up.x, &e->up.x, &zero_loc.x);
1212         VecSubf(&e->right.x, &e->right.x, &zero_loc.x);
1213         VecSubf(&e->out.x, &e->out.x, &zero_loc.x);
1214         Normalize(&e->up.x);
1215         Normalize(&e->right.x);
1216         Normalize(&e->out.x);
1217
1218         /* Initialize mirror modifier clipping */
1219         for(i=0; i<3; ++i) {
1220                 e->clip[i]= 0;
1221                 e->cliptol[i]= 0;
1222         }
1223         for(md= OBACT->modifiers.first; md; md= md->next) {
1224                 if(md->type==eModifierType_Mirror && (md->mode & eModifierMode_Realtime)) {
1225                         const MirrorModifierData *mmd = (MirrorModifierData*) md;
1226
1227                         if(mmd->flag & MOD_MIR_CLIPPING) {
1228                                 e->clip[mmd->axis]= 1;
1229                                 if(mmd->tolerance > e->cliptol[mmd->axis])
1230                                         e->cliptol[mmd->axis]= mmd->tolerance;
1231                         }
1232                 }
1233         }
1234
1235         if(sd->brush_type == GRAB_BRUSH) {
1236                 vec3f gcenter;
1237                 if(!e->grabdata) {
1238                         e->grabdata= MEM_callocN(sizeof(GrabData),"grab data");
1239                         e->grabdata->firsttime= 1;
1240                         e->grabdata->depth= mouse_depth;
1241                 }
1242                 else
1243                         e->grabdata->firsttime= 0;
1244
1245                 /* Find the delta */
1246                 gcenter= unproject(mouse[0],mouse[1],e->grabdata->depth);
1247                 oldloc= unproject(pr_mouse[0],pr_mouse[1],e->grabdata->depth);
1248                 VecSubf(&e->grabdata->delta.x,&gcenter.x,&oldloc.x);
1249         }
1250         else if(sd->brush_type == LAYER_BRUSH) {
1251                 Mesh *me= get_mesh(OBACT);
1252
1253                 if(!e->layer_disps)
1254                         e->layer_disps= MEM_callocN(sizeof(float)*me->totvert,"Layer disps");
1255                 if(!e->layer_store) {
1256                         unsigned i;
1257                         e->layer_store= MEM_mallocN(sizeof(vec3f)*me->totvert,"Layer store");
1258                         for(i=0; i<me->totvert; ++i)
1259                                 VecCopyf(&e->layer_store[i].x,me->mvert[i].co);
1260                 }
1261         }
1262 }
1263 void sculptmode_set_strength(const int delta)
1264 {
1265         int val = sculptmode_brush()->strength + delta;
1266         if(val < 1) val = 1;
1267         if(val > 100) val = 100;
1268         sculptmode_brush()->strength= val;
1269 }
1270
1271 void sculptmode_propset_calctex()
1272 {
1273         SculptData *sd= sculpt_data();
1274         SculptSession *ss= sculpt_session();
1275         PropsetData *pd= sculpt_session()->propset;
1276         if(pd) {
1277                 int i, j;
1278                 const int tsz = 128;
1279                 float *d;
1280                 if(!pd->texdata) {
1281                         pd->texdata= MEM_mallocN(sizeof(float)*tsz*tsz, "Brush preview");
1282                         if(sd->texrept!=SCULPTREPT_3D)
1283                                 sculptmode_update_tex();
1284                         for(i=0; i<tsz; ++i)
1285                                 for(j=0; j<tsz; ++j) {
1286                                         float magn= sqrt(pow(i-tsz/2,2)+pow(j-tsz/2,2));
1287                                         if(sd->texfade)
1288                                                 pd->texdata[i*tsz+j]= simple_strength(magn,tsz/2);
1289                                         else
1290                                                 pd->texdata[i*tsz+j]= magn < tsz/2 ? 1 : 0;
1291                                 }
1292                         if(sd->texact != -1 && ss->texcache) {
1293                                 for(i=0; i<tsz; ++i)
1294                                         for(j=0; j<tsz; ++j) {
1295                                                 const int col= ss->texcache[i*tsz+j];
1296                                                 pd->texdata[i*tsz+j]*= (((char*)&col)[0]+((char*)&col)[1]+((char*)&col)[2])/3.0f/255.0f;
1297                                         }
1298                         }
1299                 }
1300
1301                 /* Adjust alpha with brush strength */
1302                 d= MEM_dupallocN(pd->texdata);
1303                 for(i=0; i<tsz; ++i)
1304                         for(j=0; j<tsz; ++j)
1305                                 d[i*tsz+j]*= sculptmode_brush()->strength/200.0f+0.5f;
1306
1307
1308                 if(!pd->tex)
1309                         glGenTextures(1, (GLuint *)&pd->tex);
1310                 glBindTexture(GL_TEXTURE_2D, pd->tex);
1311
1312                 glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, tsz, tsz, 0, GL_ALPHA, GL_FLOAT, d);
1313                 MEM_freeN(d);
1314         }
1315 }
1316
1317 void sculptmode_propset_header()
1318 {
1319         SculptSession *ss= sculpt_session();
1320         PropsetData *pd= ss ? ss->propset : NULL;
1321         if(pd) {
1322                 char str[512];
1323                 const char *name= "";
1324                 int val= 0;
1325                 if(pd->mode == PropsetSize) {
1326                         name= "Size";
1327                         val= sculptmode_brush()->size;
1328                 }
1329                 else if(pd->mode == PropsetStrength) {
1330                         name= "Strength";
1331                         val= sculptmode_brush()->strength;
1332                 }
1333                 else if(pd->mode == PropsetTexRot) {
1334                         name= "Texture Angle";
1335                         val= tex_angle();
1336                 }
1337                 sprintf(str, "Brush %s: %d", name, val);
1338                 headerprint(str);
1339         }
1340 }
1341
1342 void sculptmode_propset_end(SculptSession *ss, int cancel)
1343 {
1344         if(ss && ss->propset) {
1345                 PropsetData *pd= ss->propset;
1346
1347                 if(cancel) {
1348                         sculptmode_brush()->size= pd->origsize;
1349                         sculptmode_brush()->strength= pd->origstrength;
1350                         set_tex_angle(pd->origtexrot);
1351                 } else {
1352                         if(pd->mode != PropsetSize)
1353                                 sculptmode_brush()->size= pd->origsize;
1354                         if(pd->mode != PropsetStrength)
1355                                 sculptmode_brush()->strength= pd->origstrength;
1356                         if(pd->mode != PropsetTexRot)
1357                                 set_tex_angle(pd->origtexrot);
1358                 }
1359                 glDeleteTextures(1, &pd->tex);
1360                 MEM_freeN(pd->texdata);
1361                 MEM_freeN(pd);
1362                 ss->propset= NULL;
1363                 allqueue(REDRAWVIEW3D, 0);
1364                 allqueue(REDRAWBUTSEDIT, 0);
1365                 allqueue(REDRAWHEADERS, 0);
1366         }
1367 }
1368
1369 void sculptmode_propset_init(PropsetMode mode)
1370 {
1371         SculptSession *ss= sculpt_session();
1372         PropsetData *pd= ss->propset;
1373         const float ang= tex_angle();
1374
1375         if(!pd) {
1376                 short mouse[2];
1377
1378                 pd= MEM_callocN(sizeof(PropsetData),"PropsetSize");
1379                 ss->propset= pd;
1380
1381                 getmouseco_areawin(mouse);
1382                 pd->origloc[0]= mouse[0];
1383                 pd->origloc[1]= mouse[1];
1384
1385                 if(mode == PropsetSize)
1386                         pd->origloc[0]-= sculptmode_brush()->size;
1387                 else if(mode == PropsetStrength)
1388                         pd->origloc[0]-= 200 - 2*sculptmode_brush()->strength;
1389                 else if(mode == PropsetTexRot) {
1390                         pd->origloc[0]-= 200 * cos(to_rad(ang));
1391                         pd->origloc[1]-= 200 * sin(to_rad(ang));
1392                 }
1393
1394                 pd->origsize= sculptmode_brush()->size;
1395                 pd->origstrength= sculptmode_brush()->strength;
1396                 pd->origtexrot= ang;
1397
1398                 sculptmode_propset_calctex();
1399
1400                 pd->num.idx_max= 0;
1401         }
1402
1403         pd->mode= mode;
1404         sculptmode_propset_header();
1405
1406         allqueue(REDRAWVIEW3D, 0);
1407 }
1408
1409 void sculpt_paint_brush(char clear)
1410 {
1411         if(sculpt_data()->flags & SCULPT_DRAW_BRUSH) {
1412                 static short mvalo[2];
1413                 short mval[2];
1414                 const short rad= sculptmode_brush()->size;
1415
1416                 getmouseco_areawin(mval);
1417
1418                 persp(PERSP_WIN);
1419                 if(clear)
1420                         fdrawXORcirc(mval[0], mval[1], rad);
1421                 else
1422                         draw_sel_circle(mval, mvalo, rad, rad, 0);
1423
1424                 mvalo[0]= mval[0];
1425                 mvalo[1]= mval[1];
1426         }
1427 }
1428
1429 void sculptmode_propset(unsigned short event)
1430 {
1431         PropsetData *pd= sculpt_session()->propset;
1432         short mouse[2];
1433         short tmp[2];
1434         float dist;
1435         BrushData *brush= sculptmode_brush();
1436         char valset= 0;
1437
1438         handleNumInput(&pd->num, event);
1439
1440         if(hasNumInput(&pd->num)) {
1441                 float val;
1442                 applyNumInput(&pd->num, &val);
1443                 if(pd->mode==PropsetSize)
1444                         brush->size= val;
1445                 else if(pd->mode==PropsetStrength)
1446                         brush->strength= val;
1447                 else if(pd->mode==PropsetTexRot)
1448                         set_tex_angle(val);
1449                 valset= 1;
1450                 allqueue(REDRAWVIEW3D, 0);
1451         }
1452
1453         switch(event) {
1454         case MOUSEX:
1455         case MOUSEY:
1456                 if(!hasNumInput(&pd->num)) {
1457                         char ctrl= G.qual & LR_CTRLKEY;
1458                         getmouseco_areawin(mouse);
1459                         tmp[0]= pd->origloc[0]-mouse[0];
1460                         tmp[1]= pd->origloc[1]-mouse[1];
1461                         dist= sqrt(tmp[0]*tmp[0]+tmp[1]*tmp[1]);
1462                         if(pd->mode == PropsetSize) {
1463                                 brush->size= dist;
1464                                 if(ctrl) brush->size= (brush->size+5)/10*10;
1465                         } else if(pd->mode == PropsetStrength) {
1466                                 float fin= (200.0f - dist) * 0.5f;
1467                                 brush->strength= fin>=0 ? fin : 0;
1468                                 if(ctrl) brush->strength= (brush->strength+5)/10*10;
1469                         } else if(pd->mode == PropsetTexRot) {
1470                                 set_tex_angle((int)to_deg(atan2(tmp[1], tmp[0])) + 180);
1471                                 if(ctrl)
1472                                         set_tex_angle(((int)(tex_angle())+5)/10*10);
1473                         }
1474                         valset= 1;
1475                         allqueue(REDRAWVIEW3D, 0);
1476                 }
1477                 break;
1478         case ESCKEY:
1479         case RIGHTMOUSE:
1480                 brush->size= pd->origsize;
1481                 brush->strength= pd->origstrength;
1482                 set_tex_angle(pd->origtexrot);
1483         case LEFTMOUSE:
1484                 while(get_mbut()==L_MOUSE);
1485         case RETKEY:
1486         case PADENTER:
1487                 sculptmode_propset_end(sculpt_session(), 0);
1488                 break;
1489         default:
1490                 break;
1491         };
1492
1493         if(valset) {
1494                 if(pd->mode == PropsetSize) {
1495                         if(brush->size<1) brush->size= 1;
1496                         if(brush->size>200) brush->size= 200;
1497                 }
1498                 else if(pd->mode == PropsetStrength) {
1499                         if(brush->strength > 100) brush->strength= 100;
1500                         sculptmode_propset_calctex();
1501                 }
1502                 else if(pd->mode == PropsetTexRot) {
1503                         if(tex_angle() < 0)
1504                                 set_tex_angle(0);
1505                         else if(tex_angle() > 360)
1506                                 set_tex_angle(360);
1507                 }
1508         }
1509
1510         sculptmode_propset_header();
1511 }
1512
1513 void sculptmode_selectbrush_menu(void)
1514 {
1515         SculptData *sd= sculpt_data();
1516         int val;
1517
1518         pupmenu_set_active(sd->brush_type);
1519
1520         val= pupmenu("Select Brush%t|Draw|Smooth|Pinch|Inflate|Grab|Layer|Flatten");
1521
1522         if(val>0) {
1523                 sd->brush_type= val;
1524
1525                 allqueue(REDRAWVIEW3D, 1);
1526                 allqueue(REDRAWBUTSEDIT, 1);
1527         }
1528 }
1529
1530 void sculptmode_update_all_projverts(float *vertcosnos)
1531 {
1532         Mesh *me= get_mesh(OBACT);
1533         unsigned i;
1534
1535         if(projverts) MEM_freeN(projverts);
1536         projverts= MEM_mallocN(sizeof(ProjVert)*me->totvert,"ProjVerts");
1537         for(i=0; i<me->totvert; ++i) {
1538                 project(vertcosnos ? &vertcosnos[i * 6] : me->mvert[i].co, projverts[i].co);
1539                 projverts[i].inside= 0;
1540         }
1541 }
1542
1543 void sculptmode_draw_wires(int only_damaged, Mesh *me)
1544 {
1545         int i;
1546
1547         bglPolygonOffset(1.0);
1548         glDepthMask(0);
1549         BIF_ThemeColor((OBACT==OBACT)?TH_ACTIVE:TH_SELECT);
1550
1551         for(i=0; i<me->totedge; i++) {
1552                 MEdge *med= &me->medge[i];
1553
1554                 if((!only_damaged || (projverts[med->v1].inside || projverts[med->v2].inside)) &&
1555                    (med->flag & ME_EDGEDRAW)) {
1556                         glDrawElements(GL_LINES, 2, GL_UNSIGNED_INT, &med->v1);
1557                 }
1558         }
1559
1560         glDepthMask(1);
1561         bglPolygonOffset(0.0);
1562 }
1563
1564 void sculptmode_draw_mesh(int only_damaged)
1565 {
1566         Mesh *me= get_mesh(OBACT);
1567         int i, j, dt, drawCurrentMat = 1, matnr= -1;
1568
1569         persp(PERSP_VIEW);
1570         mymultmatrix(OBACT->obmat);
1571         glEnable(GL_DEPTH_TEST);
1572         glEnable(GL_LIGHTING);
1573         init_gl_materials(OBACT, 0);
1574         glEnable(GL_CULL_FACE);
1575
1576         glShadeModel(GL_SMOOTH);
1577
1578         glVertexPointer(3, GL_FLOAT, sizeof(MVert), &me->mvert[0].co);
1579         glNormalPointer(GL_SHORT, sizeof(MVert), &me->mvert[0].no);
1580
1581         dt= MIN2(G.vd->drawtype, OBACT->dt);
1582         if(dt==OB_WIRE)
1583                 glColorMask(0,0,0,0);
1584
1585         for(i=0; i<me->totface; ++i) {
1586                 MFace *f= &me->mface[i];
1587                 char inside= 0;
1588                 int new_matnr= f->mat_nr + 1;
1589
1590                 if(new_matnr != matnr)
1591                         drawCurrentMat= set_gl_material(matnr = new_matnr);
1592
1593                 /* If only_damaged!=0, only draw faces that are partially
1594                    inside the area(s) modified by the brush */
1595                 if(only_damaged) {
1596                         for(j=0; j<(f->v4?4:3); ++j) {
1597                                 if(projverts[*((&f->v1)+j)].inside) {
1598                                         inside= 1;
1599                                         break;
1600                                 }
1601                         }
1602                 }
1603                 else
1604                         inside= 1;
1605
1606                 if(inside && drawCurrentMat)
1607                         glDrawElements(f->v4?GL_QUADS:GL_TRIANGLES, f->v4?4:3, GL_UNSIGNED_INT, &f->v1);
1608         }
1609
1610         glDisable(GL_CULL_FACE);
1611         glDisable(GL_LIGHTING);
1612         glColorMask(1,1,1,1);
1613
1614         if(dt==OB_WIRE || (OBACT->dtx & OB_DRAWWIRE))
1615                 sculptmode_draw_wires(only_damaged, me);
1616
1617         glDisable(GL_DEPTH_TEST);
1618 }
1619
1620 void sculptmode_correct_state(void)
1621 {
1622         if(!sculpt_session())
1623                 sculpt_init_session();
1624
1625         glEnableClientState(GL_VERTEX_ARRAY);
1626         glEnableClientState(GL_NORMAL_ARRAY);
1627
1628         if(!sculpt_session()->vertex_users) calc_vertex_users();
1629 }
1630
1631 /* Checks whether full update mode (slower) needs to be used to work with modifiers */
1632 char sculpt_modifiers_active(Object *ob)
1633 {
1634         ModifierData *md;
1635
1636         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1637                 if(md->mode & eModifierMode_Realtime)
1638                         return 1;
1639         }
1640
1641         return 0;
1642 }
1643
1644 void sculpt(void)
1645 {
1646         SculptData *sd= sculpt_data();
1647         SculptSession *ss= sculpt_session();
1648         Object *ob= OBACT;
1649         /* lastSigMouse is for the rake, to store the last place the mouse movement was significant */
1650         short mouse[2], mvalo[2], lastSigMouse[2],firsttime=1, mousebut;
1651         short modifier_calculations= 0;
1652         EditData e;
1653         RectNode *rn= NULL;
1654         short spacing= 32000;
1655         int scissor_box[4];
1656         float offsetRot;
1657         if(!(G.f & G_SCULPTMODE) || G.obedit || !ob || ob->id.lib || !get_mesh(ob) || (get_mesh(ob)->totface == 0))
1658                 return;
1659         if(!(ob->lay & G.vd->lay))
1660                 error("Active object is not in this layer");
1661         if(ob_get_keyblock(ob)) {
1662                 if(!(ob->shapeflag & OB_SHAPE_LOCK)) {
1663                         error("Cannot sculpt on unlocked shape key");
1664                         return;
1665                 }
1666         }
1667
1668         if(!ss) {
1669                 sculpt_init_session();
1670                 ss= sd->session;
1671         }
1672
1673         if(sd->flags & SCULPT_INPUT_SMOOTH)
1674                 sculpt_stroke_new(256);
1675
1676         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
1677         ss->damaged_verts.first = ss->damaged_verts.last = NULL;
1678         ss->vertexcosnos = NULL;
1679
1680         /* Check that vertex users are up-to-date */
1681         if(ob != active_ob || ss->vertex_users_size != get_mesh(ob)->totvert) {
1682                 sculptmode_free_vertexusers(ss);
1683                 calc_vertex_users();
1684                 active_ob= ob;
1685         }
1686
1687         glEnableClientState(GL_VERTEX_ARRAY);
1688         glEnableClientState(GL_NORMAL_ARRAY);
1689
1690         persp(PERSP_VIEW);
1691
1692         getmouseco_areawin(mvalo);
1693
1694         /* Make sure sculptdata has been init'd properly */
1695         if(!ss->vertex_users) calc_vertex_users();
1696
1697         /* Init texture
1698            FIXME: Shouldn't be doing this every time! */
1699         if(sd->texrept!=SCULPTREPT_3D)
1700                 sculptmode_update_tex();
1701
1702         getmouseco_areawin(mouse);
1703         mvalo[0]= mouse[0];
1704         mvalo[1]= mouse[1];
1705         lastSigMouse[0]=mouse[0];
1706         lastSigMouse[1]=mouse[1];
1707         mousebut = L_MOUSE;
1708
1709         /* If modifier_calculations is true, then extra time must be spent
1710            updating the mesh. This takes a *lot* longer, so it's worth
1711            skipping if the modifier stack is empty. */
1712         modifier_calculations= sculpt_modifiers_active(ob);
1713
1714         init_sculptmatrices();
1715
1716         if(modifier_calculations)
1717                 ss->vertexcosnos= mesh_get_mapped_verts_nors(ob);
1718         sculptmode_update_all_projverts(ss->vertexcosnos);
1719
1720         e.grabdata= NULL;
1721         e.layer_disps= NULL;
1722         e.layer_store= NULL;
1723
1724         /* Set scaling adjustment */
1725         e.scale[0]= 1.0f / ob->size[0];
1726         e.scale[1]= 1.0f / ob->size[1];
1727         e.scale[2]= 1.0f / ob->size[2];
1728
1729         /* Capture original copy */
1730         if(sd->flags & SCULPT_DRAW_FAST)
1731                 glAccum(GL_LOAD, 1);
1732
1733         /* Get original scissor box */
1734         glGetIntegerv(GL_SCISSOR_BOX, scissor_box);
1735
1736         /* For raking, get the original angle*/
1737         offsetRot=tex_angle();
1738
1739         while (get_mbut() & mousebut) {
1740                 getmouseco_areawin(mouse);
1741                 /* If rake, and the mouse has moved over 10 pixels (euclidean) (prevents jitter) then get the new angle */
1742                 if (sd->rake && (pow(lastSigMouse[0]-mouse[0],2)+pow(lastSigMouse[1]-mouse[1],2))>100){
1743                         /*Nasty looking, but just orig + new angle really*/
1744                         set_tex_angle(offsetRot+180.+to_deg(atan2((float)(mouse[1]-lastSigMouse[1]),(float)(mouse[0]-lastSigMouse[0]))));
1745                         lastSigMouse[0]=mouse[0];
1746                         lastSigMouse[1]=mouse[1];
1747                 }
1748
1749                 if(firsttime || mouse[0]!=mvalo[0] || mouse[1]!=mvalo[1] || sculptmode_brush()->airbrush) {
1750                         firsttime= 0;
1751
1752                         if(sd->flags & SCULPT_INPUT_SMOOTH)
1753                                 sculpt_stroke_add_point(mouse[0], mouse[1]);
1754
1755                         spacing+= sqrt(pow(mvalo[0]-mouse[0],2)+pow(mvalo[1]-mouse[1],2));
1756
1757                         if(modifier_calculations && !ss->vertexcosnos)
1758                                 ss->vertexcosnos= mesh_get_mapped_verts_nors(ob);
1759
1760                         if(G.scene->sculptdata.brush_type != GRAB_BRUSH) {
1761                                 if(sd->flags & SCULPT_INPUT_SMOOTH) {
1762                                         sculpt_stroke_apply(&e);
1763                                 }
1764                                 else if(sd->spacing==0 || spacing>sd->spacing) {
1765                                         do_symmetrical_brush_actions(&e, mouse, NULL);
1766                                         spacing= 0;
1767                                 }
1768                         }
1769                         else {
1770                                 do_symmetrical_brush_actions(&e, mouse, mvalo);
1771                                 ss->pivot= unproject(mouse[0],mouse[1],e.grabdata->depth);
1772                         }
1773
1774                         if(modifier_calculations || ob_get_keyblock(ob))
1775                                 DAG_object_flush_update(G.scene, ob, OB_RECALC_DATA);
1776
1777                         if(modifier_calculations || sd->brush_type == GRAB_BRUSH || !(sd->flags & SCULPT_DRAW_FAST)) {
1778                                 calc_damaged_verts(&ss->damaged_verts,e.grabdata);
1779                                 scrarea_do_windraw(curarea);
1780                                 screen_swapbuffers();
1781                         } else { /* Optimized drawing */
1782                                 calc_damaged_verts(&ss->damaged_verts,e.grabdata);
1783
1784                                 /* Draw the stored image to the screen */
1785                                 glAccum(GL_RETURN, 1);
1786
1787                                 /* Clear each of the area(s) modified by the brush */
1788                                 for(rn=ss->damaged_rects.first; rn; rn= rn->next) {
1789                                         rcti clp= rn->r;
1790                                         rcti *win= &curarea->winrct;
1791
1792                                         clp.xmin+= win->xmin;
1793                                         clp.xmax+= win->xmin;
1794                                         clp.ymin+= win->ymin;
1795                                         clp.ymax+= win->ymin;
1796
1797                                         if(clp.xmin<win->xmax && clp.xmax>win->xmin &&
1798                                            clp.ymin<win->ymax && clp.ymax>win->ymin) {
1799                                                 if(clp.xmin<win->xmin) clp.xmin= win->xmin;
1800                                                 if(clp.ymin<win->ymin) clp.ymin= win->ymin;
1801                                                 if(clp.xmax>win->xmax) clp.xmax= win->xmax;
1802                                                 if(clp.ymax>win->ymax) clp.ymax= win->ymax;
1803                                                 glScissor(clp.xmin+1, clp.ymin+1,
1804                                                           clp.xmax-clp.xmin-2,clp.ymax-clp.ymin-2);
1805                                         }
1806
1807                                         glClear(GL_DEPTH_BUFFER_BIT);
1808                                 }
1809
1810                                 /* Draw all the polygons that are inside the modified area(s) */
1811                                 glScissor(scissor_box[0], scissor_box[1], scissor_box[2], scissor_box[3]);
1812                                 sculptmode_draw_mesh(1);
1813                                 glAccum(GL_LOAD, 1);
1814
1815                                 projverts_clear_inside();
1816
1817                                 persp(PERSP_WIN);
1818                                 glDisable(GL_DEPTH_TEST);
1819
1820                                 /* Draw cursor */
1821                                 if(sculpt_data()->flags & SCULPT_DRAW_BRUSH)
1822                                         fdrawXORcirc((float)mouse[0],(float)mouse[1],sculptmode_brush()->size);
1823                                 if(sculpt_data()->flags & SCULPT_INPUT_SMOOTH)
1824                                         sculpt_stroke_draw();
1825
1826                                 myswapbuffers();
1827                         }
1828
1829                         BLI_freelistN(&ss->damaged_rects);
1830                         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
1831
1832                         mvalo[0]= mouse[0];
1833                         mvalo[1]= mouse[1];
1834
1835                         if(ss->vertexcosnos) {
1836                                 MEM_freeN(ss->vertexcosnos);
1837                                 ss->vertexcosnos= NULL;
1838                         }
1839
1840                 }
1841                 else BIF_wait_for_statechange();
1842         }
1843
1844         /* Set the rotation of the brush back to what it was before any rake */
1845         set_tex_angle(offsetRot);
1846
1847         if(sd->flags & SCULPT_INPUT_SMOOTH) {
1848                 sculpt_stroke_apply_all(&e);
1849                 calc_damaged_verts(&ss->damaged_verts,e.grabdata);
1850                 BLI_freelistN(&ss->damaged_rects);
1851         }
1852
1853         if(projverts) MEM_freeN(projverts);
1854         projverts= NULL;
1855         if(e.layer_disps) MEM_freeN(e.layer_disps);
1856         if(e.layer_store) MEM_freeN(e.layer_store);
1857         /* Free GrabData */
1858         if(e.grabdata) {
1859                 int i;
1860                 for(i=0; i<8; ++i)
1861                         BLI_freelistN(&e.grabdata->active_verts[i]);
1862                 MEM_freeN(e.grabdata);
1863         }
1864         sculpt_stroke_free();
1865
1866         switch(G.scene->sculptdata.brush_type) {
1867         case DRAW_BRUSH:
1868                 BIF_undo_push("Draw Brush"); break;
1869         case SMOOTH_BRUSH:
1870                 BIF_undo_push("Smooth Brush"); break;
1871         case PINCH_BRUSH:
1872                 BIF_undo_push("Pinch Brush"); break;
1873         case INFLATE_BRUSH:
1874                 BIF_undo_push("Inflate Brush"); break;
1875         case GRAB_BRUSH:
1876                 BIF_undo_push("Grab Brush"); break;
1877         case LAYER_BRUSH:
1878                 BIF_undo_push("Layer Brush"); break;
1879         case FLATTEN_BRUSH:
1880                 BIF_undo_push("Flatten Brush"); break;
1881         default:
1882                 BIF_undo_push("Sculpting"); break;
1883         }
1884
1885         if(G.vd->depths) G.vd->depths->damaged= 1;
1886
1887         allqueue(REDRAWVIEW3D, 0);
1888 }
1889
1890 void set_sculptmode(void)
1891 {
1892         if(G.f & G_SCULPTMODE) {
1893                 Mesh *me= get_mesh(OBACT);
1894
1895                 G.f &= ~G_SCULPTMODE;
1896
1897                 sculptmode_free_session(G.scene);
1898                 if(me && me->pv)
1899                         sculptmode_pmv_off(me);
1900         }
1901         else {
1902                 G.f |= G_SCULPTMODE;
1903
1904                 /* Called here to sanity-check the brush */
1905                 sculptmode_brush();
1906
1907                 sculpt_init_session();
1908
1909                 glEnableClientState(GL_VERTEX_ARRAY);
1910                 glEnableClientState(GL_NORMAL_ARRAY);
1911         }
1912
1913         active_ob= NULL;
1914
1915         allqueue(REDRAWVIEW3D, 1);
1916         allqueue(REDRAWBUTSEDIT, 0);
1917 }
1918
1919 /* Partial Mesh Visibility */
1920 PartialVisibility *sculptmode_copy_pmv(PartialVisibility *pmv)
1921 {
1922         PartialVisibility *n= MEM_dupallocN(pmv);
1923         n->vert_map= MEM_dupallocN(pmv->vert_map);
1924         n->edge_map= MEM_dupallocN(pmv->edge_map);
1925         n->old_edges= MEM_dupallocN(pmv->old_edges);
1926         n->old_faces= MEM_dupallocN(pmv->old_faces);
1927         return n;
1928 }
1929
1930 void sculptmode_pmv_free(PartialVisibility *pv)
1931 {
1932         MEM_freeN(pv->vert_map);
1933         MEM_freeN(pv->edge_map);
1934         MEM_freeN(pv->old_faces);
1935         MEM_freeN(pv->old_edges);
1936         MEM_freeN(pv);
1937 }
1938
1939 void sculptmode_revert_pmv(Mesh *me)
1940 {
1941         if(me->pv) {
1942                 unsigned i;
1943                 MVert *nve, *old_verts;
1944
1945                 active_ob= NULL;
1946
1947                 /* Reorder vertices */
1948                 nve= me->mvert;
1949                 old_verts = MEM_mallocN(sizeof(MVert)*me->pv->totvert,"PMV revert verts");
1950                 for(i=0; i<me->pv->totvert; ++i)
1951                         old_verts[i]= nve[me->pv->vert_map[i]];
1952
1953                 /* Restore verts, edges and faces */
1954                 CustomData_free_layer_active(&me->vdata, CD_MVERT, me->totvert);
1955                 CustomData_free_layer_active(&me->edata, CD_MEDGE, me->totedge);
1956                 CustomData_free_layer_active(&me->fdata, CD_MFACE, me->totface);
1957
1958                 CustomData_add_layer(&me->vdata, CD_MVERT, CD_ASSIGN, old_verts, me->pv->totvert);
1959                 CustomData_add_layer(&me->edata, CD_MEDGE, CD_ASSIGN, me->pv->old_edges, me->pv->totedge);
1960                 CustomData_add_layer(&me->fdata, CD_MFACE, CD_ASSIGN, me->pv->old_faces, me->pv->totface);
1961                 mesh_update_customdata_pointers(me);
1962
1963                 me->totvert= me->pv->totvert;
1964                 me->totedge= me->pv->totedge;
1965                 me->totface= me->pv->totface;
1966
1967                 me->pv->old_edges= NULL;
1968                 me->pv->old_faces= NULL;
1969
1970                 /* Free maps */
1971                 MEM_freeN(me->pv->edge_map);
1972                 me->pv->edge_map= NULL;
1973                 MEM_freeN(me->pv->vert_map);
1974                 me->pv->vert_map= NULL;
1975
1976                 DAG_object_flush_update(G.scene, OBACT, OB_RECALC_DATA);
1977         }
1978 }
1979
1980 void sculptmode_pmv_off(Mesh *me)
1981 {
1982         if(me->pv) {
1983                 sculptmode_revert_pmv(me);
1984                 MEM_freeN(me->pv);
1985                 me->pv= NULL;
1986         }
1987 }
1988
1989 /* mode: 0=hide outside selection, 1=hide inside selection */
1990 void sculptmode_do_pmv(Object *ob, rcti *hb_2d, int mode)
1991 {
1992         Mesh *me= get_mesh(ob);
1993         vec3f hidebox[6];
1994         vec3f plane_normals[4];
1995         float plane_ds[4];
1996         unsigned i, j;
1997         unsigned ndx_show, ndx_hide;
1998         MVert *nve;
1999         unsigned face_cnt_show= 0, face_ndx_show= 0;
2000         unsigned edge_cnt_show= 0, edge_ndx_show= 0;
2001         unsigned *old_map= NULL;
2002         const unsigned SHOW= 0, HIDE=1;
2003
2004         /* Convert hide box from 2D to 3D */
2005         hidebox[0]= unproject(hb_2d->xmin, hb_2d->ymax, 1);
2006         hidebox[1]= unproject(hb_2d->xmax, hb_2d->ymax, 1);
2007         hidebox[2]= unproject(hb_2d->xmax, hb_2d->ymin, 1);
2008         hidebox[3]= unproject(hb_2d->xmin, hb_2d->ymin, 1);
2009         hidebox[4]= unproject(hb_2d->xmin, hb_2d->ymax, 0);
2010         hidebox[5]= unproject(hb_2d->xmax, hb_2d->ymin, 0);
2011
2012         /* Calculate normals for each side of hide box */
2013         CalcNormFloat(&hidebox[0].x,&hidebox[1].x,&hidebox[4].x,&plane_normals[0].x);
2014         CalcNormFloat(&hidebox[1].x,&hidebox[2].x,&hidebox[5].x,&plane_normals[1].x);
2015         CalcNormFloat(&hidebox[2].x,&hidebox[3].x,&hidebox[5].x,&plane_normals[2].x);
2016         CalcNormFloat(&hidebox[3].x,&hidebox[0].x,&hidebox[4].x,&plane_normals[3].x);
2017
2018         /* Calculate D for each side of hide box */
2019         for(i= 0; i<4; ++i)
2020                 plane_ds[i]= hidebox[i].x*plane_normals[i].x + hidebox[i].y*plane_normals[i].y +
2021                         hidebox[i].z*plane_normals[i].z;
2022
2023         /* Add partial visibility to mesh */
2024         if(!me->pv) {
2025                 me->pv= MEM_callocN(sizeof(PartialVisibility),"PartialVisibility");
2026         } else {
2027                 old_map= MEM_callocN(sizeof(unsigned)*me->pv->totvert,"PMV oldmap");
2028                 for(i=0; i<me->pv->totvert; ++i) {
2029                         old_map[i]= me->pv->vert_map[i]<me->totvert?0:1;
2030                 }
2031                 sculptmode_revert_pmv(me);
2032         }
2033
2034         /* Kill sculpt data */
2035         active_ob= NULL;
2036
2037         /* Initalize map with which verts are to be hidden */
2038         me->pv->vert_map= MEM_mallocN(sizeof(unsigned)*me->totvert, "PMV vertmap");
2039         me->pv->totvert= me->totvert;
2040         me->totvert= 0;
2041         for(i=0; i<me->pv->totvert; ++i) {
2042                 me->pv->vert_map[i]= mode ? HIDE:SHOW;
2043                 for(j=0; j<4; ++j) {
2044                         if(me->mvert[i].co[0] * plane_normals[j].x +
2045                            me->mvert[i].co[1] * plane_normals[j].y +
2046                            me->mvert[i].co[2] * plane_normals[j].z < plane_ds[j] ) {
2047                                 me->pv->vert_map[i]= mode ? SHOW:HIDE; /* Vert is outside the hide box */
2048                                 break;
2049                         }
2050                 }
2051                 if(old_map && old_map[i]) me->pv->vert_map[i]= 1;
2052                 if(!me->pv->vert_map[i]) ++me->totvert;
2053
2054         }
2055         if(old_map) MEM_freeN(old_map);
2056
2057         /* Find out how many faces to show */
2058         for(i=0; i<me->totface; ++i) {
2059                 if(!me->pv->vert_map[me->mface[i].v1] &&
2060                    !me->pv->vert_map[me->mface[i].v2] &&
2061                    !me->pv->vert_map[me->mface[i].v3]) {
2062                         if(me->mface[i].v4) {
2063                                 if(!me->pv->vert_map[me->mface[i].v4])
2064                                         ++face_cnt_show;
2065                         }
2066                         else ++face_cnt_show;
2067                 }
2068         }
2069         /* Find out how many edges to show */
2070         for(i=0; i<me->totedge; ++i) {
2071                 if(!me->pv->vert_map[me->medge[i].v1] &&
2072                    !me->pv->vert_map[me->medge[i].v2])
2073                         ++edge_cnt_show;
2074         }
2075
2076         /* Create new vert array and reset each vert's map with map[old]=new index */
2077         nve= MEM_mallocN(sizeof(MVert)*me->pv->totvert, "PMV verts");
2078         ndx_show= 0; ndx_hide= me->totvert;
2079         for(i=0; i<me->pv->totvert; ++i) {
2080                 if(me->pv->vert_map[i]) {
2081                         me->pv->vert_map[i]= ndx_hide;
2082                         nve[me->pv->vert_map[i]]= me->mvert[i];
2083                         ++ndx_hide;
2084                 } else {
2085                         me->pv->vert_map[i]= ndx_show;
2086                         nve[me->pv->vert_map[i]]= me->mvert[i];
2087                         ++ndx_show;
2088                 }
2089         }
2090         CustomData_free_layer_active(&me->vdata, CD_MVERT, me->pv->totvert);
2091         me->mvert= CustomData_add_layer(&me->vdata, CD_MVERT, CD_ASSIGN, nve, me->totvert);
2092
2093         /* Create new face array */
2094         me->pv->old_faces= me->mface;
2095         me->pv->totface= me->totface;
2096         me->mface= MEM_mallocN(sizeof(MFace)*face_cnt_show, "PMV faces");
2097         for(i=0; i<me->totface; ++i) {
2098                 MFace *pr_f= &me->pv->old_faces[i];
2099                 char show= 0;
2100
2101                 if(me->pv->vert_map[pr_f->v1] < me->totvert &&
2102                    me->pv->vert_map[pr_f->v2] < me->totvert &&
2103                    me->pv->vert_map[pr_f->v3] < me->totvert) {
2104                         if(pr_f->v4) {
2105                                 if(me->pv->vert_map[pr_f->v4] < me->totvert)
2106                                         show= 1;
2107                         }
2108                         else show= 1;
2109                 }
2110
2111                 if(show) {
2112                         MFace *cr_f= &me->mface[face_ndx_show];
2113                         *cr_f= *pr_f;
2114                         cr_f->v1= me->pv->vert_map[pr_f->v1];
2115                         cr_f->v2= me->pv->vert_map[pr_f->v2];
2116                         cr_f->v3= me->pv->vert_map[pr_f->v3];
2117                         cr_f->v4= pr_f->v4 ? me->pv->vert_map[pr_f->v4] : 0;
2118                         test_index_face(cr_f,NULL,0,pr_f->v4?4:3);
2119                         ++face_ndx_show;
2120                 }
2121         }
2122         me->totface= face_cnt_show;
2123         CustomData_set_layer(&me->fdata, CD_MFACE, me->mface);
2124
2125         /* Create new edge array */
2126         me->pv->old_edges= me->medge;
2127         me->pv->totedge= me->totedge;
2128         me->medge= MEM_mallocN(sizeof(MEdge)*edge_cnt_show, "PMV edges");
2129         me->pv->edge_map= MEM_mallocN(sizeof(int)*me->pv->totedge,"PMV edgemap");
2130         for(i=0; i<me->totedge; ++i) {
2131                 if(me->pv->vert_map[me->pv->old_edges[i].v1] < me->totvert &&
2132                    me->pv->vert_map[me->pv->old_edges[i].v2] < me->totvert) {
2133                         MEdge *cr_e= &me->medge[edge_ndx_show];
2134                         me->pv->edge_map[i]= edge_ndx_show;
2135                         *cr_e= me->pv->old_edges[i];
2136                         cr_e->v1= me->pv->vert_map[me->pv->old_edges[i].v1];
2137                         cr_e->v2= me->pv->vert_map[me->pv->old_edges[i].v2];
2138                         ++edge_ndx_show;
2139                 }
2140                 else me->pv->edge_map[i]= -1;
2141         }
2142         me->totedge= edge_cnt_show;
2143         CustomData_set_layer(&me->edata, CD_MEDGE, me->medge);
2144
2145         DAG_object_flush_update(G.scene, OBACT, OB_RECALC_DATA);
2146 }
2147
2148 rcti sculptmode_pmv_box()
2149 {
2150         short down[2], mouse[2];
2151         rcti ret;
2152
2153         getmouseco_areawin(down);
2154
2155         while((get_mbut()&L_MOUSE) || (get_mbut()&R_MOUSE)) {
2156                 getmouseco_areawin(mouse);
2157
2158                 scrarea_do_windraw(curarea);
2159
2160                 persp(PERSP_WIN);
2161                 glLineWidth(2);
2162                 setlinestyle(2);
2163                 sdrawXORline(down[0],down[1],mouse[0],down[1]);
2164                 sdrawXORline(mouse[0],down[1],mouse[0],mouse[1]);
2165                 sdrawXORline(mouse[0],mouse[1],down[0],mouse[1]);
2166                 sdrawXORline(down[0],mouse[1],down[0],down[1]);
2167                 setlinestyle(0);
2168                 glLineWidth(1);
2169                 persp(PERSP_VIEW);
2170
2171                 screen_swapbuffers();
2172                 backdrawview3d(0);
2173         }
2174
2175         ret.xmin= down[0]<mouse[0]?down[0]:mouse[0];
2176         ret.ymin= down[1]<mouse[1]?down[1]:mouse[1];
2177         ret.xmax= down[0]>mouse[0]?down[0]:mouse[0];
2178         ret.ymax= down[1]>mouse[1]?down[1]:mouse[1];
2179         return ret;
2180 }
2181
2182 void sculptmode_pmv(int mode)
2183 {
2184         Object *ob= OBACT;
2185         rcti hb_2d;
2186
2187         if(ob_get_key(ob)) {
2188                 error("Cannot hide mesh with shape keys enabled");
2189                 return;
2190         }
2191
2192         hb_2d= sculptmode_pmv_box(); /* Get 2D hide box */
2193
2194         sculptmode_correct_state();
2195
2196         waitcursor(1);
2197
2198         if(hb_2d.xmax-hb_2d.xmin > 3 && hb_2d.ymax-hb_2d.ymin > 3) {
2199                 init_sculptmatrices();
2200
2201                 sculptmode_do_pmv(ob,&hb_2d,mode);
2202         }
2203         else sculptmode_pmv_off(get_mesh(ob));
2204
2205         scrarea_do_windraw(curarea);
2206
2207         BIF_undo_push("Partial mesh hide");
2208
2209         waitcursor(0);
2210 }