source/blender/src/sculptmode.c

   1 /*
   2  * $Id$
   3  *
   4  * ***** BEGIN GPL LICENSE BLOCK *****
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version 2
   9  * of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software  Foundation,
  18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19  *
  20  * The Original Code is Copyright (C) 2006 by Nicholas Bishop
  21  * All rights reserved.
  22  *
  23  * The Original Code is: all of this file.
  24  *
  25  * Contributor(s): none yet.
  26  *
  27  * ***** END GPL LICENSE BLOCK *****
  28  *
  29  * Implements the Sculpt Mode tools
  30  *
  31  * BDR_sculptmode.h
  32  *
  33  */
  34
  35 #include "GHOST_Types.h"
  36
  37 #include "MEM_guardedalloc.h"
  38
  39 #include "BLI_arithb.h"
  40 #include "BLI_blenlib.h"
  41 #include "BLI_dynstr.h"
  42
  43 #include "DNA_armature_types.h"
  44 #include "DNA_brush_types.h"
  45 #include "DNA_image_types.h"
  46 #include "DNA_key_types.h"
  47 #include "DNA_mesh_types.h"
  48 #include "DNA_meshdata_types.h"
  49 #include "DNA_modifier_types.h"
  50 #include "DNA_object_types.h"
  51 #include "DNA_screen_types.h"
  52 #include "DNA_scene_types.h"
  53 #include "DNA_texture_types.h"
  54 #include "DNA_view3d_types.h"
  55 #include "DNA_userdef_types.h"
  56 #include "DNA_color_types.h"
  57
  58 #include "BKE_customdata.h"
  59 #include "BKE_DerivedMesh.h"
  60 #include "BKE_depsgraph.h"
  61 #include "BKE_global.h"
  62 #include "BKE_image.h"
  63 #include "BKE_key.h"
  64 #include "BKE_library.h"
  65 #include "BKE_main.h"
  66 #include "BKE_mesh.h"
  67 #include "BKE_modifier.h"
  68 #include "BKE_sculpt.h"
  69 #include "BKE_texture.h"
  70 #include "BKE_utildefines.h"
  71 #include "BKE_colortools.h"
  72
  73 #include "BIF_editkey.h"
  74 #include "BIF_editview.h"
  75 #include "BIF_glutil.h"
  76 #include "BIF_gl.h"
  77 #include "BIF_interface.h"
  78 #include "BIF_mywindow.h"
  79 #include "BIF_radialcontrol.h"
  80 #include "BIF_resources.h"
  81 #include "BIF_screen.h"
  82 #include "BIF_space.h"
  83 #include "BIF_toolbox.h"
  84
  85 #include "BDR_sculptmode.h"
  86
  87 #include "BSE_drawview.h"
  88 #include "BSE_edit.h"
  89 #include "BSE_view.h"
  90
  91 #include "IMB_imbuf_types.h"
  92
  93 #include "blendef.h"
  94 #include "multires.h"
  95 #include "mydevice.h"
  96
  97 #include "RE_render_ext.h"
  98 #include "RE_shader_ext.h" /*for multitex_ext*/
  99
 100 #include "GPU_draw.h"
 101
 102 #include <math.h>
 103 #include <stdlib.h>
 104 #include <string.h>
 105
 106 /* Number of vertices to average in order to determine the flatten distance */
 107 #define FLATTEN_SAMPLE_SIZE 10
 108
 109 /* Texture cache size */
 110 #define TC_SIZE 256
 111
 112 /* ===== STRUCTS =====
 113  *
 114  */
 115
 116 /* Used by vertex_users to store face indices in a list */
 117 typedef struct IndexNode {
 118         struct IndexNode* next,* prev;
 119         int Index;
 120 } IndexNode;
 121
 122
 123 /* ActiveData stores an Index into the mvert array of Mesh, plus Fade, which
 124    stores how far the vertex is from the brush center, scaled to the range [0,1]. */
 125 typedef struct ActiveData {
 126         struct ActiveData *next, *prev;
 127         unsigned int Index;
 128         float Fade;
 129         float dist;
 130 } ActiveData;
 131
 132 typedef struct BrushActionSymm {
 133         float center_3d[3];
 134         char index;
 135
 136         float up[3], right[3], out[3];
 137
 138         /* Grab brush */
 139         float grab_delta[3];
 140 } BrushActionSymm;
 141
 142 typedef struct BrushAction {
 143         BrushActionSymm symm;
 144
 145         char firsttime;
 146
 147         /* Some brushes need access to original mesh vertices */
 148         vec3f *mesh_store;
 149         short (*orig_norms)[3];
 150
 151         short mouse[2];
 152         float size_3d;
 153
 154         float prev_radius;
 155         float radius;
 156
 157         float *layer_disps;
 158         char flip;
 159
 160         char clip[3];
 161         float cliptol[3];
 162
 163         float anchored_rot;
 164
 165         /* Grab brush */
 166         ListBase grab_active_verts[8];
 167         float depth;
 168
 169         /* Adjust brush strength along each axis
 170            to adjust for object scaling */
 171         float scale[3];
 172 } BrushAction;
 173
 174 typedef struct RectNode {
 175         struct RectNode *next, *prev;
 176         rcti r;
 177 } RectNode;
 178
 179 /* Used to store to 2D screen coordinates of each vertex in the mesh. */
 180 typedef struct ProjVert {
 181         short co[2];
 182
 183         /* Used to mark whether a vertex is inside a rough bounding box
 184            containing the brush. */
 185         char inside;
 186 } ProjVert;
 187
 188 static Object *active_ob= NULL;
 189
 190 SculptData *sculpt_data(void)
 191 {
 192         return &G.scene->sculptdata;
 193 }
 194
 195 static void sculpt_init_session(void);
 196 static void init_brushaction(BrushAction *a, short *, short *);
 197 static void sculpt_undo_push(const short);
 198
 199 SculptSession *sculpt_session(void)
 200 {
 201         if(!sculpt_data()->session)
 202                 sculpt_init_session();
 203         return sculpt_data()->session;
 204 }
 205
 206 /* ===== MEMORY =====
 207  *
 208  * Allocate/initialize/free data
 209  */
 210
 211 static void sculpt_init_session(void)
 212 {
 213         if(sculpt_data()->session)
 214                 sculptsession_free(G.scene);
 215         sculpt_data()->session= MEM_callocN(sizeof(SculptSession), "SculptSession");
 216 }
 217
 218 /* vertex_users is an array of Lists that store all the faces that use a
 219    particular vertex. vertex_users is in the same order as mesh.mvert */
 220 static void calc_vertex_users()
 221 {
 222         SculptSession *ss= sculpt_session();
 223         int i,j;
 224         IndexNode *node= NULL;
 225         Mesh *me= get_mesh(OBACT);
 226
 227         sculpt_vertexusers_free(ss);
 228
 229         /* For efficiency, use vertex_users_mem as a memory pool (may be larger
 230            than necessary if mesh has triangles, but only one alloc is needed.) */
 231         ss->vertex_users= MEM_callocN(sizeof(ListBase) * me->totvert, "vertex_users");
 232         ss->vertex_users_size= me->totvert;
 233         ss->vertex_users_mem= MEM_callocN(sizeof(IndexNode)*me->totface*4, "vertex_users_mem");
 234         node= ss->vertex_users_mem;
 235
 236         /* Find the users */
 237         for(i=0; i<me->totface; ++i){
 238                 for(j=0; j<(me->mface[i].v4?4:3); ++j, ++node) {
 239                         node->Index=i;
 240                         BLI_addtail(&ss->vertex_users[((unsigned int*)(&me->mface[i]))[j]], node);
 241                 }
 242         }
 243 }
 244
 245 /* ===== INTERFACE =====
 246  */
 247
 248 void sculptmode_rem_tex(void *junk0,void *junk1)
 249 {
 250         MTex *mtex= G.scene->sculptdata.mtex[G.scene->sculptdata.texact];
 251         if(mtex) {
 252                 SculptSession *ss= sculpt_session();
 253                 if(mtex->tex) mtex->tex->id.us--;
 254                 MEM_freeN(mtex);
 255                 G.scene->sculptdata.mtex[G.scene->sculptdata.texact]= NULL;
 256                 /* Clear brush preview */
 257                 if(ss->texcache) {
 258                         MEM_freeN(ss->texcache);
 259                         ss->texcache= NULL;
 260                 }
 261                 BIF_undo_push("Unlink brush texture");
 262                 allqueue(REDRAWBUTSEDIT, 0);
 263                 allqueue(REDRAWOOPS, 0);
 264         }
 265 }
 266
 267 /* ===== OPENGL =====
 268  *
 269  * Simple functions to get data from the GL
 270  */
 271
 272 /* Store the modelview and projection matrices and viewport. */
 273 void init_sculptmatrices()
 274 {
 275         SculptSession *ss= sculpt_session();
 276
 277         glMatrixMode(GL_MODELVIEW);
 278         glPushMatrix();
 279         glMultMatrixf(OBACT->obmat);
 280
 281         if(!ss->mats)
 282                 ss->mats = MEM_callocN(sizeof(bglMats), "sculpt bglmats");
 283         bgl_get_mats(ss->mats);
 284
 285         glPopMatrix();
 286
 287 }
 288
 289 /* Uses window coordinates (x,y) to find the depth in the GL depth buffer. If
 290    available, G.vd->depths is used so that the brush doesn't sculpt on top of
 291    itself (G.vd->depths is only updated at the end of a brush stroke.) */
 292 float get_depth(short x, short y)
 293 {
 294         float depth;
 295
 296         if(x<0 || y<0) return 1;
 297         if(x>=curarea->winx || y>=curarea->winy) return 1;
 298
 299         if(G.vd->depths && x<G.vd->depths->w && y<G.vd->depths->h)
 300                 return G.vd->depths->depths[y*G.vd->depths->w+x];
 301
 302         x+= curarea->winrct.xmin;
 303         y+= curarea->winrct.ymin;
 304
 305         glReadPixels(x, y, 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &depth);
 306
 307         return depth;
 308 }
 309
 310 /* Uses window coordinates (x,y) and depth component z to find a point in
 311    modelspace */
 312 void unproject(float out[3], const short x, const short y, const float z)
 313 {
 314         SculptSession *ss= sculpt_session();
 315         double ux, uy, uz;
 316
 317         gluUnProject(x,y,z, ss->mats->modelview, ss->mats->projection,
 318                      (GLint *)ss->mats->viewport, &ux, &uy, &uz );
 319         out[0] = ux;
 320         out[1] = uy;
 321         out[2] = uz;
 322 }
 323
 324 /* Convert a point in model coordinates to 2D screen coordinates. */
 325 static void projectf(const float v[3], float p[2])
 326 {
 327         SculptSession *ss= sculpt_session();
 328         double ux, uy, uz;
 329
 330         gluProject(v[0],v[1],v[2], ss->mats->modelview, ss->mats->projection,
 331                    (GLint *)ss->mats->viewport, &ux, &uy, &uz);
 332         p[0]= ux;
 333         p[1]= uy;
 334 }
 335
 336 static void project(const float v[3], short p[2])
 337 {
 338         float f[2];
 339         projectf(v, f);
 340
 341         p[0]= f[0];
 342         p[1]= f[1];
 343 }
 344
 345 /* ===== Sculpting =====
 346  *
 347  */
 348
 349 /* Return modified brush size. Uses current tablet pressure (if available) to
 350    shrink the brush. Skipped for grab brush because only the first mouse down
 351    size is used, which is small if the user has just touched the pen to the
 352    tablet */
 353 char brush_size()
 354 {
 355         const BrushData *b= sculptmode_brush();
 356         float size= b->size;
 357         float pressure= get_pressure();
 358         short activedevice= get_activedevice();
 359
 360         if(sculpt_data()->brush_type!=GRAB_BRUSH) {
 361                 const float size_factor= G.scene->sculptdata.tablet_size / 10.0f;
 362                 if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
 363                         size*= G.scene->sculptdata.tablet_size==0?1:
 364                                (1-size_factor) + pressure*size_factor;
 365         }
 366
 367         return size;
 368 }
 369
 370 /* Return modified brush strength. Includes the direction of the brush, positive
 371    values pull vertices, negative values push. Uses tablet pressure and a
 372    special multiplier found experimentally to scale the strength factor. */
 373 float brush_strength(BrushAction *a)
 374 {
 375         const BrushData* b= sculptmode_brush();
 376         float dir= b->dir==1 ? 1 : -1;
 377         float pressure= 1;
 378         short activedevice= get_activedevice();
 379         float flip= a->flip ? -1:1;
 380         float anchored = b->flag & SCULPT_BRUSH_ANCHORED ? 25 : 1;
 381
 382         const float strength_factor= G.scene->sculptdata.tablet_strength / 10.0f;
 383         if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
 384                 pressure= G.scene->sculptdata.tablet_strength==0?1:
 385                                   (1-strength_factor) + get_pressure()*strength_factor;
 386
 387         /* Flip direction for eraser */
 388         if(activedevice==DEV_ERASER)
 389                 dir= -dir;
 390
 391         switch(G.scene->sculptdata.brush_type){
 392         case DRAW_BRUSH:
 393         case LAYER_BRUSH:
 394                 return b->strength / 5000.0f * dir * pressure * flip * anchored * G.vd->grid;
 395         case SMOOTH_BRUSH:
 396                 return b->strength / 50.0f * pressure * anchored;
 397         case PINCH_BRUSH:
 398                 return b->strength / 1000.0f * dir * pressure * flip * anchored;
 399         case GRAB_BRUSH:
 400                 return 1;
 401         case INFLATE_BRUSH:
 402                 return b->strength / 5000.0f * dir * pressure * flip * anchored;
 403         case FLATTEN_BRUSH:
 404                 return b->strength / 500.0f * pressure * anchored;
 405         default:
 406                 return 0;
 407         }
 408 }
 409
 410 /* For clipping against a mirror modifier */
 411 void sculpt_clip(const BrushAction *a, float *co, const float val[3])
 412 {
 413         char i;
 414         for(i=0; i<3; ++i) {
 415                 if(a->clip[i] && (fabs(co[i]) <= a->cliptol[i]))
 416                         co[i]= 0.0f;
 417                 else
 418                         co[i]= val[i];
 419         }
 420 }
 421
 422 void sculpt_axislock(float *co)
 423 {
 424         SculptData *sd = sculpt_data();
 425         if (sd->axislock == AXISLOCK_X+AXISLOCK_Y+AXISLOCK_Z) return;
 426         if (G.vd->twmode == V3D_MANIP_LOCAL) {
 427                 float mat[3][3], imat[3][3];
 428                 Mat3CpyMat4(mat, OBACT->obmat);
 429                 Mat3Inv(imat, mat);
 430                 Mat3MulVecfl(mat, co);
 431                 if (sd->axislock & AXISLOCK_X) co[0] = 0.0;
 432                 if (sd->axislock & AXISLOCK_Y) co[1] = 0.0;
 433                 if (sd->axislock & AXISLOCK_Z) co[2] = 0.0;
 434                 Mat3MulVecfl(imat, co);
 435         } else {
 436                 if (sd->axislock & AXISLOCK_X) co[0] = 0.0;
 437                 if (sd->axislock & AXISLOCK_Y) co[1] = 0.0;
 438                 if (sd->axislock & AXISLOCK_Z) co[2] = 0.0;
 439         }
 440 }
 441
 442 static void add_norm_if(float view_vec[3], float out[3], float out_flip[3], const short no[3])
 443 {
 444         float fno[3] = {no[0], no[1], no[2]};
 445
 446         Normalize(fno);
 447
 448         if((Inpf(view_vec, fno)) > 0) {
 449                 VecAddf(out, out, fno);
 450         } else {
 451                 VecAddf(out_flip, out_flip, fno); /* out_flip is used when out is {0,0,0} */
 452         }
 453 }
 454
 455 /* Currently only for the draw brush; finds average normal for all active
 456    vertices */
 457 void calc_area_normal(float out[3], const BrushAction *a, const float *outdir, const ListBase* active_verts)
 458 {
 459         Mesh *me = get_mesh(OBACT);
 460         ActiveData *node = active_verts->first;
 461         SculptData *sd = sculpt_data();
 462         const int view = sd->brush_type==DRAW_BRUSH ? sculptmode_brush()->view : 0;
 463         float out_flip[3];
 464
 465         out[0]=out[1]=out[2] = out_flip[0]=out_flip[1]=out_flip[2] = 0;
 466
 467         if(sculptmode_brush()->flag & SCULPT_BRUSH_ANCHORED) {
 468                 for(; node; node = node->next)
 469                         add_norm_if(((BrushAction*)a)->symm.out, out, out_flip, a->orig_norms[node->Index]);
 470         }
 471         else {
 472                 for(; node; node = node->next)
 473                         add_norm_if(((BrushAction*)a)->symm.out, out, out_flip, me->mvert[node->Index].no);
 474         }
 475
 476         if (out[0]==0.0 && out[1]==0.0 && out[2]==0.0) {
 477                 VECCOPY(out, out_flip);
 478         }
 479
 480         Normalize(out);
 481
 482         if(outdir) {
 483                 out[0] = outdir[0] * view + out[0] * (10-view);
 484                 out[1] = outdir[1] * view + out[1] * (10-view);
 485                 out[2] = outdir[2] * view + out[2] * (10-view);
 486         }
 487
 488         Normalize(out);
 489 }
 490
 491 void do_draw_brush(const BrushAction *a, const ListBase* active_verts)
 492 {
 493         Mesh *me= get_mesh(OBACT);
 494         float area_normal[3];
 495         ActiveData *node= active_verts->first;
 496
 497         calc_area_normal(area_normal, a, a->symm.out, active_verts);
 498
 499         sculpt_axislock(area_normal);
 500
 501         while(node){
 502                 float *co= me->mvert[node->Index].co;
 503
 504                 const float val[3]= {co[0]+area_normal[0]*node->Fade*a->scale[0],
 505                                      co[1]+area_normal[1]*node->Fade*a->scale[1],
 506                                      co[2]+area_normal[2]*node->Fade*a->scale[2]};
 507
 508                 sculpt_clip(a, co, val);
 509
 510                 node= node->next;
 511         }
 512 }
 513
 514 /* For the smooth brush, uses the neighboring vertices around vert to calculate
 515    a smoothed location for vert. Skips corner vertices (used by only one
 516    polygon.) */
 517 vec3f neighbor_average(const int vert)
 518 {
 519         SculptSession *ss= sculpt_session();
 520         Mesh *me= get_mesh(OBACT);
 521         int i, skip= -1, total=0;
 522         IndexNode *node= ss->vertex_users[vert].first;
 523         vec3f avg= {0,0,0};
 524         char ncount= BLI_countlist(&ss->vertex_users[vert]);
 525         MFace *f;
 526
 527         /* Don't modify corner vertices */
 528         if(ncount==1) {
 529                 VecCopyf(&avg.x, me->mvert[vert].co);
 530                 return avg;
 531         }
 532
 533         while(node){
 534                 f= &me->mface[node->Index];
 535
 536                 if(f->v4) {
 537                         skip= (f->v1==vert?2:
 538                                f->v2==vert?3:
 539                                f->v3==vert?0:
 540                                f->v4==vert?1:-1);
 541                 }
 542
 543                 for(i=0; i<(f->v4?4:3); ++i) {
 544                         if(i != skip && (ncount!=2 || BLI_countlist(&ss->vertex_users[(&f->v1)[i]]) <= 2)) {
 545                                 VecAddf(&avg.x,&avg.x,me->mvert[(&f->v1)[i]].co);
 546                                 ++total;
 547                         }
 548                 }
 549
 550                 node= node->next;
 551         }
 552
 553         if(total>0) {
 554                 avg.x/= total;
 555                 avg.y/= total;
 556                 avg.z/= total;
 557         }
 558         else
 559                 VecCopyf(&avg.x, me->mvert[vert].co);
 560
 561         return avg;
 562 }
 563
 564 void do_smooth_brush(const BrushAction *a, const ListBase* active_verts)
 565 {
 566         ActiveData *node= active_verts->first;
 567         Mesh *me= get_mesh(OBACT);
 568
 569         while(node){
 570                 float *co= me->mvert[node->Index].co;
 571                 const vec3f avg= neighbor_average(node->Index);
 572                 const float val[3]= {co[0]+(avg.x-co[0])*node->Fade,
 573                                      co[1]+(avg.y-co[1])*node->Fade,
 574                                      co[2]+(avg.z-co[2])*node->Fade};
 575                 sculpt_clip(a, co, val);
 576                 node= node->next;
 577         }
 578 }
 579
 580 void do_pinch_brush(const BrushAction *a, const ListBase* active_verts)
 581 {
 582         Mesh *me= get_mesh(OBACT);
 583         ActiveData *node= active_verts->first;
 584
 585         while(node) {
 586                 float *co= me->mvert[node->Index].co;
 587                 const float val[3]= {co[0]+(a->symm.center_3d[0]-co[0])*node->Fade,
 588                                      co[1]+(a->symm.center_3d[1]-co[1])*node->Fade,
 589                                      co[2]+(a->symm.center_3d[2]-co[2])*node->Fade};
 590                 sculpt_clip(a, co, val);
 591                 node= node->next;
 592         }
 593 }
 594
 595 void do_grab_brush(BrushAction *a)
 596 {
 597         Mesh *me= get_mesh(OBACT);
 598         ActiveData *node= a->grab_active_verts[a->symm.index].first;
 599         float add[3];
 600         float grab_delta[3];
 601
 602         VecCopyf(grab_delta, a->symm.grab_delta);
 603         sculpt_axislock(grab_delta);
 604
 605         while(node) {
 606                 float *co= me->mvert[node->Index].co;
 607
 608                 VecCopyf(add, grab_delta);
 609                 VecMulf(add, node->Fade);
 610                 VecAddf(add, add, co);
 611                 sculpt_clip(a, co, add);
 612
 613                 node= node->next;
 614         }
 615
 616 }
 617
 618 void do_layer_brush(BrushAction *a, const ListBase *active_verts)
 619 {
 620         Mesh *me= get_mesh(OBACT);
 621         float area_normal[3];
 622         ActiveData *node= active_verts->first;
 623         const float bstr= brush_strength(a);
 624
 625         calc_area_normal(area_normal, a, NULL, active_verts);
 626
 627         while(node){
 628                 float *disp= &a->layer_disps[node->Index];
 629
 630                 if((bstr > 0 && *disp < bstr) ||
 631                   (bstr < 0 && *disp > bstr)) {
 632                         float *co= me->mvert[node->Index].co;
 633
 634                         *disp+= node->Fade;
 635
 636                         if(bstr < 0) {
 637                                 if(*disp < bstr)
 638                                         *disp = bstr;
 639                         } else {
 640                                 if(*disp > bstr)
 641                                         *disp = bstr;
 642                         }
 643
 644                         {
 645                                 const float val[3]= {a->mesh_store[node->Index].x+area_normal[0] * *disp*a->scale[0],
 646                                                      a->mesh_store[node->Index].y+area_normal[1] * *disp*a->scale[1],
 647                                                      a->mesh_store[node->Index].z+area_normal[2] * *disp*a->scale[2]};
 648                                 sculpt_clip(a, co, val);
 649                         }
 650                 }
 651
 652                 node= node->next;
 653         }
 654 }
 655
 656 void do_inflate_brush(const BrushAction *a, const ListBase *active_verts)
 657 {
 658         ActiveData *node= active_verts->first;
 659         float add[3];
 660         Mesh *me= get_mesh(OBACT);
 661
 662         while(node) {
 663                 float *co= me->mvert[node->Index].co;
 664                 short *no= me->mvert[node->Index].no;
 665
 666                 add[0]= no[0]/ 32767.0f;
 667                 add[1]= no[1]/ 32767.0f;
 668                 add[2]= no[2]/ 32767.0f;
 669                 VecMulf(add, node->Fade);
 670                 add[0]*= a->scale[0];
 671                 add[1]*= a->scale[1];
 672                 add[2]*= a->scale[2];
 673                 VecAddf(add, add, co);
 674
 675                 sculpt_clip(a, co, add);
 676
 677                 node= node->next;
 678         }
 679 }
 680
 681 void calc_flatten_center(Mesh *me, ActiveData *node, float co[3])
 682 {
 683         ActiveData *outer[FLATTEN_SAMPLE_SIZE];
 684         int i;
 685
 686         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
 687                 outer[i] = node;
 688
 689         for(; node; node = node->next) {
 690                 for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i) {
 691                         if(node->dist > outer[i]->dist) {
 692                                 outer[i] = node;
 693                                 break;
 694                         }
 695                 }
 696         }
 697
 698         co[0] = co[1] = co[2] = 0.0f;
 699         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
 700                 VecAddf(co, co, me->mvert[outer[i]->Index].co);
 701         VecMulf(co, 1.0f / FLATTEN_SAMPLE_SIZE);
 702 }
 703
 704 void do_flatten_brush(const BrushAction *a, const ListBase *active_verts)
 705 {
 706         Mesh *me= get_mesh(OBACT);
 707         ActiveData *node= active_verts->first;
 708         /* area_normal and cntr define the plane towards which vertices are squashed */
 709         float area_normal[3];
 710         float cntr[3];
 711
 712         calc_area_normal(area_normal, a, a->symm.out, active_verts);
 713         calc_flatten_center(me, node, cntr);
 714
 715         while(node){
 716                 float *co= me->mvert[node->Index].co;
 717                 float p1[3], sub1[3], sub2[3], intr[3], val[3];
 718
 719                 /* Find the intersection between squash-plane and vertex (along the area normal) */
 720                 VecSubf(p1, co, area_normal);
 721                 VecSubf(sub1, cntr, p1);
 722                 VecSubf(sub2, co, p1);
 723                 VecSubf(intr, co, p1);
 724                 VecMulf(intr, Inpf(area_normal, sub1) / Inpf(area_normal, sub2));
 725                 VecAddf(intr, intr, p1);
 726
 727                 VecSubf(val, intr, co);
 728                 VecMulf(val, node->Fade);
 729                 VecAddf(val, val, co);
 730
 731                 sculpt_clip(a, co, val);
 732
 733                 node= node->next;
 734         }
 735 }
 736
 737 /* Uses the brush curve control to find a strength value between 0 and 1 */
 738 float curve_strength(float p, const float len)
 739 {
 740         if(p > len) p= len;
 741         return curvemapping_evaluateF(G.scene->sculptdata.cumap, 0, p/len);
 742 }
 743
 744 /* Uses symm to selectively flip any axis of a coordinate. */
 745 void flip_coord(float co[3], const char symm)
 746 {
 747         if(symm & SYMM_X)
 748                 co[0]= -co[0];
 749         if(symm & SYMM_Y)
 750                 co[1]= -co[1];
 751         if(symm & SYMM_Z)
 752                 co[2]= -co[2];
 753 }
 754
 755 /* Use the warpfac field in MTex to store a rotation value for sculpt textures. Value is in degrees */
 756 float sculpt_tex_angle(void)
 757 {
 758         SculptData *sd= sculpt_data();
 759         if(sd->texact!=-1 && sd->mtex[sd->texact])
 760                 return sd->mtex[sd->texact]->warpfac;
 761         return 0;
 762 }
 763
 764 void set_tex_angle(const float f)
 765 {
 766         SculptData *sd = sculpt_data();
 767         if(sd->texact != -1 && sd->mtex[sd->texact])
 768                 sd->mtex[sd->texact]->warpfac = f;
 769 }
 770
 771 float to_rad(const float deg)
 772 {
 773         return deg * (M_PI/180.0f);
 774 }
 775
 776 float to_deg(const float rad)
 777 {
 778         return rad * (180.0f/M_PI);
 779 }
 780
 781 /* Get a pixel from the texcache at (px, py) */
 782 static unsigned char get_texcache_pixel(const SculptSession *ss, int px, int py)
 783 {
 784         unsigned *p;
 785         p = ss->texcache + py * ss->texcache_w + px;
 786         return ((unsigned char*)(p))[0];
 787 }
 788
 789 static float get_texcache_pixel_bilinear(const SculptSession *ss, float u, float v)
 790 {
 791         int x, y, x2, y2;
 792         const int tc_max = TC_SIZE - 1;
 793         float urat, vrat, uopp;
 794
 795         if(u < 0) u = 0;
 796         else if(u >= TC_SIZE) u = tc_max;
 797         if(v < 0) v = 0;
 798         else if(v >= TC_SIZE) v = tc_max;
 799
 800         x = floor(u);
 801         y = floor(v);
 802         x2 = x + 1;
 803         y2 = y + 1;
 804
 805         if(x2 > TC_SIZE) x2 = tc_max;
 806         if(y2 > TC_SIZE) y2 = tc_max;
 807
 808         urat = u - x;
 809         vrat = v - y;
 810         uopp = 1 - urat;
 811
 812         return ((get_texcache_pixel(ss, x, y) * uopp +
 813                  get_texcache_pixel(ss, x2, y) * urat) * (1 - vrat) +
 814                 (get_texcache_pixel(ss, x, y2) * uopp +
 815                  get_texcache_pixel(ss, x2, y2) * urat) * vrat) / 255.0;
 816 }
 817
 818 /* Return a multiplier for brush strength on a particular vertex. */
 819 float tex_strength(BrushAction *a, float *point, const float len,const unsigned vindex)
 820 {
 821         SculptData *sd= sculpt_data();
 822         SculptSession *ss= sculpt_session();
 823         float avg= 1;
 824
 825         if(sd->texact==-1 || !sd->mtex[sd->texact])
 826                 avg= 1;
 827         else if(sd->texrept==SCULPTREPT_3D) {
 828                 /* Get strength by feeding the vertex location directly
 829                    into a texture */
 830                 float jnk;
 831                 const float factor= 0.01;
 832                 MTex mtex;
 833                 memset(&mtex,0,sizeof(MTex));
 834                 mtex.tex= sd->mtex[sd->texact]->tex;
 835                 mtex.projx= 1;
 836                 mtex.projy= 2;
 837                 mtex.projz= 3;
 838                 VecCopyf(mtex.size, sd->mtex[sd->texact]->size);
 839                 VecMulf(mtex.size, factor);
 840                 if(!sd->texsep)
 841                         mtex.size[1]= mtex.size[2]= mtex.size[0];
 842
 843                 externtex(&mtex,point,&avg,&jnk,&jnk,&jnk,&jnk);
 844         }
 845         else if(ss->texcache) {
 846                 const float bsize= a->radius * 2;
 847                 const float rot= to_rad(sculpt_tex_angle()) + a->anchored_rot;
 848                 int px, py;
 849                 float flip[3], point_2d[2];
 850
 851                 /* If the active area is being applied for symmetry, flip it
 852                    across the symmetry axis in order to project it. This insures
 853                    that the brush texture will be oriented correctly. */
 854                 VecCopyf(flip, point);
 855                 flip_coord(flip, a->symm.index);
 856                 projectf(flip, point_2d);
 857
 858                 /* For Tile and Drag modes, get the 2D screen coordinates of the
 859                    and scale them up or down to the texture size. */
 860                 if(sd->texrept==SCULPTREPT_TILE) {
 861                         const int sx= (const int)sd->mtex[sd->texact]->size[0];
 862                         const int sy= (const int)sd->texsep ? sd->mtex[sd->texact]->size[1] : sx;
 863
 864                         float fx= point_2d[0];
 865                         float fy= point_2d[1];
 866
 867                         float angle= atan2(fy, fx) - rot;
 868                         float flen= sqrtf(fx*fx + fy*fy);
 869
 870                         if(rot<0.001 && rot>-0.001) {
 871                                 px= point_2d[0];
 872                                 py= point_2d[1];
 873                         } else {
 874                                 px= flen * cos(angle) + 2000;
 875                                 py= flen * sin(angle) + 2000;
 876                         }
 877                         if(sx != 1)
 878                                 px %= sx-1;
 879                         if(sy != 1)
 880                                 py %= sy-1;
 881                         avg= get_texcache_pixel_bilinear(ss, TC_SIZE*px/sx, TC_SIZE*py/sy);
 882                 } else {
 883                         float fx= (point_2d[0] - a->mouse[0]) / bsize;
 884                         float fy= (point_2d[1] - a->mouse[1]) / bsize;
 885
 886                         float angle= atan2(fy, fx) - rot;
 887                         float flen= sqrtf(fx*fx + fy*fy);
 888
 889                         fx = flen * cos(angle) + 0.5;
 890                         fy = flen * sin(angle) + 0.5;
 891
 892                         avg= get_texcache_pixel_bilinear(ss, fx * TC_SIZE, fy * TC_SIZE);
 893                 }
 894         }
 895
 896         if(sd->texfade)
 897                 avg*= curve_strength(len, a->size_3d); /* Smooth curve */
 898
 899         return avg;
 900 }
 901
 902 /* Mark area around the brush as damaged. projverts are marked if they are
 903    inside the area and the damaged rectangle in 2D screen coordinates is
 904    added to damaged_rects. */
 905 void sculpt_add_damaged_rect(BrushAction *a)
 906 {
 907         short p[2];
 908         RectNode *rn= MEM_mallocN(sizeof(RectNode),"RectNode");
 909         Mesh *me= get_mesh(OBACT);
 910         SculptSession *ss = sculpt_session();
 911         const float radius = a->radius > a->prev_radius ? a->radius : a->prev_radius;
 912         unsigned i;
 913
 914         /* Find center */
 915         project(a->symm.center_3d, p);
 916         rn->r.xmin= p[0] - radius;
 917         rn->r.ymin= p[1] - radius;
 918         rn->r.xmax= p[0] + radius;
 919         rn->r.ymax= p[1] + radius;
 920
 921         BLI_addtail(&sculpt_session()->damaged_rects, rn);
 922
 923         /* Update insides */
 924         for(i=0; i<me->totvert; ++i) {
 925                 if(!ss->projverts[i].inside) {
 926                         if(ss->projverts[i].co[0] > rn->r.xmin && ss->projverts[i].co[1] > rn->r.ymin &&
 927                            ss->projverts[i].co[0] < rn->r.xmax && ss->projverts[i].co[1] < rn->r.ymax) {
 928                                 ss->projverts[i].inside= 1;
 929                         }
 930                 }
 931         }
 932 }
 933
 934 /* Clears the depth buffer in each modified area. */
 935 void sculpt_clear_damaged_areas(SculptSession *ss)
 936 {
 937         RectNode *rn= NULL;
 938
 939         for(rn = ss->damaged_rects.first; rn; rn = rn->next) {
 940                 rcti clp = rn->r;
 941                 rcti *win = &curarea->winrct;
 942
 943                 clp.xmin += win->xmin;
 944                 clp.xmax += win->xmin;
 945                 clp.ymin += win->ymin;
 946                 clp.ymax += win->ymin;
 947
 948                 if(clp.xmin < win->xmax && clp.xmax > win->xmin &&
 949                    clp.ymin < win->ymax && clp.ymax > win->ymin) {
 950                         if(clp.xmin < win->xmin) clp.xmin = win->xmin;
 951                         if(clp.ymin < win->ymin) clp.ymin = win->ymin;
 952                         if(clp.xmax > win->xmax) clp.xmax = win->xmax;
 953                         if(clp.ymax > win->ymax) clp.ymax = win->ymax;
 954
 955                         glScissor(clp.xmin + 1, clp.ymin + 1,
 956                                   clp.xmax - clp.xmin - 2,
 957                                   clp.ymax - clp.ymin - 2);
 958                 }
 959
 960                 glClear(GL_DEPTH_BUFFER_BIT);
 961         }
 962 }
 963
 964 void do_brush_action(BrushAction *a)
 965 {
 966         int i;
 967         float av_dist;
 968         ListBase active_verts={0,0};
 969         ActiveData *adata= 0;
 970         float *vert;
 971         Mesh *me= get_mesh(OBACT);
 972         const float bstrength= brush_strength(a);
 973         KeyBlock *keyblock= ob_get_keyblock(OBACT);
 974         SculptData *sd = sculpt_data();
 975         SculptSession *ss = sculpt_session();
 976
 977         sculpt_add_damaged_rect(a);
 978
 979         /* Build a list of all vertices that are potentially within the brush's
 980            area of influence. Only do this once for the grab brush. */
 981         if((sd->brush_type != GRAB_BRUSH) || a->firsttime) {
 982                 for(i=0; i<me->totvert; ++i) {
 983                         /* Projverts.inside provides a rough bounding box */
 984                         if(ss->projverts[i].inside) {
 985                                 vert= ss->vertexcosnos ? &ss->vertexcosnos[i*6] : me->mvert[i].co;
 986                                 av_dist= VecLenf(a->symm.center_3d, vert);
 987                                 if(av_dist < a->size_3d) {
 988                                         adata= (ActiveData*)MEM_mallocN(sizeof(ActiveData), "ActiveData");
 989
 990                                         adata->Index = i;
 991                                         /* Fade is used to store the final strength at which the brush
 992                                            should modify a particular vertex. */
 993                                         adata->Fade= tex_strength(a, vert, av_dist, i) * bstrength;
 994                                         adata->dist = av_dist;
 995
 996                                         if(sd->brush_type == GRAB_BRUSH && a->firsttime)
 997                                                 BLI_addtail(&a->grab_active_verts[a->symm.index], adata);
 998                                         else
 999                                                 BLI_addtail(&active_verts, adata);
1000                                 }
1001                         }
1002                 }
1003         }
1004
1005         /* Only act if some verts are inside the brush area */
1006         if(active_verts.first || (sd->brush_type == GRAB_BRUSH && a->grab_active_verts[a->symm.index].first)) {
1007                 /* Apply one type of brush action */
1008                 switch(G.scene->sculptdata.brush_type){
1009                 case DRAW_BRUSH:
1010                         do_draw_brush(a, &active_verts);
1011                         break;
1012                 case SMOOTH_BRUSH:
1013                         do_smooth_brush(a, &active_verts);
1014                         break;
1015                 case PINCH_BRUSH:
1016                         do_pinch_brush(a, &active_verts);
1017                         break;
1018                 case INFLATE_BRUSH:
1019                         do_inflate_brush(a, &active_verts);
1020                         break;
1021                 case GRAB_BRUSH:
1022                         do_grab_brush(a);
1023                         break;
1024                 case LAYER_BRUSH:
1025                         do_layer_brush(a, &active_verts);
1026                         break;
1027                 case FLATTEN_BRUSH:
1028                         do_flatten_brush(a, &active_verts);
1029                         break;
1030                 }
1031
1032                 /* Copy the modified vertices from mesh to the active key */
1033                 if(keyblock) {
1034                         float *co= keyblock->data;
1035                         if(co) {
1036                                 if(sd->brush_type == GRAB_BRUSH)
1037                                         adata = a->grab_active_verts[a->symm.index].first;
1038                                 else
1039                                         adata = active_verts.first;
1040
1041                                 for(; adata; adata= adata->next)
1042                                         if(adata->Index < keyblock->totelem)
1043                                                 VecCopyf(&co[adata->Index*3], me->mvert[adata->Index].co);
1044                         }
1045                 }
1046
1047                 if(ss->vertexcosnos)
1048                         BLI_freelistN(&active_verts);
1049                 else {
1050                         if(sd->brush_type != GRAB_BRUSH)
1051                                 addlisttolist(&ss->damaged_verts, &active_verts);
1052                 }
1053         }
1054 }
1055
1056 /* Flip all the editdata across the axis/axes specified by symm. Used to
1057    calculate multiple modifications to the mesh when symmetry is enabled. */
1058 void calc_brushdata_symm(BrushAction *a, const char symm)
1059 {
1060         flip_coord(a->symm.center_3d, symm);
1061         flip_coord(a->symm.up, symm);
1062         flip_coord(a->symm.right, symm);
1063         flip_coord(a->symm.out, symm);
1064
1065         a->symm.index= symm;
1066
1067         flip_coord(a->symm.grab_delta, symm);
1068 }
1069
1070 void do_symmetrical_brush_actions(BrushAction *a, short co[2], short pr_co[2])
1071 {
1072         const char symm = sculpt_data()->symm;
1073         BrushActionSymm orig;
1074         int i;
1075
1076         init_brushaction(a, co, pr_co);
1077         orig = a->symm;
1078         do_brush_action(a);
1079
1080         for(i = 1; i <= symm; ++i) {
1081                 if(symm & i && (symm != 5 || i != 3) && (symm != 6 || (i != 3 && i != 5))) {
1082                         // Restore the original symmetry data
1083                         a->symm = orig;
1084
1085                         calc_brushdata_symm(a, i);
1086                         do_brush_action(a);
1087                 }
1088         }
1089
1090         a->symm = orig;
1091 }
1092
1093 void add_face_normal(vec3f *norm, const MFace* face)
1094 {
1095         Mesh *me= get_mesh(OBACT);
1096
1097         vec3f c= {me->mvert[face->v1].co[0],me->mvert[face->v1].co[1],me->mvert[face->v1].co[2]};
1098         vec3f b= {me->mvert[face->v2].co[0],me->mvert[face->v2].co[1],me->mvert[face->v2].co[2]};
1099         vec3f a= {me->mvert[face->v3].co[0],me->mvert[face->v3].co[1],me->mvert[face->v3].co[2]};
1100         vec3f s1, s2;
1101
1102         VecSubf(&s1.x,&a.x,&b.x);
1103         VecSubf(&s2.x,&c.x,&b.x);
1104
1105         norm->x+= s1.y * s2.z - s1.z * s2.y;
1106         norm->y+= s1.z * s2.x - s1.x * s2.z;
1107         norm->z+= s1.x * s2.y - s1.y * s2.x;
1108 }
1109
1110 void update_damaged_vert(Mesh *me, ListBase *lb)
1111 {
1112         ActiveData *vert;
1113
1114         for(vert= lb->first; vert; vert= vert->next) {
1115                 vec3f norm= {0,0,0};
1116                 IndexNode *face= sculpt_session()->vertex_users[vert->Index].first;
1117
1118                 while(face){
1119                         add_face_normal(&norm,&me->mface[face->Index]);
1120                         face= face->next;
1121                 }
1122                 Normalize(&norm.x);
1123
1124                 me->mvert[vert->Index].no[0]=norm.x*32767;
1125                 me->mvert[vert->Index].no[1]=norm.y*32767;
1126                 me->mvert[vert->Index].no[2]=norm.z*32767;
1127         }
1128 }
1129
1130 void calc_damaged_verts(ListBase *damaged_verts, BrushAction *a)
1131 {
1132         Mesh *me= get_mesh(OBACT);
1133         int i;
1134
1135         for(i=0; i<8; ++i)
1136                 update_damaged_vert(me, &a->grab_active_verts[i]);
1137
1138         update_damaged_vert(me, damaged_verts);
1139         BLI_freelistN(damaged_verts);
1140         damaged_verts->first = damaged_verts->last = NULL;
1141 }
1142
1143 void projverts_clear_inside(SculptSession *ss)
1144 {
1145         Mesh *me = get_mesh(OBACT);
1146         if(me) {
1147                 int i;
1148                 for(i = 0; i < me->totvert; ++i)
1149                         ss->projverts[i].inside = 0;
1150         }
1151 }
1152
1153 BrushData *sculptmode_brush(void)
1154 {
1155         SculptData *sd= &G.scene->sculptdata;
1156
1157         BrushData *bd =
1158                 (sd->brush_type==DRAW_BRUSH ? &sd->drawbrush :
1159                  sd->brush_type==SMOOTH_BRUSH ? &sd->smoothbrush :
1160                  sd->brush_type==PINCH_BRUSH ? &sd->pinchbrush :
1161                  sd->brush_type==INFLATE_BRUSH ? &sd->inflatebrush :
1162                  sd->brush_type==GRAB_BRUSH ? &sd->grabbrush :
1163                  sd->brush_type==LAYER_BRUSH ? &sd->layerbrush :
1164                  sd->brush_type==FLATTEN_BRUSH ? &sd->flattenbrush : NULL);
1165
1166         if(!bd) {
1167                 sculptdata_init(G.scene);
1168                 bd = &sd->drawbrush;
1169         }
1170
1171         return bd;
1172 }
1173
1174 void sculptmode_update_tex()
1175 {
1176         SculptData *sd= sculpt_data();
1177         SculptSession *ss= sculpt_session();
1178         MTex *mtex;
1179         TexResult texres = {0};
1180         float x, y, step=2.0/TC_SIZE, co[3];
1181         int hasrgb, ix, iy;
1182
1183         /* Skip Default brush shape and non-textures */
1184         if(sd->texact == -1 || !sd->mtex[sd->texact]) return;
1185
1186         mtex = sd->mtex[sd->texact];
1187
1188         if(ss->texcache) {
1189                 MEM_freeN(ss->texcache);
1190                 ss->texcache= NULL;
1191         }
1192
1193         ss->texcache_w = ss->texcache_h = TC_SIZE;
1194         ss->texcache = MEM_callocN(sizeof(int) * ss->texcache_w * ss->texcache_h, "Sculpt Texture cache");
1195
1196         if(mtex && mtex->tex) {
1197                 BKE_image_get_ibuf(sd->mtex[sd->texact]->tex->ima, NULL);
1198
1199                 /*do normalized cannonical view coords for texture*/
1200                 for (y=-1.0, iy=0; iy<TC_SIZE; iy++, y += step) {
1201                         for (x=-1.0, ix=0; ix<TC_SIZE; ix++, x += step) {
1202                                 co[0]= x;
1203                                 co[1]= y;
1204                                 co[2]= 0.0f;
1205
1206                                 /* This is copied from displace modifier code */
1207                                 hasrgb = multitex_ext(mtex->tex, co, NULL, NULL, 1, &texres);
1208
1209                                 /* if the texture gave an RGB value, we assume it didn't give a valid
1210                                  * intensity, so calculate one (formula from do_material_tex).
1211                                  * if the texture didn't give an RGB value, copy the intensity across
1212                                  */
1213                                 if(hasrgb & TEX_RGB)
1214                                         texres.tin = (0.35 * texres.tr + 0.45 *
1215                                                       texres.tg + 0.2 * texres.tb);
1216
1217                                 texres.tin = texres.tin * 255.0;
1218                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4] = (char)texres.tin;
1219                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+1] = (char)texres.tin;
1220                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+2] = (char)texres.tin;
1221                                 ((char*)ss->texcache)[(iy*TC_SIZE+ix)*4+3] = (char)texres.tin;
1222                         }
1223                 }
1224         }
1225 }
1226
1227 /* pr_mouse is only used for the grab brush, can be NULL otherwise */
1228 static void init_brushaction(BrushAction *a, short *mouse, short *pr_mouse)
1229 {
1230         SculptData *sd = sculpt_data();
1231         const float mouse_depth = get_depth(mouse[0], mouse[1]);
1232         float brush_edge_loc[3], zero_loc[3], oldloc[3];
1233         ModifierData *md;
1234         int i;
1235         const char flip = (get_qual() == LR_SHIFTKEY);
1236         const char anchored = sculptmode_brush()->flag & SCULPT_BRUSH_ANCHORED;
1237         short orig_mouse[2], dx=0, dy=0;
1238
1239         a->flip = flip;
1240         a->symm.index = 0;
1241
1242         if(a->firsttime)
1243                 a->depth = mouse_depth;
1244
1245         /* Convert the location and size of the brush to
1246            modelspace coords */
1247         if(a->firsttime || !anchored) {
1248                 unproject(a->symm.center_3d, mouse[0], mouse[1], mouse_depth);
1249                 a->mouse[0] = mouse[0];
1250                 a->mouse[1] = mouse[1];
1251         }
1252
1253         if(anchored) {
1254                 project(a->symm.center_3d, orig_mouse);
1255                 dx = mouse[0] - orig_mouse[0];
1256                 dy = mouse[1] - orig_mouse[1];
1257         }
1258
1259         if(anchored) {
1260                 unproject(brush_edge_loc, mouse[0], mouse[1], a->depth);
1261                 a->anchored_rot = atan2(dy, dx);
1262         }
1263         else
1264                 unproject(brush_edge_loc, mouse[0] + brush_size(), mouse[1], mouse_depth);
1265
1266         a->size_3d = VecLenf(a->symm.center_3d, brush_edge_loc);
1267
1268         a->prev_radius = a->radius;
1269
1270         if(anchored)
1271                 a->radius = sqrt(dx*dx + dy*dy);
1272         else
1273                 a->radius = brush_size();
1274
1275         /* Set the pivot to allow the model to rotate around the center of the brush */
1276         if(get_depth(mouse[0],mouse[1]) < 1.0)
1277                 VecCopyf(sd->pivot, a->symm.center_3d);
1278
1279         /* Now project the Up, Right, and Out normals from view to model coords */
1280         unproject(zero_loc, 0, 0, 0);
1281         unproject(a->symm.up, 0, -1, 0);
1282         unproject(a->symm.right, 1, 0, 0);
1283         unproject(a->symm.out, 0, 0, -1);
1284         VecSubf(a->symm.up, a->symm.up, zero_loc);
1285         VecSubf(a->symm.right, a->symm.right, zero_loc);
1286         VecSubf(a->symm.out, a->symm.out, zero_loc);
1287         Normalize(a->symm.up);
1288         Normalize(a->symm.right);
1289         Normalize(a->symm.out);
1290
1291         /* Initialize mirror modifier clipping */
1292         for(i=0; i<3; ++i) {
1293                 a->clip[i]= 0;
1294                 a->cliptol[i]= 0;
1295         }
1296         for(md= OBACT->modifiers.first; md; md= md->next) {
1297                 if(md->type==eModifierType_Mirror && (md->mode & eModifierMode_Realtime)) {
1298                         const MirrorModifierData *mmd = (MirrorModifierData*) md;
1299
1300                         if(mmd->flag & MOD_MIR_CLIPPING) {
1301                                 a->clip[mmd->axis]= 1;
1302                                 if(mmd->tolerance > a->cliptol[mmd->axis])
1303                                         a->cliptol[mmd->axis] = mmd->tolerance;
1304                         }
1305                 }
1306         }
1307
1308         if(sd->brush_type == GRAB_BRUSH) {
1309                 float gcenter[3];
1310
1311                 /* Find the delta */
1312                 unproject(gcenter, mouse[0], mouse[1], a->depth);
1313                 unproject(oldloc, pr_mouse[0], pr_mouse[1], a->depth);
1314                 VecSubf(a->symm.grab_delta, gcenter, oldloc);
1315         }
1316         else if(sd->brush_type == LAYER_BRUSH) {
1317                 Mesh *me= get_mesh(OBACT);
1318
1319                 if(!a->layer_disps)
1320                         a->layer_disps= MEM_callocN(sizeof(float)*me->totvert,"Layer disps");
1321         }
1322
1323         if(sd->brush_type == LAYER_BRUSH || anchored) {
1324                 Mesh *me= get_mesh(OBACT);
1325                 unsigned i;
1326
1327                 if(!a->mesh_store) {
1328                         a->mesh_store= MEM_mallocN(sizeof(vec3f) * me->totvert, "Sculpt mesh store");
1329                         for(i = 0; i < me->totvert; ++i)
1330                                 VecCopyf(&a->mesh_store[i].x, me->mvert[i].co);
1331                 }
1332
1333                 if(anchored && a->layer_disps)
1334                         memset(a->layer_disps, 0, sizeof(float) * me->totvert);
1335
1336                 if(anchored && !a->orig_norms) {
1337                         a->orig_norms= MEM_mallocN(sizeof(short) * 3 * me->totvert, "Sculpt orig norm");
1338                         for(i = 0; i < me->totvert; ++i) {
1339                                 a->orig_norms[i][0] = me->mvert[i].no[0];
1340                                 a->orig_norms[i][1] = me->mvert[i].no[1];
1341                                 a->orig_norms[i][2] = me->mvert[i].no[2];
1342                         }
1343                 }
1344         }
1345 }
1346 void sculptmode_set_strength(const int delta)
1347 {
1348         int val = sculptmode_brush()->strength + delta;
1349         if(val < 1) val = 1;
1350         if(val > 100) val = 100;
1351         sculptmode_brush()->strength= val;
1352 }
1353
1354 static void sculpt_radialcontrol_callback(const int mode, const int val)
1355 {
1356         SculptSession *ss = sculpt_session();
1357         BrushData *br = sculptmode_brush();
1358
1359         if(mode == RADIALCONTROL_SIZE)
1360                 br->size = val;
1361         else if(mode == RADIALCONTROL_STRENGTH)
1362                 br->strength = val;
1363         else if(mode == RADIALCONTROL_ROTATION)
1364                 set_tex_angle(val);
1365
1366         ss->radialcontrol = NULL;
1367 }
1368
1369 /* Returns GL handle to brush texture */
1370 static GLuint sculpt_radialcontrol_calctex()
1371 {
1372         SculptData *sd= sculpt_data();
1373         SculptSession *ss= sculpt_session();
1374         int i, j;
1375         const int tsz = TC_SIZE;
1376         float *texdata= MEM_mallocN(sizeof(float)*tsz*tsz, "Brush preview");
1377         GLuint tex;
1378
1379         if(sd->texrept!=SCULPTREPT_3D)
1380                 sculptmode_update_tex();
1381         for(i=0; i<tsz; ++i)
1382                 for(j=0; j<tsz; ++j) {
1383                         float magn= sqrt(pow(i-tsz/2,2)+pow(j-tsz/2,2));
1384                         if(sd->texfade)
1385                                 texdata[i*tsz+j]= curve_strength(magn,tsz/2);
1386                         else
1387                                 texdata[i*tsz+j]= magn < tsz/2 ? 1 : 0;
1388                 }
1389         if(sd->texact != -1 && ss->texcache) {
1390                 for(i=0; i<tsz; ++i)
1391                         for(j=0; j<tsz; ++j) {
1392                                 const int col= ss->texcache[i*tsz+j];
1393                                 texdata[i*tsz+j]*= (((char*)&col)[0]+((char*)&col)[1]+((char*)&col)[2])/3.0f/255.0f;
1394                         }
1395         }
1396
1397         glGenTextures(1, &tex);
1398         glBindTexture(GL_TEXTURE_2D, tex);
1399         glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, tsz, tsz, 0, GL_ALPHA, GL_FLOAT, texdata);
1400         MEM_freeN(texdata);
1401
1402         return tex;
1403 }
1404
1405 void sculpt_radialcontrol_start(int mode)
1406 {
1407         SculptData *sd = sculpt_data();
1408         SculptSession *ss = sculpt_session();
1409         BrushData *br = sculptmode_brush();
1410         int orig=1, max=100;
1411
1412         if(mode == RADIALCONTROL_SIZE) {
1413                 orig = br->size;
1414                 max = 200;
1415         }
1416         else if(mode == RADIALCONTROL_STRENGTH) {
1417                 orig = br->strength;
1418                 max = 100;
1419         }
1420         else if(mode == RADIALCONTROL_ROTATION) {
1421                 if(sd->texact!=-1 && sd->mtex[sd->texact]) {
1422                         orig = sculpt_tex_angle();
1423                         max = 360;
1424                 }
1425                 else
1426                         mode = RADIALCONTROL_NONE;
1427         }
1428
1429         if(mode != RADIALCONTROL_NONE) {
1430                 ss->radialcontrol= radialcontrol_start(mode, sculpt_radialcontrol_callback, orig, max,
1431                                                        sculpt_radialcontrol_calctex());
1432         }
1433 }
1434
1435 void sculpt_paint_brush(char clear)
1436 {
1437         if(sculpt_data()->flags & SCULPT_DRAW_BRUSH) {
1438                 static short mvalo[2];
1439                 short mval[2];
1440                 const short rad= sculptmode_brush()->size;
1441
1442                 getmouseco_areawin(mval);
1443
1444                 persp(PERSP_WIN);
1445                 if(clear)
1446                         fdrawXORcirc(mval[0], mval[1], rad);
1447                 else
1448                         draw_sel_circle(mval, mvalo, rad, rad, 0);
1449
1450                 mvalo[0]= mval[0];
1451                 mvalo[1]= mval[1];
1452         }
1453 }
1454
1455 void sculptmode_selectbrush_menu(void)
1456 {
1457         SculptData *sd= sculpt_data();
1458         int val;
1459
1460         pupmenu_set_active(sd->brush_type);
1461
1462         val= pupmenu("Select Brush%t|Draw|Smooth|Pinch|Inflate|Grab|Layer|Flatten");
1463
1464         if(val>0) {
1465                 sd->brush_type= val;
1466
1467                 BIF_undo_push("Brush type");
1468
1469                 allqueue(REDRAWVIEW3D, 1);
1470                 allqueue(REDRAWBUTSEDIT, 1);
1471         }
1472 }
1473
1474 void sculptmode_update_all_projverts(float *vertcosnos)
1475 {
1476         SculptSession *ss = sculpt_session();
1477         Mesh *me= get_mesh(OBACT);
1478         unsigned i;
1479
1480         if(!ss->projverts)
1481                 ss->projverts = MEM_mallocN(sizeof(ProjVert)*me->totvert,"ProjVerts");
1482
1483         for(i=0; i<me->totvert; ++i) {
1484                 project(vertcosnos ? &vertcosnos[i * 6] : me->mvert[i].co, ss->projverts[i].co);
1485                 ss->projverts[i].inside= 0;
1486         }
1487 }
1488
1489 void sculptmode_draw_wires(SculptSession *ss, int only_damaged, Mesh *me)
1490 {
1491         int i;
1492
1493         bglPolygonOffset(1.0);
1494         glDepthMask(0);
1495         BIF_ThemeColor((OBACT==OBACT)?TH_ACTIVE:TH_SELECT);
1496
1497         for(i=0; i<me->totedge; i++) {
1498                 MEdge *med= &me->medge[i];
1499
1500                 if((!only_damaged || (ss->projverts[med->v1].inside || ss->projverts[med->v2].inside)) &&
1501                    (med->flag & ME_EDGEDRAW)) {
1502                         glDrawElements(GL_LINES, 2, GL_UNSIGNED_INT, &med->v1);
1503                 }
1504         }
1505
1506         glDepthMask(1);
1507         bglPolygonOffset(0.0);
1508 }
1509
1510 void sculptmode_draw_mesh(int only_damaged)
1511 {
1512         Mesh *me= get_mesh(OBACT);
1513         int i, j, dt, drawCurrentMat = 1, matnr= -1;
1514         SculptSession *ss = sculpt_session();
1515
1516         persp(PERSP_VIEW);
1517         mymultmatrix(OBACT->obmat);
1518         glEnable(GL_DEPTH_TEST);
1519         glEnable(GL_LIGHTING);
1520         GPU_set_object_materials(G.scene, OBACT, 0, NULL);
1521         glEnable(GL_CULL_FACE);
1522
1523         glShadeModel(GL_SMOOTH);
1524
1525         glVertexPointer(3, GL_FLOAT, sizeof(MVert), &me->mvert[0].co);
1526         glNormalPointer(GL_SHORT, sizeof(MVert), &me->mvert[0].no);
1527
1528         dt= MIN2(G.vd->drawtype, OBACT->dt);
1529         if(dt==OB_WIRE)
1530                 glColorMask(0,0,0,0);
1531
1532         for(i=0; i<me->totface; ++i) {
1533                 MFace *f= &me->mface[i];
1534                 char inside= 0;
1535                 int new_matnr= f->mat_nr + 1;
1536
1537                 if(new_matnr != matnr)
1538                         drawCurrentMat= GPU_enable_material(matnr = new_matnr, NULL);
1539
1540                 /* If only_damaged!=0, only draw faces that are partially
1541                    inside the area(s) modified by the brush */
1542                 if(only_damaged) {
1543                         for(j=0; j<(f->v4?4:3); ++j) {
1544                                 if(ss->projverts[*((&f->v1)+j)].inside) {
1545                                         inside= 1;
1546                                         break;
1547                                 }
1548                         }
1549                 }
1550                 else
1551                         inside= 1;
1552
1553                 if(inside && drawCurrentMat)
1554                         glDrawElements(f->v4?GL_QUADS:GL_TRIANGLES, f->v4?4:3, GL_UNSIGNED_INT, &f->v1);
1555         }
1556
1557         glDisable(GL_CULL_FACE);
1558         glDisable(GL_LIGHTING);
1559         glColorMask(1,1,1,1);
1560
1561         if(dt==OB_WIRE || (OBACT->dtx & OB_DRAWWIRE))
1562                 sculptmode_draw_wires(ss, only_damaged, me);
1563
1564         glDisable(GL_DEPTH_TEST);
1565 }
1566
1567 void sculptmode_correct_state(void)
1568 {
1569         if(!sculpt_session())
1570                 sculpt_init_session();
1571
1572         glEnableClientState(GL_VERTEX_ARRAY);
1573         glEnableClientState(GL_NORMAL_ARRAY);
1574
1575         if(!sculpt_session()->vertex_users) calc_vertex_users();
1576 }
1577
1578 /* Checks whether full update mode (slower) needs to be used to work with modifiers */
1579 char sculpt_modifiers_active(Object *ob)
1580 {
1581         ModifierData *md;
1582
1583         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1584                 if(md->mode & eModifierMode_Realtime)
1585                         return 1;
1586         }
1587
1588         return 0;
1589 }
1590
1591 void sculpt(void)
1592 {
1593         SculptData *sd= sculpt_data();
1594         SculptSession *ss= sculpt_session();
1595         Object *ob= OBACT;
1596         /* lastSigMouse is for the rake, to store the last place the mouse movement was significant */
1597         short mouse[2], mvalo[2], lastSigMouse[2],firsttime=1, mousebut;
1598         short modifier_calculations= 0;
1599         BrushAction *a = MEM_callocN(sizeof(BrushAction), "brush action");
1600         short spacing= 32000;
1601         int scissor_box[4];
1602         float offsetRot;
1603         int smooth_stroke = 0, i;
1604         int anchored;
1605
1606         if(!(G.f & G_SCULPTMODE) || G.obedit || !ob || ob->id.lib || !get_mesh(ob) || (get_mesh(ob)->totface == 0))
1607                 return;
1608         if(!(ob->lay & G.vd->lay))
1609                 error("Active object is not in this layer");
1610         if(ob_get_keyblock(ob)) {
1611                 if(!(ob->shapeflag & OB_SHAPE_LOCK)) {
1612                         error("Cannot sculpt on unlocked shape key");
1613                         return;
1614                 }
1615         }
1616
1617         if(!ss) {
1618                 sculpt_init_session();
1619                 ss= sd->session;
1620         }
1621
1622         anchored = sculptmode_brush()->flag & SCULPT_BRUSH_ANCHORED;
1623         smooth_stroke = (sd->flags & SCULPT_INPUT_SMOOTH) && (sd->brush_type != GRAB_BRUSH) && !anchored;
1624
1625         if(smooth_stroke)
1626                 sculpt_stroke_new(256);
1627
1628         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
1629         ss->damaged_verts.first = ss->damaged_verts.last = NULL;
1630         ss->vertexcosnos = NULL;
1631
1632         /* Check that vertex users are up-to-date */
1633         if(ob != active_ob || !ss->vertex_users || ss->vertex_users_size != get_mesh(ob)->totvert) {
1634                 sculpt_vertexusers_free(ss);
1635                 calc_vertex_users();
1636                 if(ss->projverts)
1637                         MEM_freeN(ss->projverts);
1638                 ss->projverts = NULL;
1639                 active_ob= ob;
1640         }
1641
1642         glEnableClientState(GL_VERTEX_ARRAY);
1643         glEnableClientState(GL_NORMAL_ARRAY);
1644
1645         persp(PERSP_VIEW);
1646
1647         getmouseco_areawin(mvalo);
1648
1649         /* Init texture
1650            FIXME: Shouldn't be doing this every time! */
1651         if(sd->texrept!=SCULPTREPT_3D)
1652                 sculptmode_update_tex();
1653
1654         getmouseco_areawin(mouse);
1655         mvalo[0]= mouse[0];
1656         mvalo[1]= mouse[1];
1657         lastSigMouse[0]=mouse[0];
1658         lastSigMouse[1]=mouse[1];
1659         mousebut = L_MOUSE;
1660
1661         /* If modifier_calculations is true, then extra time must be spent
1662            updating the mesh. This takes a *lot* longer, so it's worth
1663            skipping if the modifier stack is empty. */
1664         modifier_calculations= sculpt_modifiers_active(ob);
1665
1666         init_sculptmatrices();
1667
1668         if(modifier_calculations)
1669                 ss->vertexcosnos= mesh_get_mapped_verts_nors(ob);
1670         sculptmode_update_all_projverts(ss->vertexcosnos);
1671
1672         /* Set scaling adjustment */
1673         a->scale[0]= 1.0f / ob->size[0];
1674         a->scale[1]= 1.0f / ob->size[1];
1675         a->scale[2]= 1.0f / ob->size[2];
1676
1677         /* Capture original copy */
1678         if(sd->flags & SCULPT_DRAW_FAST)
1679                 glAccum(GL_LOAD, 1);
1680
1681         /* Get original scissor box */
1682         glGetIntegerv(GL_SCISSOR_BOX, scissor_box);
1683
1684         /* For raking, get the original angle*/
1685         offsetRot=sculpt_tex_angle();
1686
1687         while (get_mbut() & mousebut) {
1688                 getmouseco_areawin(mouse);
1689                 /* If rake, and the mouse has moved over 10 pixels (euclidean) (prevents jitter) then get the new angle */
1690                 if (sd->rake && (pow(lastSigMouse[0]-mouse[0],2)+pow(lastSigMouse[1]-mouse[1],2))>100){
1691                         /*Nasty looking, but just orig + new angle really*/
1692                         set_tex_angle(offsetRot+180.+to_deg(atan2((float)(mouse[1]-lastSigMouse[1]),(float)(mouse[0]-lastSigMouse[0]))));
1693                         lastSigMouse[0]=mouse[0];
1694                         lastSigMouse[1]=mouse[1];
1695                 }
1696
1697                 if(firsttime || mouse[0]!=mvalo[0] || mouse[1]!=mvalo[1] ||
1698                    sculptmode_brush()->flag & SCULPT_BRUSH_AIRBRUSH) {
1699                         a->firsttime = firsttime;
1700                         firsttime= 0;
1701
1702                         if(smooth_stroke)
1703                                 sculpt_stroke_add_point(mouse[0], mouse[1]);
1704
1705                         spacing+= sqrt(pow(mvalo[0]-mouse[0],2)+pow(mvalo[1]-mouse[1],2));
1706
1707                         if(modifier_calculations && !ss->vertexcosnos)
1708                                 ss->vertexcosnos= mesh_get_mapped_verts_nors(ob);
1709
1710                         if(G.scene->sculptdata.brush_type != GRAB_BRUSH) {
1711                                 if(anchored) {
1712                                         Mesh *me = get_mesh(ob);
1713
1714                                         /* Restore the mesh before continuing with anchored stroke */
1715                                         if(a->mesh_store) {
1716                                                 for(i = 0; i < me->totvert; ++i) {
1717                                                         VecCopyf(me->mvert[i].co, &a->mesh_store[i].x);
1718                                                         me->mvert[i].no[0] = a->orig_norms[i][0];
1719                                                         me->mvert[i].no[1] = a->orig_norms[i][1];
1720                                                         me->mvert[i].no[2] = a->orig_norms[i][2];
1721                                                 }
1722                                         }
1723
1724                                         do_symmetrical_brush_actions(a, mouse, NULL);
1725                                 }
1726                                 else {
1727                                         if(smooth_stroke) {
1728                                                 sculpt_stroke_apply(a);
1729                                         }
1730                                         else if(sd->spacing==0 || spacing>sd->spacing) {
1731                                                 do_symmetrical_brush_actions(a, mouse, NULL);
1732                                                 spacing= 0;
1733                                         }
1734                                 }
1735                         }
1736                         else {
1737                                 do_symmetrical_brush_actions(a, mouse, mvalo);
1738                                 unproject(sd->pivot, mouse[0], mouse[1], a->depth);
1739                         }
1740
1741                         if(modifier_calculations || ob_get_keyblock(ob))
1742                                 DAG_object_flush_update(G.scene, ob, OB_RECALC_DATA);
1743
1744                         if(modifier_calculations || sd->brush_type == GRAB_BRUSH || !(sd->flags & SCULPT_DRAW_FAST)) {
1745                                 calc_damaged_verts(&ss->damaged_verts, a);
1746                                 scrarea_do_windraw(curarea);
1747                                 screen_swapbuffers();
1748                         } else { /* Optimized drawing */
1749                                 calc_damaged_verts(&ss->damaged_verts, a);
1750
1751                                 /* Draw the stored image to the screen */
1752                                 glAccum(GL_RETURN, 1);
1753
1754                                 sculpt_clear_damaged_areas(ss);
1755
1756                                 /* Draw all the polygons that are inside the modified area(s) */
1757                                 glScissor(scissor_box[0], scissor_box[1], scissor_box[2], scissor_box[3]);
1758                                 sculptmode_draw_mesh(1);
1759                                 glAccum(GL_LOAD, 1);
1760
1761                                 projverts_clear_inside(ss);
1762
1763                                 persp(PERSP_WIN);
1764                                 glDisable(GL_DEPTH_TEST);
1765
1766                                 /* Draw cursor */
1767                                 if(sculpt_data()->flags & SCULPT_DRAW_BRUSH)
1768                                         fdrawXORcirc((float)mouse[0],(float)mouse[1],sculptmode_brush()->size);
1769                                 if(smooth_stroke)
1770                                         sculpt_stroke_draw();
1771
1772                                 myswapbuffers();
1773                         }
1774
1775                         BLI_freelistN(&ss->damaged_rects);
1776                         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
1777
1778                         mvalo[0]= mouse[0];
1779                         mvalo[1]= mouse[1];
1780
1781                         if(ss->vertexcosnos) {
1782                                 MEM_freeN(ss->vertexcosnos);
1783                                 ss->vertexcosnos= NULL;
1784                         }
1785
1786                 }
1787                 else BIF_wait_for_statechange();
1788         }
1789
1790         /* Set the rotation of the brush back to what it was before any rake */
1791         set_tex_angle(offsetRot);
1792
1793         if(smooth_stroke) {
1794                 sculpt_stroke_apply_all(a);
1795                 calc_damaged_verts(&ss->damaged_verts, a);
1796                 BLI_freelistN(&ss->damaged_rects);
1797         }
1798
1799         if(a->layer_disps) MEM_freeN(a->layer_disps);
1800         if(a->mesh_store) MEM_freeN(a->mesh_store);
1801         if(a->orig_norms) MEM_freeN(a->orig_norms);
1802         for(i=0; i<8; ++i)
1803                 BLI_freelistN(&a->grab_active_verts[i]);
1804         MEM_freeN(a);
1805         sculpt_stroke_free();
1806
1807         sculpt_undo_push(G.scene->sculptdata.brush_type);
1808
1809         if(G.vd->depths) G.vd->depths->damaged= 1;
1810
1811         allqueue(REDRAWVIEW3D, 0);
1812 }
1813
1814 static void sculpt_undo_push(const short brush_type)
1815 {
1816         switch(brush_type) {
1817         case DRAW_BRUSH:
1818                 BIF_undo_push("Draw Brush"); break;
1819         case SMOOTH_BRUSH:
1820                 BIF_undo_push("Smooth Brush"); break;
1821         case PINCH_BRUSH:
1822                 BIF_undo_push("Pinch Brush"); break;
1823         case INFLATE_BRUSH:
1824                 BIF_undo_push("Inflate Brush"); break;
1825         case GRAB_BRUSH:
1826                 BIF_undo_push("Grab Brush"); break;
1827         case LAYER_BRUSH:
1828                 BIF_undo_push("Layer Brush"); break;
1829         case FLATTEN_BRUSH:
1830                 BIF_undo_push("Flatten Brush"); break;
1831         default:
1832                 BIF_undo_push("Sculpting"); break;
1833         }
1834 }
1835
1836 void set_sculptmode(void)
1837 {
1838         if(G.f & G_SCULPTMODE) {
1839                 Mesh *me= get_mesh(OBACT);
1840
1841                 G.f &= ~G_SCULPTMODE;
1842
1843                 sculptsession_free(G.scene);
1844                 if(me && me->pv)
1845                         mesh_pmv_off(OBACT, me);
1846         }
1847         else {
1848                 G.f |= G_SCULPTMODE;
1849
1850                 /* Called here to sanity-check the brush */
1851                 sculptmode_brush();
1852
1853                 sculpt_init_session();
1854
1855                 glEnableClientState(GL_VERTEX_ARRAY);
1856                 glEnableClientState(GL_NORMAL_ARRAY);
1857         }
1858
1859         active_ob= NULL;
1860
1861         allqueue(REDRAWVIEW3D, 1);
1862         allqueue(REDRAWBUTSEDIT, 0);
1863 }
1864
1865 /* Partial Mesh Visibility */
1866
1867 /* mode: 0=hide outside selection, 1=hide inside selection */
1868 static void sculptmode_do_pmv(Object *ob, rcti *hb_2d, int mode)
1869 {
1870         Mesh *me= get_mesh(ob);
1871         float hidebox[6][3];
1872         vec3f plane_normals[4];
1873         float plane_ds[4];
1874         unsigned i, j;
1875         unsigned ndx_show, ndx_hide;
1876         MVert *nve;
1877         unsigned face_cnt_show= 0, face_ndx_show= 0;
1878         unsigned edge_cnt_show= 0, edge_ndx_show= 0;
1879         unsigned *old_map= NULL;
1880         const unsigned SHOW= 0, HIDE=1;
1881
1882         /* Convert hide box from 2D to 3D */
1883         unproject(hidebox[0], hb_2d->xmin, hb_2d->ymax, 1);
1884         unproject(hidebox[1], hb_2d->xmax, hb_2d->ymax, 1);
1885         unproject(hidebox[2], hb_2d->xmax, hb_2d->ymin, 1);
1886         unproject(hidebox[3], hb_2d->xmin, hb_2d->ymin, 1);
1887         unproject(hidebox[4], hb_2d->xmin, hb_2d->ymax, 0);
1888         unproject(hidebox[5], hb_2d->xmax, hb_2d->ymin, 0);
1889
1890         /* Calculate normals for each side of hide box */
1891         CalcNormFloat(hidebox[0], hidebox[1], hidebox[4], &plane_normals[0].x);
1892         CalcNormFloat(hidebox[1], hidebox[2], hidebox[5], &plane_normals[1].x);
1893         CalcNormFloat(hidebox[2], hidebox[3], hidebox[5], &plane_normals[2].x);
1894         CalcNormFloat(hidebox[3], hidebox[0], hidebox[4], &plane_normals[3].x);
1895
1896         /* Calculate D for each side of hide box */
1897         for(i= 0; i<4; ++i)
1898                 plane_ds[i]= hidebox[i][0]*plane_normals[i].x + hidebox[i][1]*plane_normals[i].y +
1899                         hidebox[i][2]*plane_normals[i].z;
1900
1901         /* Add partial visibility to mesh */
1902         if(!me->pv) {
1903                 me->pv= MEM_callocN(sizeof(PartialVisibility),"PartialVisibility");
1904         } else {
1905                 old_map= MEM_callocN(sizeof(unsigned)*me->pv->totvert,"PMV oldmap");
1906                 for(i=0; i<me->pv->totvert; ++i) {
1907                         old_map[i]= me->pv->vert_map[i]<me->totvert?0:1;
1908                 }
1909                 mesh_pmv_revert(ob, me);
1910         }
1911
1912         /* Kill sculpt data */
1913         active_ob= NULL;
1914
1915         /* Initalize map with which verts are to be hidden */
1916         me->pv->vert_map= MEM_mallocN(sizeof(unsigned)*me->totvert, "PMV vertmap");
1917         me->pv->totvert= me->totvert;
1918         me->totvert= 0;
1919         for(i=0; i<me->pv->totvert; ++i) {
1920                 me->pv->vert_map[i]= mode ? HIDE:SHOW;
1921                 for(j=0; j<4; ++j) {
1922                         if(me->mvert[i].co[0] * plane_normals[j].x +
1923                            me->mvert[i].co[1] * plane_normals[j].y +
1924                            me->mvert[i].co[2] * plane_normals[j].z < plane_ds[j] ) {
1925                                 me->pv->vert_map[i]= mode ? SHOW:HIDE; /* Vert is outside the hide box */
1926                                 break;
1927                         }
1928                 }
1929                 if(old_map && old_map[i]) me->pv->vert_map[i]= 1;
1930                 if(!me->pv->vert_map[i]) ++me->totvert;
1931
1932         }
1933         if(old_map) MEM_freeN(old_map);
1934
1935         /* Find out how many faces to show */
1936         for(i=0; i<me->totface; ++i) {
1937                 if(!me->pv->vert_map[me->mface[i].v1] &&
1938                    !me->pv->vert_map[me->mface[i].v2] &&
1939                    !me->pv->vert_map[me->mface[i].v3]) {
1940                         if(me->mface[i].v4) {
1941                                 if(!me->pv->vert_map[me->mface[i].v4])
1942                                         ++face_cnt_show;
1943                         }
1944                         else ++face_cnt_show;
1945                 }
1946         }
1947         /* Find out how many edges to show */
1948         for(i=0; i<me->totedge; ++i) {
1949                 if(!me->pv->vert_map[me->medge[i].v1] &&
1950                    !me->pv->vert_map[me->medge[i].v2])
1951                         ++edge_cnt_show;
1952         }
1953
1954         /* Create new vert array and reset each vert's map with map[old]=new index */
1955         nve= MEM_mallocN(sizeof(MVert)*me->pv->totvert, "PMV verts");
1956         ndx_show= 0; ndx_hide= me->totvert;
1957         for(i=0; i<me->pv->totvert; ++i) {
1958                 if(me->pv->vert_map[i]) {
1959                         me->pv->vert_map[i]= ndx_hide;
1960                         nve[me->pv->vert_map[i]]= me->mvert[i];
1961                         ++ndx_hide;
1962                 } else {
1963                         me->pv->vert_map[i]= ndx_show;
1964                         nve[me->pv->vert_map[i]]= me->mvert[i];
1965                         ++ndx_show;
1966                 }
1967         }
1968         CustomData_free_layer_active(&me->vdata, CD_MVERT, me->pv->totvert);
1969         me->mvert= CustomData_add_layer(&me->vdata, CD_MVERT, CD_ASSIGN, nve, me->totvert);
1970
1971         /* Create new face array */
1972         me->pv->old_faces= me->mface;
1973         me->pv->totface= me->totface;
1974         me->mface= MEM_mallocN(sizeof(MFace)*face_cnt_show, "PMV faces");
1975         for(i=0; i<me->totface; ++i) {
1976                 MFace *pr_f= &me->pv->old_faces[i];
1977                 char show= 0;
1978
1979                 if(me->pv->vert_map[pr_f->v1] < me->totvert &&
1980                    me->pv->vert_map[pr_f->v2] < me->totvert &&
1981                    me->pv->vert_map[pr_f->v3] < me->totvert) {
1982                         if(pr_f->v4) {
1983                                 if(me->pv->vert_map[pr_f->v4] < me->totvert)
1984                                         show= 1;
1985                         }
1986                         else show= 1;
1987                 }
1988
1989                 if(show) {
1990                         MFace *cr_f= &me->mface[face_ndx_show];
1991                         *cr_f= *pr_f;
1992                         cr_f->v1= me->pv->vert_map[pr_f->v1];
1993                         cr_f->v2= me->pv->vert_map[pr_f->v2];
1994                         cr_f->v3= me->pv->vert_map[pr_f->v3];
1995                         cr_f->v4= pr_f->v4 ? me->pv->vert_map[pr_f->v4] : 0;
1996                         test_index_face(cr_f,NULL,0,pr_f->v4?4:3);
1997                         ++face_ndx_show;
1998                 }
1999         }
2000         me->totface= face_cnt_show;
2001         CustomData_set_layer(&me->fdata, CD_MFACE, me->mface);
2002
2003         /* Create new edge array */
2004         me->pv->old_edges= me->medge;
2005         me->pv->totedge= me->totedge;
2006         me->medge= MEM_mallocN(sizeof(MEdge)*edge_cnt_show, "PMV edges");
2007         me->pv->edge_map= MEM_mallocN(sizeof(int)*me->pv->totedge,"PMV edgemap");
2008         for(i=0; i<me->totedge; ++i) {
2009                 if(me->pv->vert_map[me->pv->old_edges[i].v1] < me->totvert &&
2010                    me->pv->vert_map[me->pv->old_edges[i].v2] < me->totvert) {
2011                         MEdge *cr_e= &me->medge[edge_ndx_show];
2012                         me->pv->edge_map[i]= edge_ndx_show;
2013                         *cr_e= me->pv->old_edges[i];
2014                         cr_e->v1= me->pv->vert_map[me->pv->old_edges[i].v1];
2015                         cr_e->v2= me->pv->vert_map[me->pv->old_edges[i].v2];
2016                         ++edge_ndx_show;
2017                 }
2018                 else me->pv->edge_map[i]= -1;
2019         }
2020         me->totedge= edge_cnt_show;
2021         CustomData_set_layer(&me->edata, CD_MEDGE, me->medge);
2022
2023         DAG_object_flush_update(G.scene, OBACT, OB_RECALC_DATA);
2024 }
2025
2026 static rcti sculptmode_pmv_box()
2027 {
2028         short down[2], mouse[2];
2029         rcti ret;
2030
2031         getmouseco_areawin(down);
2032
2033         while((get_mbut()&L_MOUSE) || (get_mbut()&R_MOUSE)) {
2034                 getmouseco_areawin(mouse);
2035
2036                 scrarea_do_windraw(curarea);
2037
2038                 persp(PERSP_WIN);
2039                 glLineWidth(2);
2040                 setlinestyle(2);
2041                 sdrawXORline(down[0],down[1],mouse[0],down[1]);
2042                 sdrawXORline(mouse[0],down[1],mouse[0],mouse[1]);
2043                 sdrawXORline(mouse[0],mouse[1],down[0],mouse[1]);
2044                 sdrawXORline(down[0],mouse[1],down[0],down[1]);
2045                 setlinestyle(0);
2046                 glLineWidth(1);
2047                 persp(PERSP_VIEW);
2048
2049                 screen_swapbuffers();
2050                 backdrawview3d(0);
2051         }
2052
2053         ret.xmin= down[0]<mouse[0]?down[0]:mouse[0];
2054         ret.ymin= down[1]<mouse[1]?down[1]:mouse[1];
2055         ret.xmax= down[0]>mouse[0]?down[0]:mouse[0];
2056         ret.ymax= down[1]>mouse[1]?down[1]:mouse[1];
2057         return ret;
2058 }
2059
2060 void sculptmode_pmv(int mode)
2061 {
2062         Object *ob= OBACT;
2063         rcti hb_2d;
2064
2065         if(ob_get_key(ob)) {
2066                 error("Cannot hide mesh with shape keys enabled");
2067                 return;
2068         }
2069
2070         hb_2d= sculptmode_pmv_box(); /* Get 2D hide box */
2071
2072         sculptmode_correct_state();
2073
2074         waitcursor(1);
2075
2076         if(hb_2d.xmax-hb_2d.xmin > 3 && hb_2d.ymax-hb_2d.ymin > 3) {
2077                 init_sculptmatrices();
2078
2079                 sculptmode_do_pmv(ob,&hb_2d,mode);
2080         }
2081         else mesh_pmv_off(ob, get_mesh(ob));
2082
2083         scrarea_do_windraw(curarea);
2084
2085         BIF_undo_push("Partial mesh hide");
2086
2087         waitcursor(0);
2088 }