source/blender/src/sculptmode.c

   1 /*
   2  * $Id$
   3  *
   4  * ***** BEGIN GPL LICENSE BLOCK *****
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version 2
   9  * of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software  Foundation,
  18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19  *
  20  * The Original Code is Copyright (C) 2006 by Nicholas Bishop
  21  * All rights reserved.
  22  *
  23  * The Original Code is: all of this file.
  24  *
  25  * Contributor(s): none yet.
  26  *
  27  * ***** END GPL LICENSE BLOCK *****
  28  *
  29  * Implements the Sculpt Mode tools
  30  *
  31  * BDR_sculptmode.h
  32  *
  33  */
  34
  35 #include "GHOST_Types.h"
  36
  37 #include "MEM_guardedalloc.h"
  38
  39 #include "BLI_arithb.h"
  40 #include "BLI_blenlib.h"
  41 #include "BLI_dynstr.h"
  42
  43 #include "DNA_armature_types.h"
  44 #include "DNA_brush_types.h"
  45 #include "DNA_image_types.h"
  46 #include "DNA_key_types.h"
  47 #include "DNA_mesh_types.h"
  48 #include "DNA_meshdata_types.h"
  49 #include "DNA_modifier_types.h"
  50 #include "DNA_object_types.h"
  51 #include "DNA_screen_types.h"
  52 #include "DNA_scene_types.h"
  53 #include "DNA_texture_types.h"
  54 #include "DNA_view3d_types.h"
  55 #include "DNA_userdef_types.h"
  56
  57 #include "BKE_customdata.h"
  58 #include "BKE_DerivedMesh.h"
  59 #include "BKE_depsgraph.h"
  60 #include "BKE_global.h"
  61 #include "BKE_image.h"
  62 #include "BKE_key.h"
  63 #include "BKE_library.h"
  64 #include "BKE_main.h"
  65 #include "BKE_mesh.h"
  66 #include "BKE_modifier.h"
  67 #include "BKE_texture.h"
  68 #include "BKE_utildefines.h"
  69
  70 #include "BIF_editkey.h"
  71 #include "BIF_editview.h"
  72 #include "BIF_glutil.h"
  73 #include "BIF_gl.h"
  74 #include "BIF_interface.h"
  75 #include "BIF_mywindow.h"
  76 #include "BIF_resources.h"
  77 #include "BIF_screen.h"
  78 #include "BIF_space.h"
  79 #include "BIF_toolbox.h"
  80
  81 #include "BDR_drawobject.h"
  82 #include "BDR_sculptmode.h"
  83
  84 #include "BSE_drawview.h"
  85 #include "BSE_edit.h"
  86 #include "BSE_view.h"
  87
  88 #include "IMB_imbuf_types.h"
  89
  90 #include "blendef.h"
  91 #include "multires.h"
  92 #include "mydevice.h"
  93
  94 #include "RE_render_ext.h"
  95 #include "RE_shader_ext.h" /*for multitex_ext*/
  96
  97 #include <math.h>
  98 #include <stdlib.h>
  99 #include <string.h>
 100
 101 /* Number of vertices to average in order to determine the flatten distance */
 102 #define FLATTEN_SAMPLE_SIZE 10
 103
 104 /* ===== STRUCTS =====
 105  *
 106  */
 107
 108 /* Used by vertex_users to store face indices in a list */
 109 typedef struct IndexNode {
 110         struct IndexNode* next,* prev;
 111         int Index;
 112 } IndexNode;
 113
 114
 115 /* ActiveData stores an Index into the mvert array of Mesh, plus Fade, which
 116    stores how far the vertex is from the brush center, scaled to the range [0,1]. */
 117 typedef struct ActiveData {
 118         struct ActiveData *next, *prev;
 119         unsigned int Index;
 120         float Fade;
 121         float dist;
 122 } ActiveData;
 123
 124 typedef struct GrabData {
 125         char firsttime;
 126         ListBase active_verts[8];
 127         unsigned char index;
 128         vec3f delta, delta_symm;
 129         float depth;
 130 } GrabData;
 131
 132 typedef struct EditData {
 133         vec3f center;
 134         float size;
 135         char flip;
 136         short mouse[2];
 137
 138         /* Adjust brush strength along each axis
 139            to adjust for object scaling */
 140         float scale[3];
 141
 142         /* View normals */
 143         vec3f up, right, out;
 144
 145         GrabData *grabdata;
 146         float *layer_disps;
 147         vec3f *layer_store;
 148
 149         char clip[3];
 150         float cliptol[3];
 151
 152         char symm;
 153 } EditData;
 154
 155 typedef struct RectNode {
 156         struct RectNode *next, *prev;
 157         rcti r;
 158 } RectNode;
 159
 160 /* Used to store to 2D screen coordinates of each vertex in the mesh. */
 161 typedef struct ProjVert {
 162         short co[2];
 163
 164         /* Used to mark whether a vertex is inside a rough bounding box
 165            containing the brush. */
 166         char inside;
 167 } ProjVert;
 168
 169 static ProjVert *projverts= NULL;
 170 static Object *active_ob= NULL;
 171
 172 SculptData *sculpt_data(void)
 173 {
 174         return &G.scene->sculptdata;
 175 }
 176
 177 void sculpt_init_session(void);
 178 void init_editdata(EditData *e, short *, short *);
 179
 180 SculptSession *sculpt_session(void)
 181 {
 182         if(!sculpt_data()->session)
 183                 sculpt_init_session();
 184         return sculpt_data()->session;
 185 }
 186
 187 /* ===== MEMORY =====
 188  *
 189  * Allocate/initialize/free data
 190  */
 191
 192 /* Initialize 'permanent' sculpt data that is saved with file kept after
 193    switching out of sculptmode. */
 194 void sculptmode_init(Scene *sce)
 195 {
 196         SculptData *sd;
 197
 198         if(!sce) {
 199                 error("Unable to initialize sculptmode: bad scene");
 200                 return;
 201         }
 202
 203         sd= &sce->sculptdata;
 204
 205         memset(sd, 0, sizeof(SculptData));
 206
 207         sd->drawbrush.size = sd->smoothbrush.size = sd->pinchbrush.size =
 208                 sd->inflatebrush.size = sd->grabbrush.size =
 209                 sd->layerbrush.size = sd->flattenbrush.size = 50;
 210         sd->drawbrush.strength = sd->smoothbrush.strength =
 211                 sd->pinchbrush.strength = sd->inflatebrush.strength =
 212                 sd->grabbrush.strength = sd->layerbrush.strength =
 213                 sd->flattenbrush.strength = 25;
 214         sd->drawbrush.dir = sd->pinchbrush.dir = sd->inflatebrush.dir = sd->layerbrush.dir= 1;
 215         sd->drawbrush.airbrush = sd->smoothbrush.airbrush =
 216                 sd->pinchbrush.airbrush = sd->inflatebrush.airbrush =
 217                 sd->layerbrush.airbrush = sd->flattenbrush.airbrush = 0;
 218         sd->drawbrush.view= 0;
 219         sd->brush_type= DRAW_BRUSH;
 220         sd->texact= -1;
 221         sd->texfade= 1;
 222         sd->averaging= 1;
 223         sd->texsep= 0;
 224         sd->texrept= SCULPTREPT_DRAG;
 225         sd->flags= SCULPT_DRAW_BRUSH;
 226         sd->tablet_size=3;
 227         sd->tablet_strength=10;
 228 }
 229
 230 void sculptmode_free_session(Scene *);
 231 void sculpt_init_session(void)
 232 {
 233         if(sculpt_data()->session)
 234                 sculptmode_free_session(G.scene);
 235         sculpt_data()->session= MEM_callocN(sizeof(SculptSession), "SculptSession");
 236 }
 237
 238 void sculptmode_free_vertexusers(SculptSession *ss)
 239 {
 240         if(ss && ss->vertex_users){
 241                 MEM_freeN(ss->vertex_users);
 242                 MEM_freeN(ss->vertex_users_mem);
 243                 ss->vertex_users= NULL;
 244                 ss->vertex_users_mem= NULL;
 245                 ss->vertex_users_size= 0;
 246         }
 247 }
 248
 249 void sculptmode_propset_end(SculptSession *ss, int);
 250 void sculptmode_free_session(Scene *sce)
 251 {
 252         SculptSession *ss= sce->sculptdata.session;
 253         if(ss) {
 254                 sculptmode_free_vertexusers(ss);
 255                 if(ss->texcache)
 256                         MEM_freeN(ss->texcache);
 257                 sculptmode_propset_end(ss, 1);
 258                 MEM_freeN(ss);
 259                 sce->sculptdata.session= NULL;
 260         }
 261 }
 262
 263 void sculptmode_free_all(Scene *sce)
 264 {
 265         SculptData *sd= &sce->sculptdata;
 266         int a;
 267
 268         sculptmode_free_session(sce);
 269
 270         for(a=0; a<MAX_MTEX; a++) {
 271                 MTex *mtex= sd->mtex[a];
 272                 if(mtex) {
 273                         if(mtex->tex) mtex->tex->id.us--;
 274                         MEM_freeN(mtex);
 275                 }
 276         }
 277 }
 278
 279 /* vertex_users is an array of Lists that store all the faces that use a
 280    particular vertex. vertex_users is in the same order as mesh.mvert */
 281 void calc_vertex_users()
 282 {
 283         SculptSession *ss= sculpt_session();
 284         int i,j;
 285         IndexNode *node= NULL;
 286         Mesh *me= get_mesh(OBACT);
 287
 288         sculptmode_free_vertexusers(ss);
 289
 290         /* For efficiency, use vertex_users_mem as a memory pool (may be larger
 291            than necessary if mesh has triangles, but only one alloc is needed.) */
 292         ss->vertex_users= MEM_callocN(sizeof(ListBase) * me->totvert, "vertex_users");
 293         ss->vertex_users_size= me->totvert;
 294         ss->vertex_users_mem= MEM_callocN(sizeof(IndexNode)*me->totface*4, "vertex_users_mem");
 295         node= ss->vertex_users_mem;
 296
 297         /* Find the users */
 298         for(i=0; i<me->totface; ++i){
 299                 for(j=0; j<(me->mface[i].v4?4:3); ++j, ++node) {
 300                         node->Index=i;
 301                         BLI_addtail(&ss->vertex_users[((unsigned int*)(&me->mface[i]))[j]], node);
 302                 }
 303         }
 304 }
 305
 306 /* ===== INTERFACE =====
 307  */
 308
 309 void sculptmode_rem_tex(void *junk0,void *junk1)
 310 {
 311         MTex *mtex= G.scene->sculptdata.mtex[G.scene->sculptdata.texact];
 312         if(mtex) {
 313                 SculptSession *ss= sculpt_session();
 314                 if(mtex->tex) mtex->tex->id.us--;
 315                 MEM_freeN(mtex);
 316                 G.scene->sculptdata.mtex[G.scene->sculptdata.texact]= NULL;
 317                 /* Clear brush preview */
 318                 if(ss->texcache) {
 319                         MEM_freeN(ss->texcache);
 320                         ss->texcache= NULL;
 321                 }
 322                 BIF_undo_push("Unlink brush texture");
 323                 allqueue(REDRAWBUTSEDIT, 0);
 324                 allqueue(REDRAWOOPS, 0);
 325         }
 326 }
 327
 328 /* ===== OPENGL =====
 329  *
 330  * Simple functions to get data from the GL
 331  */
 332
 333 /* Store the modelview and projection matrices and viewport. */
 334 void init_sculptmatrices()
 335 {
 336         SculptSession *ss= sculpt_session();
 337
 338         glMatrixMode(GL_MODELVIEW);
 339         glPushMatrix();
 340         glMultMatrixf(OBACT->obmat);
 341
 342         bgl_get_mats(&ss->mats);
 343
 344         glPopMatrix();
 345
 346 }
 347
 348 /* Uses window coordinates (x,y) to find the depth in the GL depth buffer. If
 349    available, G.vd->depths is used so that the brush doesn't sculpt on top of
 350    itself (G.vd->depths is only updated at the end of a brush stroke.) */
 351 float get_depth(short x, short y)
 352 {
 353         float depth;
 354
 355         if(x<0 || y<0) return 1;
 356         if(x>=curarea->winx || y>=curarea->winy) return 1;
 357
 358         if(G.vd->depths && x<G.vd->depths->w && y<G.vd->depths->h)
 359                 return G.vd->depths->depths[y*G.vd->depths->w+x];
 360
 361         x+= curarea->winrct.xmin;
 362         y+= curarea->winrct.ymin;
 363
 364         glReadPixels(x, y, 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &depth);
 365
 366         return depth;
 367 }
 368
 369 /* Uses window coordinates (x,y) and depth component z to find a point in
 370    modelspace */
 371 vec3f unproject(const short x, const short y, const float z)
 372 {
 373         SculptSession *ss= sculpt_session();
 374         double ux, uy, uz;
 375         vec3f p;
 376
 377         gluUnProject(x,y,z, ss->mats.modelview, ss->mats.projection,
 378                      (GLint *)ss->mats.viewport, &ux, &uy, &uz );
 379         p.x= ux;
 380         p.y= uy;
 381         p.z= uz;
 382         return p;
 383 }
 384
 385 /* Convert a point in model coordinates to 2D screen coordinates. */
 386 void project(const float v[3], short p[2])
 387 {
 388         SculptSession *ss= sculpt_session();
 389         double ux, uy, uz;
 390
 391         gluProject(v[0],v[1],v[2], ss->mats.modelview, ss->mats.projection,
 392                    (GLint *)ss->mats.viewport, &ux, &uy, &uz);
 393         p[0]= ux;
 394         p[1]= uy;
 395 }
 396
 397 /* ===== Sculpting =====
 398  *
 399  */
 400
 401 /* Return modified brush size. Uses current tablet pressure (if available) to
 402    shrink the brush. Skipped for grab brush because only the first mouse down
 403    size is used, which is small if the user has just touched the pen to the
 404    tablet */
 405 char brush_size()
 406 {
 407         const BrushData *b= sculptmode_brush();
 408         float size= b->size;
 409         float pressure= get_pressure();
 410         short activedevice= get_activedevice();
 411
 412         if(sculpt_data()->brush_type!=GRAB_BRUSH) {
 413                 const float size_factor= G.scene->sculptdata.tablet_size / 10.0f;
 414                 if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
 415                         size*= G.scene->sculptdata.tablet_size==0?1:
 416                                (1-size_factor) + pressure*size_factor;
 417         }
 418
 419         return size;
 420 }
 421
 422 /* Return modified brush strength. Includes the direction of the brush, positive
 423    values pull vertices, negative values push. Uses tablet pressure and a
 424    special multiplier found experimentally to scale the strength factor. */
 425 float brush_strength(EditData *e)
 426 {
 427         const BrushData* b= sculptmode_brush();
 428         float dir= b->dir==1 ? 1 : -1;
 429         float pressure= 1;
 430         short activedevice= get_activedevice();
 431         float flip= e->flip ? -1:1;
 432
 433         const float strength_factor= G.scene->sculptdata.tablet_strength / 10.0f;
 434         if(ELEM(activedevice, DEV_STYLUS, DEV_ERASER))
 435                 pressure= G.scene->sculptdata.tablet_strength==0?1:
 436                                   (1-strength_factor) + get_pressure()*strength_factor;
 437
 438         /* Flip direction for eraser */
 439         if(activedevice==DEV_ERASER)
 440                 dir= -dir;
 441
 442         switch(G.scene->sculptdata.brush_type){
 443         case DRAW_BRUSH:
 444         case LAYER_BRUSH:
 445                 return b->strength / 5000.0f * dir * pressure * flip;
 446         case SMOOTH_BRUSH:
 447                 return b->strength / 50.0f * pressure;
 448         case PINCH_BRUSH:
 449                 return b->strength / 1000.0f * dir * pressure * flip;
 450         case GRAB_BRUSH:
 451                 return 1;
 452         case INFLATE_BRUSH:
 453                 return b->strength / 5000.0f * dir * pressure * flip;
 454         case FLATTEN_BRUSH:
 455                 return b->strength / 500.0f * pressure;
 456         default:
 457                 return 0;
 458         }
 459 }
 460
 461 /* For clipping against a mirror modifier */
 462 void sculpt_clip(const EditData *e, float *co, const float val[3])
 463 {
 464         char i;
 465         for(i=0; i<3; ++i) {
 466                 if(e->clip[i] && (fabs(co[i]) <= e->cliptol[i]))
 467                         co[i]= 0.0f;
 468                 else
 469                         co[i]= val[i];
 470         }
 471 }
 472
 473 /* Currently only for the draw brush; finds average normal for all active
 474    vertices */
 475 vec3f calc_area_normal(const vec3f *outdir, const ListBase* active_verts)
 476 {
 477         Mesh *me= get_mesh(OBACT);
 478         vec3f area_normal= {0,0,0};
 479         ActiveData *node= active_verts->first;
 480         const int view= sculpt_data()->brush_type==DRAW_BRUSH ? sculptmode_brush()->view : 0;
 481
 482         while(node){
 483                 area_normal.x+= me->mvert[node->Index].no[0];
 484                 area_normal.y+= me->mvert[node->Index].no[1];
 485                 area_normal.z+= me->mvert[node->Index].no[2];
 486                 node= node->next;
 487         }
 488         Normalize(&area_normal.x);
 489         if(outdir) {
 490                 area_normal.x= outdir->x * view + area_normal.x * (10-view);
 491                 area_normal.y= outdir->y * view + area_normal.y * (10-view);
 492                 area_normal.z= outdir->z * view + area_normal.z * (10-view);
 493         }
 494         Normalize(&area_normal.x);
 495         return area_normal;
 496 }
 497 void do_draw_brush(const EditData *e, const ListBase* active_verts)
 498 {
 499         Mesh *me= get_mesh(OBACT);
 500         const vec3f area_normal= calc_area_normal(&e->out, active_verts);
 501         ActiveData *node= active_verts->first;
 502
 503         while(node){
 504                 float *co= me->mvert[node->Index].co;
 505
 506                 const float val[3]= {co[0]+area_normal.x*node->Fade*e->scale[0],
 507                                      co[1]+area_normal.y*node->Fade*e->scale[1],
 508                                      co[2]+area_normal.z*node->Fade*e->scale[2]};
 509
 510                 sculpt_clip(e, co, val);
 511
 512                 node= node->next;
 513         }
 514 }
 515
 516 /* For the smooth brush, uses the neighboring vertices around vert to calculate
 517    a smoothed location for vert. Skips corner vertices (used by only one
 518    polygon.) */
 519 vec3f neighbor_average(const int vert)
 520 {
 521         SculptSession *ss= sculpt_session();
 522         Mesh *me= get_mesh(OBACT);
 523         int i, skip= -1, total=0;
 524         IndexNode *node= ss->vertex_users[vert].first;
 525         vec3f avg= {0,0,0};
 526         char ncount= BLI_countlist(&ss->vertex_users[vert]);
 527         MFace *f;
 528
 529         /* Don't modify corner vertices */
 530         if(ncount==1) {
 531                 VecCopyf(&avg.x, me->mvert[vert].co);
 532                 return avg;
 533         }
 534
 535         while(node){
 536                 f= &me->mface[node->Index];
 537
 538                 if(f->v4) {
 539                         skip= (f->v1==vert?2:
 540                                f->v2==vert?3:
 541                                f->v3==vert?0:
 542                                f->v4==vert?1:-1);
 543                 }
 544
 545                 for(i=0; i<(f->v4?4:3); ++i) {
 546                         if(i != skip && (ncount!=2 || BLI_countlist(&ss->vertex_users[(&f->v1)[i]]) <= 2)) {
 547                                 VecAddf(&avg.x,&avg.x,me->mvert[(&f->v1)[i]].co);
 548                                 ++total;
 549                         }
 550                 }
 551
 552                 node= node->next;
 553         }
 554
 555         if(total>0) {
 556                 avg.x/= total;
 557                 avg.y/= total;
 558                 avg.z/= total;
 559         }
 560         else
 561                 VecCopyf(&avg.x, me->mvert[vert].co);
 562
 563         return avg;
 564 }
 565
 566 void do_smooth_brush(const EditData *e, const ListBase* active_verts)
 567 {
 568         ActiveData *node= active_verts->first;
 569         Mesh *me= get_mesh(OBACT);
 570
 571         while(node){
 572                 float *co= me->mvert[node->Index].co;
 573                 const vec3f avg= neighbor_average(node->Index);
 574                 const float val[3]= {co[0]+(avg.x-co[0])*node->Fade,
 575                                      co[1]+(avg.y-co[1])*node->Fade,
 576                                      co[2]+(avg.z-co[2])*node->Fade};
 577                 sculpt_clip(e, co, val);
 578                 node= node->next;
 579         }
 580 }
 581
 582 void do_pinch_brush(const EditData *e, const ListBase* active_verts)
 583 {
 584         Mesh *me= get_mesh(OBACT);
 585         ActiveData *node= active_verts->first;
 586
 587         while(node) {
 588                 float *co= me->mvert[node->Index].co;
 589                 const float val[3]= {co[0]+(e->center.x-co[0])*node->Fade,
 590                                      co[1]+(e->center.y-co[1])*node->Fade,
 591                                      co[2]+(e->center.z-co[2])*node->Fade};
 592                 sculpt_clip(e, co, val);
 593                 node= node->next;
 594         }
 595 }
 596
 597 void do_grab_brush(EditData *e)
 598 {
 599         Mesh *me= get_mesh(OBACT);
 600         ActiveData *node= e->grabdata->active_verts[e->grabdata->index].first;
 601         float add[3];
 602
 603         while(node) {
 604                 float *co= me->mvert[node->Index].co;
 605
 606                 VecCopyf(add, &e->grabdata->delta_symm.x);
 607                 VecMulf(add, node->Fade);
 608                 VecAddf(add, add, co);
 609                 sculpt_clip(e, co, add);
 610
 611                 node= node->next;
 612         }
 613 }
 614
 615 void do_layer_brush(EditData *e, const ListBase *active_verts)
 616 {
 617         Mesh *me= get_mesh(OBACT);
 618         vec3f area_normal= calc_area_normal(NULL, active_verts);
 619         ActiveData *node= active_verts->first;
 620         const float bstr= brush_strength(e);
 621
 622         while(node){
 623                 float *disp= &e->layer_disps[node->Index];
 624
 625                 if((bstr > 0 && *disp < bstr) ||
 626                   (bstr < 0 && *disp > bstr)) {
 627                         float *co= me->mvert[node->Index].co;
 628
 629                         *disp+= node->Fade;
 630
 631                         if(bstr < 0) {
 632                                 if(*disp < bstr)
 633                                         *disp = bstr;
 634                         } else {
 635                                 if(*disp > bstr)
 636                                         *disp = bstr;
 637                         }
 638
 639                         {
 640                                 const float val[3]= {e->layer_store[node->Index].x+area_normal.x * *disp*e->scale[0],
 641                                                      e->layer_store[node->Index].y+area_normal.y * *disp*e->scale[1],
 642                                                      e->layer_store[node->Index].z+area_normal.z * *disp*e->scale[2]};
 643                                 sculpt_clip(e, co, val);
 644                         }
 645                 }
 646
 647                 node= node->next;
 648         }
 649 }
 650
 651 void do_inflate_brush(const EditData *e, const ListBase *active_verts)
 652 {
 653         ActiveData *node= active_verts->first;
 654         float add[3];
 655         Mesh *me= get_mesh(OBACT);
 656
 657         while(node) {
 658                 float *co= me->mvert[node->Index].co;
 659                 short *no= me->mvert[node->Index].no;
 660
 661                 add[0]= no[0]/ 32767.0f;
 662                 add[1]= no[1]/ 32767.0f;
 663                 add[2]= no[2]/ 32767.0f;
 664                 VecMulf(add, node->Fade);
 665                 add[0]*= e->scale[0];
 666                 add[1]*= e->scale[1];
 667                 add[2]*= e->scale[2];
 668                 VecAddf(add, add, co);
 669
 670                 sculpt_clip(e, co, add);
 671
 672                 node= node->next;
 673         }
 674 }
 675
 676 void calc_flatten_center(Mesh *me, ActiveData *node, const EditData *e, float co[3])
 677 {
 678         ActiveData *outer[FLATTEN_SAMPLE_SIZE];
 679         int i;
 680
 681         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
 682                 outer[i] = node;
 683
 684         for(; node; node = node->next) {
 685                 for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i) {
 686                         if(node->dist > outer[i]->dist) {
 687                                 outer[i] = node;
 688                                 break;
 689                         }
 690                 }
 691         }
 692
 693         co[0] = co[1] = co[2] = 0.0f;
 694         for(i = 0; i < FLATTEN_SAMPLE_SIZE; ++i)
 695                 VecAddf(co, co, me->mvert[outer[i]->Index].co);
 696         VecMulf(co, 1.0f / FLATTEN_SAMPLE_SIZE);
 697 }
 698
 699 void do_flatten_brush(const EditData *e, const ListBase *active_verts)
 700 {
 701         Mesh *me= get_mesh(OBACT);
 702         ActiveData *node= active_verts->first;
 703         /* area_normal and cntr define the plane towards which vertices are squashed */
 704         vec3f area_normal= calc_area_normal(&e->out, active_verts);
 705         float cntr[3];
 706
 707         calc_flatten_center(me, node, e, cntr);
 708
 709         while(node){
 710                 float *co= me->mvert[node->Index].co;
 711                 float p1[3], sub1[3], sub2[3], intr[3], val[3];
 712
 713                 /* Find the intersection between squash-plane and vertex (along the area normal) */
 714                 VecSubf(p1, co, &area_normal.x);
 715                 VecSubf(sub1, cntr, p1);
 716                 VecSubf(sub2, co, p1);
 717                 VecSubf(intr, co, p1);
 718                 VecMulf(intr, Inpf(&area_normal.x, sub1) / Inpf(&area_normal.x, sub2));
 719                 VecAddf(intr, intr, p1);
 720
 721                 VecSubf(val, intr, co);
 722                 VecMulf(val, node->Fade);
 723                 VecAddf(val, val, co);
 724
 725                 sculpt_clip(e, co, val);
 726
 727                 node= node->next;
 728         }
 729 }
 730
 731 /* Creates a smooth curve for the brush shape. This is the cos(x) curve from
 732    [0,PI] scaled to [0,len]. The range is scaled to [0,1]. */
 733 float simple_strength(float p, const float len)
 734 {
 735         if(p > len) p= len;
 736         return 0.5f * (cos(M_PI*p/len) + 1);
 737 }
 738
 739 /* Uses symm to selectively flip any axis of a coordinate. */
 740 void flip_coord(float co[3], const char symm)
 741 {
 742         if(symm & SYMM_X)
 743                 co[0]= -co[0];
 744         if(symm & SYMM_Y)
 745                 co[1]= -co[1];
 746         if(symm & SYMM_Z)
 747                 co[2]= -co[2];
 748 }
 749
 750 /* Use the warpfac field in MTex to store a rotation value for sculpt textures. Value is in degrees */
 751 float tex_angle(void)
 752 {
 753         SculptData *sd= sculpt_data();
 754         if(sd->texact!=-1 && sd->mtex[sd->texact])
 755                 return sd->mtex[sd->texact]->warpfac;
 756         return 0;
 757 }
 758
 759 void set_tex_angle(const float f)
 760 {
 761         SculptData *sd = sculpt_data();
 762         if(sd->texact != -1 && sd->mtex[sd->texact])
 763                 sd->mtex[sd->texact]->warpfac = f;
 764 }
 765
 766 float to_rad(const float deg)
 767 {
 768         return deg * (M_PI/180.0f);
 769 }
 770
 771 float to_deg(const float rad)
 772 {
 773         return rad * (180.0f/M_PI);
 774 }
 775
 776 /* Get a pixel from the texcache at (px, py) */
 777 unsigned *get_texcache_pixel(const SculptSession *ss, int px, int py)
 778 {
 779         if(px < 0) px= 0;
 780         if(py < 0) py= 0;
 781         if(px > ss->texcache_w - 1) px= ss->texcache_w - 1;
 782         if(py > ss->texcache_h - 1) py= ss->texcache_h - 1;
 783         return ss->texcache + py * ss->texcache_w + px;
 784 }
 785
 786 /* Return a multiplier for brush strength on a particular vertex. */
 787 float tex_strength(EditData *e, float *point, const float len,const unsigned vindex)
 788 {
 789         SculptData *sd= sculpt_data();
 790         SculptSession *ss= sculpt_session();
 791         float avg= 1;
 792
 793         if(sd->texact==-1 || !sd->mtex[sd->texact])
 794                 avg= 1;
 795         else if(sd->texrept==SCULPTREPT_3D) {
 796                 /* Get strength by feeding the vertex location directly
 797                    into a texture */
 798                 float jnk;
 799                 const float factor= 0.01;
 800                 MTex mtex;
 801                 memset(&mtex,0,sizeof(MTex));
 802                 mtex.tex= sd->mtex[sd->texact]->tex;
 803                 mtex.projx= 1;
 804                 mtex.projy= 2;
 805                 mtex.projz= 3;
 806                 VecCopyf(mtex.size, sd->mtex[sd->texact]->size);
 807                 VecMulf(mtex.size, factor);
 808                 if(!sd->texsep)
 809                         mtex.size[1]= mtex.size[2]= mtex.size[0];
 810
 811                 externtex(&mtex,point,&avg,&jnk,&jnk,&jnk,&jnk);
 812         }
 813         else if(ss->texcache) {
 814                 const short bsize= sculptmode_brush()->size * 2;
 815                 const short half= sculptmode_brush()->size;
 816                 const float rot= to_rad(tex_angle());
 817                 const unsigned tcw = ss->texcache_w, tch = ss->texcache_h;
 818                 int px, py;
 819                 unsigned i, *p;
 820                 ProjVert pv;
 821
 822                 /* If the active area is being applied for symmetry, flip it
 823                    across the symmetry axis in order to project it. This insures
 824                    that the brush texture will be oriented correctly. */
 825                 if(!e->symm)
 826                         pv= projverts[vindex];
 827                 else {
 828                         float co[3];
 829                         VecCopyf(co, point);
 830                         flip_coord(co, e->symm);
 831                         project(co, pv.co);
 832                 }
 833
 834                 /* For Tile and Drag modes, get the 2D screen coordinates of the
 835                    and scale them up or down to the texture size. */
 836                 if(sd->texrept==SCULPTREPT_TILE) {
 837                         const int sx= (const int)sd->mtex[sd->texact]->size[0];
 838                         const int sy= (const int)sd->texsep ? sd->mtex[sd->texact]->size[1] : sx;
 839
 840                         float fx= pv.co[0];
 841                         float fy= pv.co[1];
 842
 843                         float angle= atan2(fy, fx) - rot;
 844                         float len= sqrtf(fx*fx + fy*fy);
 845
 846                         if(rot<0.001 && rot>-0.001) {
 847                                 px= pv.co[0];
 848                                 py= pv.co[1];
 849                         } else {
 850                                 px= len * cos(angle) + 2000;
 851                                 py= len * sin(angle) + 2000;
 852                         }
 853                         px %= sx-1;
 854                         py %= sy-1;
 855                         p= get_texcache_pixel(ss, tcw*px/sx, tch*py/sy);
 856                 } else {
 857                         float fx= (pv.co[0] - e->mouse[0] + half) * (tcw*1.0f/bsize) - tcw/2;
 858                         float fy= (pv.co[1] - e->mouse[1] + half) * (tch*1.0f/bsize) - tch/2;
 859
 860                         float angle= atan2(fy, fx) - rot;
 861                         float len= sqrtf(fx*fx + fy*fy);
 862
 863                         px= tcw/2 + len * cos(angle);
 864                         py= tch/2 + len * sin(angle);
 865
 866                         p= get_texcache_pixel(ss, px, py);
 867                 }
 868
 869                 avg= 0;
 870                 for(i=0; i<3; ++i)
 871                         avg+= ((unsigned char*)(p))[i] / 255.0f;
 872
 873                 avg/= 3;
 874         }
 875
 876         if(sd->texfade)
 877                 avg*= simple_strength(len,e->size); /* Smooth curve */
 878
 879         return avg;
 880 }
 881
 882 /* Mark area around the brush as damaged. projverts are marked if they are
 883    inside the area and the damaged rectangle in 2D screen coordinates is
 884    added to damaged_rects. */
 885 void sculptmode_add_damaged_rect(EditData *e)
 886 {
 887         short p[2];
 888         const float radius= brush_size();
 889         RectNode *rn= MEM_mallocN(sizeof(RectNode),"RectNode");
 890         Mesh *me= get_mesh(OBACT);
 891         unsigned i;
 892
 893         /* Find center */
 894         project(&e->center.x, p);
 895         rn->r.xmin= p[0]-radius;
 896         rn->r.ymin= p[1]-radius;
 897         rn->r.xmax= p[0]+radius;
 898         rn->r.ymax= p[1]+radius;
 899
 900         BLI_addtail(&sculpt_session()->damaged_rects, rn);
 901
 902         /* Update insides */
 903         for(i=0; i<me->totvert; ++i) {
 904                 if(!projverts[i].inside) {
 905                         if(projverts[i].co[0] > rn->r.xmin && projverts[i].co[1] > rn->r.ymin &&
 906                            projverts[i].co[0] < rn->r.xmax && projverts[i].co[1] < rn->r.ymax) {
 907                                 projverts[i].inside= 1;
 908                         }
 909                 }
 910         }
 911 }
 912
 913 void do_brush_action(EditData e)
 914 {
 915         int i;
 916         float av_dist;
 917         ListBase active_verts={0,0};
 918         ActiveData *adata= 0;
 919         float *vert;
 920         Mesh *me= get_mesh(OBACT);
 921         const float bstrength= brush_strength(&e);
 922         KeyBlock *keyblock= ob_get_keyblock(OBACT);
 923         SculptSession *ss = sculpt_session();
 924
 925         sculptmode_add_damaged_rect(&e);
 926
 927         /* Build a list of all vertices that are potentially within the brush's
 928            area of influence. Only do this once for the grab brush. */
 929         if(!e.grabdata || (e.grabdata && e.grabdata->firsttime)) {
 930                 for(i=0; i<me->totvert; ++i) {
 931                         /* Projverts.inside provides a rough bounding box */
 932                         if(projverts[i].inside) {
 933                                 vert= ss->vertexcosnos ? &ss->vertexcosnos[i*6] : me->mvert[i].co;
 934                                 av_dist= VecLenf(&e.center.x,vert);
 935                                 if(av_dist < e.size) {
 936                                         adata= (ActiveData*)MEM_mallocN(sizeof(ActiveData), "ActiveData");
 937                                         adata->Index = i;
 938                                         /* Fade is used to store the final strength at which the brush
 939                                            should modify a particular vertex. */
 940                                         adata->Fade= tex_strength(&e,vert,av_dist,i) * bstrength;
 941                                         adata->dist = av_dist;
 942                                         if(e.grabdata && e.grabdata->firsttime)
 943                                                 BLI_addtail(&e.grabdata->active_verts[e.grabdata->index], adata);
 944                                         else
 945                                                 BLI_addtail(&active_verts, adata);
 946                                 }
 947                         }
 948                 }
 949         }
 950
 951         /* Only act if some verts are inside the brush area */
 952         if(active_verts.first || (e.grabdata && e.grabdata->active_verts[e.grabdata->index].first)) {
 953                 /* Apply one type of brush action */
 954                 switch(G.scene->sculptdata.brush_type){
 955                 case DRAW_BRUSH:
 956                         do_draw_brush(&e, &active_verts);
 957                         break;
 958                 case SMOOTH_BRUSH:
 959                         do_smooth_brush(&e, &active_verts);
 960                         break;
 961                 case PINCH_BRUSH:
 962                         do_pinch_brush(&e, &active_verts);
 963                         break;
 964                 case INFLATE_BRUSH:
 965                         do_inflate_brush(&e, &active_verts);
 966                         break;
 967                 case GRAB_BRUSH:
 968                         do_grab_brush(&e);
 969                         break;
 970                 case LAYER_BRUSH:
 971                         do_layer_brush(&e, &active_verts);
 972                         break;
 973                 case FLATTEN_BRUSH:
 974                         do_flatten_brush(&e, &active_verts);
 975                         break;
 976                 }
 977
 978                 /* Copy the modified vertices from mesh to the active key */
 979                 if(keyblock) {
 980                         float *co= keyblock->data;
 981                         if(co) {
 982                                 adata = e.grabdata ? e.grabdata->active_verts[e.grabdata->index].first : active_verts.first;
 983                                 for(; adata; adata= adata->next)
 984                                         if(adata->Index < keyblock->totelem)
 985                                                 VecCopyf(&co[adata->Index*3], me->mvert[adata->Index].co);
 986                         }
 987                 }
 988
 989                 if(ss->vertexcosnos)
 990                         BLI_freelistN(&active_verts);
 991                 else {
 992                         if(!e.grabdata)
 993                                 addlisttolist(&ss->damaged_verts, &active_verts);
 994                 }
 995         }
 996 }
 997
 998 /* Flip all the editdata across the axis/axes specified by symm. Used to
 999    calculate multiple modifications to the mesh when symmetry is enabled. */
1000 EditData flip_editdata(EditData *e, const char symm)
1001 {
1002         EditData fe= *e;
1003         GrabData *gd= fe.grabdata;
1004
1005         flip_coord(&fe.center.x, symm);
1006         flip_coord(&fe.up.x, symm);
1007         flip_coord(&fe.right.x, symm);
1008         flip_coord(&fe.out.x, symm);
1009
1010         fe.symm= symm;
1011
1012         project(&e->center.x,fe.mouse);
1013
1014         if(gd) {
1015                 gd->index= symm;
1016                 gd->delta_symm= gd->delta;
1017                 flip_coord(&gd->delta_symm.x, symm);
1018         }
1019
1020         return fe;
1021 }
1022
1023 void do_symmetrical_brush_actions(EditData * e, short co[2], short pr_co[2])
1024 {
1025         const char symm= sculpt_data()->symm;
1026
1027         init_editdata(e, co, pr_co);
1028
1029         do_brush_action(flip_editdata(e, 0));
1030
1031         if(symm & SYMM_X)
1032                 do_brush_action(flip_editdata(e, SYMM_X));
1033         if(symm & SYMM_Y)
1034                 do_brush_action(flip_editdata(e, SYMM_Y));
1035         if(symm & SYMM_Z)
1036                 do_brush_action(flip_editdata(e, SYMM_Z));
1037         if(symm & SYMM_X && symm & SYMM_Y)
1038                 do_brush_action(flip_editdata(e, SYMM_X | SYMM_Y));
1039         if(symm & SYMM_X && symm & SYMM_Z)
1040                 do_brush_action(flip_editdata(e, SYMM_X | SYMM_Z));
1041         if(symm & SYMM_Y && symm & SYMM_Z)
1042                 do_brush_action(flip_editdata(e, SYMM_Y | SYMM_Z));
1043         if(symm & SYMM_X && symm & SYMM_Y && symm & SYMM_Z)
1044                 do_brush_action(flip_editdata(e, SYMM_X | SYMM_Y | SYMM_Z));
1045 }
1046
1047 void add_face_normal(vec3f *norm, const MFace* face)
1048 {
1049         Mesh *me= get_mesh(OBACT);
1050
1051         vec3f c= {me->mvert[face->v1].co[0],me->mvert[face->v1].co[1],me->mvert[face->v1].co[2]};
1052         vec3f b= {me->mvert[face->v2].co[0],me->mvert[face->v2].co[1],me->mvert[face->v2].co[2]};
1053         vec3f a= {me->mvert[face->v3].co[0],me->mvert[face->v3].co[1],me->mvert[face->v3].co[2]};
1054         vec3f s1, s2;
1055
1056         VecSubf(&s1.x,&a.x,&b.x);
1057         VecSubf(&s2.x,&c.x,&b.x);
1058
1059         norm->x+= s1.y * s2.z - s1.z * s2.y;
1060         norm->y+= s1.z * s2.x - s1.x * s2.z;
1061         norm->z+= s1.x * s2.y - s1.y * s2.x;
1062 }
1063
1064 void update_damaged_vert(Mesh *me, ListBase *lb)
1065 {
1066         ActiveData *vert;
1067
1068         for(vert= lb->first; vert; vert= vert->next) {
1069                 vec3f norm= {0,0,0};
1070                 IndexNode *face= sculpt_session()->vertex_users[vert->Index].first;
1071
1072                 while(face){
1073                         add_face_normal(&norm,&me->mface[face->Index]);
1074                         face= face->next;
1075                 }
1076                 Normalize(&norm.x);
1077
1078                 me->mvert[vert->Index].no[0]=norm.x*32767;
1079                 me->mvert[vert->Index].no[1]=norm.y*32767;
1080                 me->mvert[vert->Index].no[2]=norm.z*32767;
1081         }
1082 }
1083
1084 void calc_damaged_verts(ListBase *damaged_verts, GrabData *grabdata)
1085 {
1086         Mesh *me= get_mesh(OBACT);
1087
1088         if(grabdata) {
1089                 int i;
1090                 for(i=0; i<8; ++i)
1091                         update_damaged_vert(me,&grabdata->active_verts[i]);
1092         } else {
1093                 update_damaged_vert(me,damaged_verts);
1094                 BLI_freelistN(damaged_verts);
1095                 damaged_verts->first = damaged_verts->last = NULL;
1096         }
1097 }
1098
1099 void projverts_clear_inside()
1100 {
1101         Mesh *me = get_mesh(OBACT);
1102         if(me) {
1103                 int i;
1104                 for(i = 0; i < me->totvert; ++i)
1105                         projverts[i].inside = 0;
1106         }
1107 }
1108
1109 BrushData *sculptmode_brush(void)
1110 {
1111         SculptData *sd= &G.scene->sculptdata;
1112
1113         BrushData *bd =
1114                 (sd->brush_type==DRAW_BRUSH ? &sd->drawbrush :
1115                  sd->brush_type==SMOOTH_BRUSH ? &sd->smoothbrush :
1116                  sd->brush_type==PINCH_BRUSH ? &sd->pinchbrush :
1117                  sd->brush_type==INFLATE_BRUSH ? &sd->inflatebrush :
1118                  sd->brush_type==GRAB_BRUSH ? &sd->grabbrush :
1119                  sd->brush_type==LAYER_BRUSH ? &sd->layerbrush :
1120                  sd->brush_type==FLATTEN_BRUSH ? &sd->flattenbrush : NULL);
1121
1122         if(!bd) {
1123                 sculptmode_init(G.scene);
1124                 bd = &sd->drawbrush;
1125         }
1126
1127         return bd;
1128 }
1129
1130 void sculptmode_update_tex()
1131 {
1132         SculptData *sd= sculpt_data();
1133         SculptSession *ss= sculpt_session();
1134         MTex *mtex = sd->mtex[sd->texact];
1135         TexResult texres = {0};
1136         float x, y, step=2.0/128.0, co[3];
1137         int hasrgb, ix, iy;
1138
1139         /* Skip Default brush shape and non-textures */
1140         if(sd->texact == -1 || !sd->mtex[sd->texact]) return;
1141
1142         if(ss->texcache) {
1143                 MEM_freeN(ss->texcache);
1144                 ss->texcache= NULL;
1145         }
1146
1147         ss->texcache_w = ss->texcache_h = 128;
1148         ss->texcache = MEM_callocN(sizeof(int) * ss->texcache_w * ss->texcache_h, "Sculpt Texture cache");
1149
1150         if(mtex && mtex->tex) {
1151                 BKE_image_get_ibuf(sd->mtex[sd->texact]->tex->ima, NULL);
1152
1153                 /*do normalized cannonical view coords for texture*/
1154                 for (y=-1.0, iy=0; iy<128; iy++, y += step) {
1155                         for (x=-1.0, ix=0; ix<128; ix++, x += step) {
1156                                 co[0]= x;
1157                                 co[1]= y;
1158                                 co[2]= 0.0f;
1159
1160                                 /* This is copied from displace modifier code */
1161                                 hasrgb = multitex_ext(mtex->tex, co, NULL, NULL, 1, &texres);
1162
1163                                 /* if the texture gave an RGB value, we assume it didn't give a valid
1164                                  * intensity, so calculate one (formula from do_material_tex).
1165                                  * if the texture didn't give an RGB value, copy the intensity across
1166                                  */
1167                                 if(hasrgb & TEX_RGB)
1168                                         texres.tin = (0.35 * texres.tr + 0.45 *
1169                                                       texres.tg + 0.2 * texres.tb);
1170
1171                                 texres.tin = texres.tin * 255.0;
1172                                 ((char*)ss->texcache)[(iy*128+ix)*4] = (char)texres.tin;
1173                                 ((char*)ss->texcache)[(iy*128+ix)*4+1] = (char)texres.tin;
1174                                 ((char*)ss->texcache)[(iy*128+ix)*4+2] = (char)texres.tin;
1175                                 ((char*)ss->texcache)[(iy*128+ix)*4+3] = (char)texres.tin;
1176                         }
1177                 }
1178         }
1179 }
1180
1181 /* pr_mouse is only used for the grab brush, can be NULL otherwise */
1182 void init_editdata(EditData *e, short *mouse, short *pr_mouse)
1183 {
1184         SculptData *sd = sculpt_data();
1185         const float mouse_depth= get_depth(mouse[0],mouse[1]);
1186         vec3f brush_edge_loc, zero_loc, oldloc;
1187         ModifierData *md;
1188         int i;
1189         const char flip = (get_qual() == LR_SHIFTKEY);
1190
1191         e->flip= flip;
1192
1193         /* Convert the location and size of the brush to
1194            modelspace coords */
1195         e->center= unproject(mouse[0],mouse[1],mouse_depth);
1196         brush_edge_loc= unproject(mouse[0] +
1197                                   brush_size(),mouse[1],
1198                                   mouse_depth);
1199         e->size= VecLenf(&e->center.x,&brush_edge_loc.x);
1200
1201         /* Set the pivot to allow the model to rotate around the center of the brush */
1202         if(get_depth(mouse[0],mouse[1]) < 1.0)
1203                 sculpt_session()->pivot= e->center;
1204
1205         /* Now project the Up, Right, and Out normals from view to model coords */
1206         zero_loc= unproject(0, 0, 0);
1207         e->up= unproject(0, -1, 0);
1208         e->right= unproject(1, 0, 0);
1209         e->out= unproject(0, 0, -1);
1210         VecSubf(&e->up.x, &e->up.x, &zero_loc.x);
1211         VecSubf(&e->right.x, &e->right.x, &zero_loc.x);
1212         VecSubf(&e->out.x, &e->out.x, &zero_loc.x);
1213         Normalize(&e->up.x);
1214         Normalize(&e->right.x);
1215         Normalize(&e->out.x);
1216
1217         /* Initialize mirror modifier clipping */
1218         for(i=0; i<3; ++i) {
1219                 e->clip[i]= 0;
1220                 e->cliptol[i]= 0;
1221         }
1222         for(md= OBACT->modifiers.first; md; md= md->next) {
1223                 if(md->type==eModifierType_Mirror && (md->mode & eModifierMode_Realtime)) {
1224                         const MirrorModifierData *mmd = (MirrorModifierData*) md;
1225
1226                         if(mmd->flag & MOD_MIR_CLIPPING) {
1227                                 e->clip[mmd->axis]= 1;
1228                                 if(mmd->tolerance > e->cliptol[mmd->axis])
1229                                         e->cliptol[mmd->axis]= mmd->tolerance;
1230                         }
1231                 }
1232         }
1233
1234         if(sd->brush_type == GRAB_BRUSH) {
1235                 vec3f gcenter;
1236                 if(!e->grabdata) {
1237                         e->grabdata= MEM_callocN(sizeof(GrabData),"grab data");
1238                         e->grabdata->firsttime= 1;
1239                         e->grabdata->depth= mouse_depth;
1240                 }
1241                 else
1242                         e->grabdata->firsttime= 0;
1243
1244                 /* Find the delta */
1245                 gcenter= unproject(mouse[0],mouse[1],e->grabdata->depth);
1246                 oldloc= unproject(pr_mouse[0],pr_mouse[1],e->grabdata->depth);
1247                 VecSubf(&e->grabdata->delta.x,&gcenter.x,&oldloc.x);
1248         }
1249         else if(sd->brush_type == LAYER_BRUSH) {
1250                 Mesh *me= get_mesh(OBACT);
1251
1252                 if(!e->layer_disps)
1253                         e->layer_disps= MEM_callocN(sizeof(float)*me->totvert,"Layer disps");
1254                 if(!e->layer_store) {
1255                         unsigned i;
1256                         e->layer_store= MEM_mallocN(sizeof(vec3f)*me->totvert,"Layer store");
1257                         for(i=0; i<me->totvert; ++i)
1258                                 VecCopyf(&e->layer_store[i].x,me->mvert[i].co);
1259                 }
1260         }
1261 }
1262 void sculptmode_set_strength(const int delta)
1263 {
1264         int val = sculptmode_brush()->strength + delta;
1265         if(val < 1) val = 1;
1266         if(val > 100) val = 100;
1267         sculptmode_brush()->strength= val;
1268 }
1269
1270 void sculptmode_propset_calctex()
1271 {
1272         SculptData *sd= sculpt_data();
1273         SculptSession *ss= sculpt_session();
1274         PropsetData *pd= sculpt_session()->propset;
1275         if(pd) {
1276                 int i, j;
1277                 const int tsz = 128;
1278                 float *d;
1279                 if(!pd->texdata) {
1280                         pd->texdata= MEM_mallocN(sizeof(float)*tsz*tsz, "Brush preview");
1281                         if(sd->texrept!=SCULPTREPT_3D)
1282                                 sculptmode_update_tex();
1283                         for(i=0; i<tsz; ++i)
1284                                 for(j=0; j<tsz; ++j) {
1285                                         float magn= sqrt(pow(i-tsz/2,2)+pow(j-tsz/2,2));
1286                                         if(sd->texfade)
1287                                                 pd->texdata[i*tsz+j]= simple_strength(magn,tsz/2);
1288                                         else
1289                                                 pd->texdata[i*tsz+j]= magn < tsz/2 ? 1 : 0;
1290                                 }
1291                         if(sd->texact != -1 && ss->texcache) {
1292                                 for(i=0; i<tsz; ++i)
1293                                         for(j=0; j<tsz; ++j) {
1294                                                 const int col= ss->texcache[i*tsz+j];
1295                                                 pd->texdata[i*tsz+j]*= (((char*)&col)[0]+((char*)&col)[1]+((char*)&col)[2])/3.0f/255.0f;
1296                                         }
1297                         }
1298                 }
1299
1300                 /* Adjust alpha with brush strength */
1301                 d= MEM_dupallocN(pd->texdata);
1302                 for(i=0; i<tsz; ++i)
1303                         for(j=0; j<tsz; ++j)
1304                                 d[i*tsz+j]*= sculptmode_brush()->strength/200.0f+0.5f;
1305
1306
1307                 if(!pd->tex)
1308                         glGenTextures(1, (GLuint *)&pd->tex);
1309                 glBindTexture(GL_TEXTURE_2D, pd->tex);
1310
1311                 glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, tsz, tsz, 0, GL_ALPHA, GL_FLOAT, d);
1312                 MEM_freeN(d);
1313         }
1314 }
1315
1316 void sculptmode_propset_header()
1317 {
1318         SculptSession *ss= sculpt_session();
1319         PropsetData *pd= ss ? ss->propset : NULL;
1320         if(pd) {
1321                 char str[512];
1322                 const char *name= "";
1323                 int val= 0;
1324                 if(pd->mode == PropsetSize) {
1325                         name= "Size";
1326                         val= sculptmode_brush()->size;
1327                 }
1328                 else if(pd->mode == PropsetStrength) {
1329                         name= "Strength";
1330                         val= sculptmode_brush()->strength;
1331                 }
1332                 else if(pd->mode == PropsetTexRot) {
1333                         name= "Texture Angle";
1334                         val= tex_angle();
1335                 }
1336                 sprintf(str, "Brush %s: %d", name, val);
1337                 headerprint(str);
1338         }
1339 }
1340
1341 void sculptmode_propset_end(SculptSession *ss, int cancel)
1342 {
1343         if(ss && ss->propset) {
1344                 PropsetData *pd= ss->propset;
1345
1346                 if(cancel) {
1347                         sculptmode_brush()->size= pd->origsize;
1348                         sculptmode_brush()->strength= pd->origstrength;
1349                         set_tex_angle(pd->origtexrot);
1350                 } else {
1351                         if(pd->mode != PropsetSize)
1352                                 sculptmode_brush()->size= pd->origsize;
1353                         if(pd->mode != PropsetStrength)
1354                                 sculptmode_brush()->strength= pd->origstrength;
1355                         if(pd->mode != PropsetTexRot)
1356                                 set_tex_angle(pd->origtexrot);
1357                 }
1358                 glDeleteTextures(1, &pd->tex);
1359                 MEM_freeN(pd->texdata);
1360                 MEM_freeN(pd);
1361                 ss->propset= NULL;
1362                 allqueue(REDRAWVIEW3D, 0);
1363                 allqueue(REDRAWBUTSEDIT, 0);
1364                 allqueue(REDRAWHEADERS, 0);
1365         }
1366 }
1367
1368 void sculptmode_propset_init(PropsetMode mode)
1369 {
1370         SculptSession *ss= sculpt_session();
1371         PropsetData *pd= ss->propset;
1372         const float ang= tex_angle();
1373
1374         if(!pd) {
1375                 short mouse[2];
1376
1377                 pd= MEM_callocN(sizeof(PropsetData),"PropsetSize");
1378                 ss->propset= pd;
1379
1380                 getmouseco_areawin(mouse);
1381                 pd->origloc[0]= mouse[0];
1382                 pd->origloc[1]= mouse[1];
1383
1384                 if(mode == PropsetSize)
1385                         pd->origloc[0]-= sculptmode_brush()->size;
1386                 else if(mode == PropsetStrength)
1387                         pd->origloc[0]-= 200 - 2*sculptmode_brush()->strength;
1388                 else if(mode == PropsetTexRot) {
1389                         pd->origloc[0]-= 200 * cos(to_rad(ang));
1390                         pd->origloc[1]-= 200 * sin(to_rad(ang));
1391                 }
1392
1393                 pd->origsize= sculptmode_brush()->size;
1394                 pd->origstrength= sculptmode_brush()->strength;
1395                 pd->origtexrot= ang;
1396
1397                 sculptmode_propset_calctex();
1398
1399                 pd->num.idx_max= 0;
1400         }
1401
1402         pd->mode= mode;
1403         sculptmode_propset_header();
1404
1405         allqueue(REDRAWVIEW3D, 0);
1406 }
1407
1408 void sculpt_paint_brush(char clear)
1409 {
1410         if(sculpt_data()->flags & SCULPT_DRAW_BRUSH) {
1411                 static short mvalo[2];
1412                 short mval[2];
1413                 const short rad= sculptmode_brush()->size;
1414
1415                 getmouseco_areawin(mval);
1416
1417                 persp(PERSP_WIN);
1418                 if(clear)
1419                         fdrawXORcirc(mval[0], mval[1], rad);
1420                 else
1421                         draw_sel_circle(mval, mvalo, rad, rad, 0);
1422
1423                 mvalo[0]= mval[0];
1424                 mvalo[1]= mval[1];
1425         }
1426 }
1427
1428 void sculptmode_propset(unsigned short event)
1429 {
1430         PropsetData *pd= sculpt_session()->propset;
1431         short mouse[2];
1432         short tmp[2];
1433         float dist;
1434         BrushData *brush= sculptmode_brush();
1435         char valset= 0;
1436
1437         handleNumInput(&pd->num, event);
1438
1439         if(hasNumInput(&pd->num)) {
1440                 float val;
1441                 applyNumInput(&pd->num, &val);
1442                 if(pd->mode==PropsetSize)
1443                         brush->size= val;
1444                 else if(pd->mode==PropsetStrength)
1445                         brush->strength= val;
1446                 else if(pd->mode==PropsetTexRot)
1447                         set_tex_angle(val);
1448                 valset= 1;
1449                 allqueue(REDRAWVIEW3D, 0);
1450         }
1451
1452         switch(event) {
1453         case MOUSEX:
1454         case MOUSEY:
1455                 if(!hasNumInput(&pd->num)) {
1456                         char ctrl= G.qual & LR_CTRLKEY;
1457                         getmouseco_areawin(mouse);
1458                         tmp[0]= pd->origloc[0]-mouse[0];
1459                         tmp[1]= pd->origloc[1]-mouse[1];
1460                         dist= sqrt(tmp[0]*tmp[0]+tmp[1]*tmp[1]);
1461                         if(pd->mode == PropsetSize) {
1462                                 brush->size= dist;
1463                                 if(ctrl) brush->size= (brush->size+5)/10*10;
1464                         } else if(pd->mode == PropsetStrength) {
1465                                 float fin= (200.0f - dist) * 0.5f;
1466                                 brush->strength= fin>=0 ? fin : 0;
1467                                 if(ctrl) brush->strength= (brush->strength+5)/10*10;
1468                         } else if(pd->mode == PropsetTexRot) {
1469                                 set_tex_angle((int)to_deg(atan2(tmp[1], tmp[0])) + 180);
1470                                 if(ctrl)
1471                                         set_tex_angle(((int)(tex_angle())+5)/10*10);
1472                         }
1473                         valset= 1;
1474                         allqueue(REDRAWVIEW3D, 0);
1475                 }
1476                 break;
1477         case ESCKEY:
1478         case RIGHTMOUSE:
1479                 brush->size= pd->origsize;
1480                 brush->strength= pd->origstrength;
1481                 set_tex_angle(pd->origtexrot);
1482         case LEFTMOUSE:
1483                 while(get_mbut()==L_MOUSE);
1484         case RETKEY:
1485         case PADENTER:
1486                 sculptmode_propset_end(sculpt_session(), 0);
1487                 break;
1488         default:
1489                 break;
1490         };
1491
1492         if(valset) {
1493                 if(pd->mode == PropsetSize) {
1494                         if(brush->size<1) brush->size= 1;
1495                         if(brush->size>200) brush->size= 200;
1496                 }
1497                 else if(pd->mode == PropsetStrength) {
1498                         if(brush->strength > 100) brush->strength= 100;
1499                         sculptmode_propset_calctex();
1500                 }
1501                 else if(pd->mode == PropsetTexRot) {
1502                         if(tex_angle() < 0)
1503                                 set_tex_angle(0);
1504                         else if(tex_angle() > 360)
1505                                 set_tex_angle(360);
1506                 }
1507         }
1508
1509         sculptmode_propset_header();
1510 }
1511
1512 void sculptmode_selectbrush_menu(void)
1513 {
1514         SculptData *sd= sculpt_data();
1515         int val;
1516
1517         pupmenu_set_active(sd->brush_type);
1518
1519         val= pupmenu("Select Brush%t|Draw|Smooth|Pinch|Inflate|Grab|Layer|Flatten");
1520
1521         if(val>0) {
1522                 sd->brush_type= val;
1523
1524                 allqueue(REDRAWVIEW3D, 1);
1525                 allqueue(REDRAWBUTSEDIT, 1);
1526         }
1527 }
1528
1529 void sculptmode_update_all_projverts(float *vertcosnos)
1530 {
1531         Mesh *me= get_mesh(OBACT);
1532         unsigned i;
1533
1534         if(projverts) MEM_freeN(projverts);
1535         projverts= MEM_mallocN(sizeof(ProjVert)*me->totvert,"ProjVerts");
1536         for(i=0; i<me->totvert; ++i) {
1537                 project(vertcosnos ? &vertcosnos[i * 6] : me->mvert[i].co, projverts[i].co);
1538                 projverts[i].inside= 0;
1539         }
1540 }
1541
1542 void sculptmode_draw_wires(int only_damaged, Mesh *me)
1543 {
1544         int i;
1545
1546         bglPolygonOffset(1.0);
1547         glDepthMask(0);
1548         BIF_ThemeColor((OBACT==OBACT)?TH_ACTIVE:TH_SELECT);
1549
1550         for(i=0; i<me->totedge; i++) {
1551                 MEdge *med= &me->medge[i];
1552
1553                 if((!only_damaged || (projverts[med->v1].inside || projverts[med->v2].inside)) &&
1554                    (med->flag & ME_EDGEDRAW)) {
1555                         glDrawElements(GL_LINES, 2, GL_UNSIGNED_INT, &med->v1);
1556                 }
1557         }
1558
1559         glDepthMask(1);
1560         bglPolygonOffset(0.0);
1561 }
1562
1563 void sculptmode_draw_mesh(int only_damaged)
1564 {
1565         Mesh *me= get_mesh(OBACT);
1566         int i, j, dt, drawCurrentMat = 1, matnr= -1;
1567
1568         persp(PERSP_VIEW);
1569         mymultmatrix(OBACT->obmat);
1570         glEnable(GL_DEPTH_TEST);
1571         glEnable(GL_LIGHTING);
1572         init_gl_materials(OBACT, 0);
1573         glEnable(GL_CULL_FACE);
1574
1575         glShadeModel(GL_SMOOTH);
1576
1577         glVertexPointer(3, GL_FLOAT, sizeof(MVert), &me->mvert[0].co);
1578         glNormalPointer(GL_SHORT, sizeof(MVert), &me->mvert[0].no);
1579
1580         dt= MIN2(G.vd->drawtype, OBACT->dt);
1581         if(dt==OB_WIRE)
1582                 glColorMask(0,0,0,0);
1583
1584         for(i=0; i<me->totface; ++i) {
1585                 MFace *f= &me->mface[i];
1586                 char inside= 0;
1587                 int new_matnr= f->mat_nr + 1;
1588
1589                 if(new_matnr != matnr)
1590                         drawCurrentMat= set_gl_material(matnr = new_matnr);
1591
1592                 /* If only_damaged!=0, only draw faces that are partially
1593                    inside the area(s) modified by the brush */
1594                 if(only_damaged) {
1595                         for(j=0; j<(f->v4?4:3); ++j) {
1596                                 if(projverts[*((&f->v1)+j)].inside) {
1597                                         inside= 1;
1598                                         break;
1599                                 }
1600                         }
1601                 }
1602                 else
1603                         inside= 1;
1604
1605                 if(inside && drawCurrentMat)
1606                         glDrawElements(f->v4?GL_QUADS:GL_TRIANGLES, f->v4?4:3, GL_UNSIGNED_INT, &f->v1);
1607         }
1608
1609         glDisable(GL_CULL_FACE);
1610         glDisable(GL_LIGHTING);
1611         glColorMask(1,1,1,1);
1612
1613         if(dt==OB_WIRE || (OBACT->dtx & OB_DRAWWIRE))
1614                 sculptmode_draw_wires(only_damaged, me);
1615
1616         glDisable(GL_DEPTH_TEST);
1617 }
1618
1619 void sculptmode_correct_state(void)
1620 {
1621         if(!sculpt_session())
1622                 sculpt_init_session();
1623
1624         glEnableClientState(GL_VERTEX_ARRAY);
1625         glEnableClientState(GL_NORMAL_ARRAY);
1626
1627         if(!sculpt_session()->vertex_users) calc_vertex_users();
1628 }
1629
1630 /* Checks whether full update mode (slower) needs to be used to work with modifiers */
1631 char sculpt_modifiers_active(Object *ob)
1632 {
1633         ModifierData *md;
1634
1635         for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
1636                 if(md->mode & eModifierMode_Realtime)
1637                         return 1;
1638         }
1639
1640         return 0;
1641 }
1642
1643 void sculpt(void)
1644 {
1645         SculptData *sd= sculpt_data();
1646         SculptSession *ss= sculpt_session();
1647         Object *ob= OBACT;
1648         short mouse[2], mvalo[2], firsttime=1, mousebut;
1649         short modifier_calculations= 0;
1650         EditData e;
1651         RectNode *rn= NULL;
1652         short spacing= 32000;
1653         int scissor_box[4];
1654
1655         if(!(G.f & G_SCULPTMODE) || G.obedit || !ob || ob->id.lib || !get_mesh(ob) || (get_mesh(ob)->totface == 0))
1656                 return;
1657         if(!(ob->lay & G.vd->lay))
1658                 error("Active object is not in this layer");
1659         if(ob_get_keyblock(ob)) {
1660                 if(!(ob->shapeflag & OB_SHAPE_LOCK)) {
1661                         error("Cannot sculpt on unlocked shape key");
1662                         return;
1663                 }
1664         }
1665
1666         if(!ss) {
1667                 sculpt_init_session();
1668                 ss= sd->session;
1669         }
1670
1671         if(sd->flags & SCULPT_INPUT_SMOOTH)
1672                 sculpt_stroke_new(256);
1673
1674         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
1675         ss->damaged_verts.first = ss->damaged_verts.last = NULL;
1676         ss->vertexcosnos = NULL;
1677
1678         /* Check that vertex users are up-to-date */
1679         if(ob != active_ob || ss->vertex_users_size != get_mesh(ob)->totvert) {
1680                 sculptmode_free_vertexusers(ss);
1681                 calc_vertex_users();
1682                 active_ob= ob;
1683         }
1684
1685         glEnableClientState(GL_VERTEX_ARRAY);
1686         glEnableClientState(GL_NORMAL_ARRAY);
1687
1688         persp(PERSP_VIEW);
1689
1690         getmouseco_areawin(mvalo);
1691
1692         /* Make sure sculptdata has been init'd properly */
1693         if(!ss->vertex_users) calc_vertex_users();
1694
1695         /* Init texture
1696            FIXME: Shouldn't be doing this every time! */
1697         if(sd->texrept!=SCULPTREPT_3D)
1698                 sculptmode_update_tex();
1699
1700         getmouseco_areawin(mouse);
1701         mvalo[0]= mouse[0];
1702         mvalo[1]= mouse[1];
1703
1704         mousebut = L_MOUSE;
1705
1706         /* If modifier_calculations is true, then extra time must be spent
1707            updating the mesh. This takes a *lot* longer, so it's worth
1708            skipping if the modifier stack is empty. */
1709         modifier_calculations= sculpt_modifiers_active(ob);
1710
1711         init_sculptmatrices();
1712
1713         if(modifier_calculations)
1714                 ss->vertexcosnos= mesh_get_mapped_verts_nors(ob);
1715         sculptmode_update_all_projverts(ss->vertexcosnos);
1716
1717         e.grabdata= NULL;
1718         e.layer_disps= NULL;
1719         e.layer_store= NULL;
1720
1721         /* Set scaling adjustment */
1722         e.scale[0]= 1.0f / ob->size[0];
1723         e.scale[1]= 1.0f / ob->size[1];
1724         e.scale[2]= 1.0f / ob->size[2];
1725
1726         /* Capture original copy */
1727         if(sd->flags & SCULPT_DRAW_FAST)
1728                 glAccum(GL_LOAD, 1);
1729
1730         /* Get original scissor box */
1731         glGetIntegerv(GL_SCISSOR_BOX, scissor_box);
1732
1733         while (get_mbut() & mousebut) {
1734                 getmouseco_areawin(mouse);
1735
1736                 if(firsttime || mouse[0]!=mvalo[0] || mouse[1]!=mvalo[1] || sculptmode_brush()->airbrush) {
1737                         firsttime= 0;
1738
1739                         if(sd->flags & SCULPT_INPUT_SMOOTH)
1740                                 sculpt_stroke_add_point(mouse[0], mouse[1]);
1741
1742                         spacing+= sqrt(pow(mvalo[0]-mouse[0],2)+pow(mvalo[1]-mouse[1],2));
1743
1744                         if(modifier_calculations && !ss->vertexcosnos)
1745                                 ss->vertexcosnos= mesh_get_mapped_verts_nors(ob);
1746
1747                         if(G.scene->sculptdata.brush_type != GRAB_BRUSH) {
1748                                 if(sd->flags & SCULPT_INPUT_SMOOTH) {
1749                                         sculpt_stroke_apply(&e);
1750                                 }
1751                                 else if(sd->spacing==0 || spacing>sd->spacing) {
1752                                         do_symmetrical_brush_actions(&e, mouse, NULL);
1753                                         spacing= 0;
1754                                 }
1755                         }
1756                         else {
1757                                 do_symmetrical_brush_actions(&e, mouse, mvalo);
1758                                 ss->pivot= unproject(mouse[0],mouse[1],e.grabdata->depth);
1759                         }
1760
1761                         if(modifier_calculations || ob_get_keyblock(ob))
1762                                 DAG_object_flush_update(G.scene, ob, OB_RECALC_DATA);
1763
1764                         if(modifier_calculations || sd->brush_type == GRAB_BRUSH || !(sd->flags & SCULPT_DRAW_FAST)) {
1765                                 calc_damaged_verts(&ss->damaged_verts,e.grabdata);
1766                                 scrarea_do_windraw(curarea);
1767                                 screen_swapbuffers();
1768                         } else { /* Optimized drawing */
1769                                 calc_damaged_verts(&ss->damaged_verts,e.grabdata);
1770
1771                                 /* Draw the stored image to the screen */
1772                                 glAccum(GL_RETURN, 1);
1773
1774                                 /* Clear each of the area(s) modified by the brush */
1775                                 for(rn=ss->damaged_rects.first; rn; rn= rn->next) {
1776                                         rcti clp= rn->r;
1777                                         rcti *win= &curarea->winrct;
1778
1779                                         clp.xmin+= win->xmin;
1780                                         clp.xmax+= win->xmin;
1781                                         clp.ymin+= win->ymin;
1782                                         clp.ymax+= win->ymin;
1783
1784                                         if(clp.xmin<win->xmax && clp.xmax>win->xmin &&
1785                                            clp.ymin<win->ymax && clp.ymax>win->ymin) {
1786                                                 if(clp.xmin<win->xmin) clp.xmin= win->xmin;
1787                                                 if(clp.ymin<win->ymin) clp.ymin= win->ymin;
1788                                                 if(clp.xmax>win->xmax) clp.xmax= win->xmax;
1789                                                 if(clp.ymax>win->ymax) clp.ymax= win->ymax;
1790                                                 glScissor(clp.xmin+1, clp.ymin+1,
1791                                                           clp.xmax-clp.xmin-2,clp.ymax-clp.ymin-2);
1792                                         }
1793
1794                                         glClear(GL_DEPTH_BUFFER_BIT);
1795                                 }
1796
1797                                 /* Draw all the polygons that are inside the modified area(s) */
1798                                 glScissor(scissor_box[0], scissor_box[1], scissor_box[2], scissor_box[3]);
1799                                 sculptmode_draw_mesh(1);
1800                                 glAccum(GL_LOAD, 1);
1801
1802                                 projverts_clear_inside();
1803
1804                                 persp(PERSP_WIN);
1805                                 glDisable(GL_DEPTH_TEST);
1806
1807                                 /* Draw cursor */
1808                                 if(sculpt_data()->flags & SCULPT_DRAW_BRUSH)
1809                                         fdrawXORcirc((float)mouse[0],(float)mouse[1],sculptmode_brush()->size);
1810                                 if(sculpt_data()->flags & SCULPT_INPUT_SMOOTH)
1811                                         sculpt_stroke_draw();
1812
1813                                 myswapbuffers();
1814                         }
1815
1816                         BLI_freelistN(&ss->damaged_rects);
1817                         ss->damaged_rects.first = ss->damaged_rects.last = NULL;
1818
1819                         mvalo[0]= mouse[0];
1820                         mvalo[1]= mouse[1];
1821
1822                         if(ss->vertexcosnos) {
1823                                 MEM_freeN(ss->vertexcosnos);
1824                                 ss->vertexcosnos= NULL;
1825                         }
1826
1827                 }
1828                 else BIF_wait_for_statechange();
1829         }
1830
1831         if(sd->flags & SCULPT_INPUT_SMOOTH) {
1832                 sculpt_stroke_apply_all(&e);
1833                 calc_damaged_verts(&ss->damaged_verts,e.grabdata);
1834                 BLI_freelistN(&ss->damaged_rects);
1835         }
1836
1837         if(projverts) MEM_freeN(projverts);
1838         projverts= NULL;
1839         if(e.layer_disps) MEM_freeN(e.layer_disps);
1840         if(e.layer_store) MEM_freeN(e.layer_store);
1841         /* Free GrabData */
1842         if(e.grabdata) {
1843                 int i;
1844                 for(i=0; i<8; ++i)
1845                         BLI_freelistN(&e.grabdata->active_verts[i]);
1846                 MEM_freeN(e.grabdata);
1847         }
1848         sculpt_stroke_free();
1849
1850         switch(G.scene->sculptdata.brush_type) {
1851         case DRAW_BRUSH:
1852                 BIF_undo_push("Draw Brush"); break;
1853         case SMOOTH_BRUSH:
1854                 BIF_undo_push("Smooth Brush"); break;
1855         case PINCH_BRUSH:
1856                 BIF_undo_push("Pinch Brush"); break;
1857         case INFLATE_BRUSH:
1858                 BIF_undo_push("Inflate Brush"); break;
1859         case GRAB_BRUSH:
1860                 BIF_undo_push("Grab Brush"); break;
1861         case LAYER_BRUSH:
1862                 BIF_undo_push("Layer Brush"); break;
1863         case FLATTEN_BRUSH:
1864                 BIF_undo_push("Flatten Brush"); break;
1865         default:
1866                 BIF_undo_push("Sculpting"); break;
1867         }
1868
1869         if(G.vd->depths) G.vd->depths->damaged= 1;
1870
1871         allqueue(REDRAWVIEW3D, 0);
1872 }
1873
1874 void set_sculptmode(void)
1875 {
1876         if(G.f & G_SCULPTMODE) {
1877                 Mesh *me= get_mesh(OBACT);
1878
1879                 G.f &= ~G_SCULPTMODE;
1880
1881                 sculptmode_free_session(G.scene);
1882                 if(me && me->pv)
1883                         sculptmode_pmv_off(me);
1884         }
1885         else {
1886                 G.f |= G_SCULPTMODE;
1887
1888                 /* Called here to sanity-check the brush */
1889                 sculptmode_brush();
1890
1891                 sculpt_init_session();
1892
1893                 glEnableClientState(GL_VERTEX_ARRAY);
1894                 glEnableClientState(GL_NORMAL_ARRAY);
1895         }
1896
1897         active_ob= NULL;
1898
1899         allqueue(REDRAWVIEW3D, 1);
1900         allqueue(REDRAWBUTSEDIT, 0);
1901 }
1902
1903 /* Partial Mesh Visibility */
1904 PartialVisibility *sculptmode_copy_pmv(PartialVisibility *pmv)
1905 {
1906         PartialVisibility *n= MEM_dupallocN(pmv);
1907         n->vert_map= MEM_dupallocN(pmv->vert_map);
1908         n->edge_map= MEM_dupallocN(pmv->edge_map);
1909         n->old_edges= MEM_dupallocN(pmv->old_edges);
1910         n->old_faces= MEM_dupallocN(pmv->old_faces);
1911         return n;
1912 }
1913
1914 void sculptmode_pmv_free(PartialVisibility *pv)
1915 {
1916         MEM_freeN(pv->vert_map);
1917         MEM_freeN(pv->edge_map);
1918         MEM_freeN(pv->old_faces);
1919         MEM_freeN(pv->old_edges);
1920         MEM_freeN(pv);
1921 }
1922
1923 void sculptmode_revert_pmv(Mesh *me)
1924 {
1925         if(me->pv) {
1926                 unsigned i;
1927                 MVert *nve, *old_verts;
1928
1929                 active_ob= NULL;
1930
1931                 /* Reorder vertices */
1932                 nve= me->mvert;
1933                 old_verts = MEM_mallocN(sizeof(MVert)*me->pv->totvert,"PMV revert verts");
1934                 for(i=0; i<me->pv->totvert; ++i)
1935                         old_verts[i]= nve[me->pv->vert_map[i]];
1936
1937                 /* Restore verts, edges and faces */
1938                 CustomData_free_layer_active(&me->vdata, CD_MVERT, me->totvert);
1939                 CustomData_free_layer_active(&me->edata, CD_MEDGE, me->totedge);
1940                 CustomData_free_layer_active(&me->fdata, CD_MFACE, me->totface);
1941
1942                 CustomData_add_layer(&me->vdata, CD_MVERT, CD_ASSIGN, old_verts, me->pv->totvert);
1943                 CustomData_add_layer(&me->edata, CD_MEDGE, CD_ASSIGN, me->pv->old_edges, me->pv->totedge);
1944                 CustomData_add_layer(&me->fdata, CD_MFACE, CD_ASSIGN, me->pv->old_faces, me->pv->totface);
1945                 mesh_update_customdata_pointers(me);
1946
1947                 me->totvert= me->pv->totvert;
1948                 me->totedge= me->pv->totedge;
1949                 me->totface= me->pv->totface;
1950
1951                 me->pv->old_edges= NULL;
1952                 me->pv->old_faces= NULL;
1953
1954                 /* Free maps */
1955                 MEM_freeN(me->pv->edge_map);
1956                 me->pv->edge_map= NULL;
1957                 MEM_freeN(me->pv->vert_map);
1958                 me->pv->vert_map= NULL;
1959
1960                 DAG_object_flush_update(G.scene, OBACT, OB_RECALC_DATA);
1961         }
1962 }
1963
1964 void sculptmode_pmv_off(Mesh *me)
1965 {
1966         if(me->pv) {
1967                 sculptmode_revert_pmv(me);
1968                 MEM_freeN(me->pv);
1969                 me->pv= NULL;
1970         }
1971 }
1972
1973 /* mode: 0=hide outside selection, 1=hide inside selection */
1974 void sculptmode_do_pmv(Object *ob, rcti *hb_2d, int mode)
1975 {
1976         Mesh *me= get_mesh(ob);
1977         vec3f hidebox[6];
1978         vec3f plane_normals[4];
1979         float plane_ds[4];
1980         unsigned i, j;
1981         unsigned ndx_show, ndx_hide;
1982         MVert *nve;
1983         unsigned face_cnt_show= 0, face_ndx_show= 0;
1984         unsigned edge_cnt_show= 0, edge_ndx_show= 0;
1985         unsigned *old_map= NULL;
1986         const unsigned SHOW= 0, HIDE=1;
1987
1988         /* Convert hide box from 2D to 3D */
1989         hidebox[0]= unproject(hb_2d->xmin, hb_2d->ymax, 1);
1990         hidebox[1]= unproject(hb_2d->xmax, hb_2d->ymax, 1);
1991         hidebox[2]= unproject(hb_2d->xmax, hb_2d->ymin, 1);
1992         hidebox[3]= unproject(hb_2d->xmin, hb_2d->ymin, 1);
1993         hidebox[4]= unproject(hb_2d->xmin, hb_2d->ymax, 0);
1994         hidebox[5]= unproject(hb_2d->xmax, hb_2d->ymin, 0);
1995
1996         /* Calculate normals for each side of hide box */
1997         CalcNormFloat(&hidebox[0].x,&hidebox[1].x,&hidebox[4].x,&plane_normals[0].x);
1998         CalcNormFloat(&hidebox[1].x,&hidebox[2].x,&hidebox[5].x,&plane_normals[1].x);
1999         CalcNormFloat(&hidebox[2].x,&hidebox[3].x,&hidebox[5].x,&plane_normals[2].x);
2000         CalcNormFloat(&hidebox[3].x,&hidebox[0].x,&hidebox[4].x,&plane_normals[3].x);
2001
2002         /* Calculate D for each side of hide box */
2003         for(i= 0; i<4; ++i)
2004                 plane_ds[i]= hidebox[i].x*plane_normals[i].x + hidebox[i].y*plane_normals[i].y +
2005                         hidebox[i].z*plane_normals[i].z;
2006
2007         /* Add partial visibility to mesh */
2008         if(!me->pv) {
2009                 me->pv= MEM_callocN(sizeof(PartialVisibility),"PartialVisibility");
2010         } else {
2011                 old_map= MEM_callocN(sizeof(unsigned)*me->pv->totvert,"PMV oldmap");
2012                 for(i=0; i<me->pv->totvert; ++i) {
2013                         old_map[i]= me->pv->vert_map[i]<me->totvert?0:1;
2014                 }
2015                 sculptmode_revert_pmv(me);
2016         }
2017
2018         /* Kill sculpt data */
2019         active_ob= NULL;
2020
2021         /* Initalize map with which verts are to be hidden */
2022         me->pv->vert_map= MEM_mallocN(sizeof(unsigned)*me->totvert, "PMV vertmap");
2023         me->pv->totvert= me->totvert;
2024         me->totvert= 0;
2025         for(i=0; i<me->pv->totvert; ++i) {
2026                 me->pv->vert_map[i]= mode ? HIDE:SHOW;
2027                 for(j=0; j<4; ++j) {
2028                         if(me->mvert[i].co[0] * plane_normals[j].x +
2029                            me->mvert[i].co[1] * plane_normals[j].y +
2030                            me->mvert[i].co[2] * plane_normals[j].z < plane_ds[j] ) {
2031                                 me->pv->vert_map[i]= mode ? SHOW:HIDE; /* Vert is outside the hide box */
2032                                 break;
2033                         }
2034                 }
2035                 if(old_map && old_map[i]) me->pv->vert_map[i]= 1;
2036                 if(!me->pv->vert_map[i]) ++me->totvert;
2037
2038         }
2039         if(old_map) MEM_freeN(old_map);
2040
2041         /* Find out how many faces to show */
2042         for(i=0; i<me->totface; ++i) {
2043                 if(!me->pv->vert_map[me->mface[i].v1] &&
2044                    !me->pv->vert_map[me->mface[i].v2] &&
2045                    !me->pv->vert_map[me->mface[i].v3]) {
2046                         if(me->mface[i].v4) {
2047                                 if(!me->pv->vert_map[me->mface[i].v4])
2048                                         ++face_cnt_show;
2049                         }
2050                         else ++face_cnt_show;
2051                 }
2052         }
2053         /* Find out how many edges to show */
2054         for(i=0; i<me->totedge; ++i) {
2055                 if(!me->pv->vert_map[me->medge[i].v1] &&
2056                    !me->pv->vert_map[me->medge[i].v2])
2057                         ++edge_cnt_show;
2058         }
2059
2060         /* Create new vert array and reset each vert's map with map[old]=new index */
2061         nve= MEM_mallocN(sizeof(MVert)*me->pv->totvert, "PMV verts");
2062         ndx_show= 0; ndx_hide= me->totvert;
2063         for(i=0; i<me->pv->totvert; ++i) {
2064                 if(me->pv->vert_map[i]) {
2065                         me->pv->vert_map[i]= ndx_hide;
2066                         nve[me->pv->vert_map[i]]= me->mvert[i];
2067                         ++ndx_hide;
2068                 } else {
2069                         me->pv->vert_map[i]= ndx_show;
2070                         nve[me->pv->vert_map[i]]= me->mvert[i];
2071                         ++ndx_show;
2072                 }
2073         }
2074         CustomData_free_layer_active(&me->vdata, CD_MVERT, me->pv->totvert);
2075         me->mvert= CustomData_add_layer(&me->vdata, CD_MVERT, CD_ASSIGN, nve, me->totvert);
2076
2077         /* Create new face array */
2078         me->pv->old_faces= me->mface;
2079         me->pv->totface= me->totface;
2080         me->mface= MEM_mallocN(sizeof(MFace)*face_cnt_show, "PMV faces");
2081         for(i=0; i<me->totface; ++i) {
2082                 MFace *pr_f= &me->pv->old_faces[i];
2083                 char show= 0;
2084
2085                 if(me->pv->vert_map[pr_f->v1] < me->totvert &&
2086                    me->pv->vert_map[pr_f->v2] < me->totvert &&
2087                    me->pv->vert_map[pr_f->v3] < me->totvert) {
2088                         if(pr_f->v4) {
2089                                 if(me->pv->vert_map[pr_f->v4] < me->totvert)
2090                                         show= 1;
2091                         }
2092                         else show= 1;
2093                 }
2094
2095                 if(show) {
2096                         MFace *cr_f= &me->mface[face_ndx_show];
2097                         *cr_f= *pr_f;
2098                         cr_f->v1= me->pv->vert_map[pr_f->v1];
2099                         cr_f->v2= me->pv->vert_map[pr_f->v2];
2100                         cr_f->v3= me->pv->vert_map[pr_f->v3];
2101                         cr_f->v4= pr_f->v4 ? me->pv->vert_map[pr_f->v4] : 0;
2102                         test_index_face(cr_f,NULL,0,pr_f->v4?4:3);
2103                         ++face_ndx_show;
2104                 }
2105         }
2106         me->totface= face_cnt_show;
2107         CustomData_set_layer(&me->fdata, CD_MFACE, me->mface);
2108
2109         /* Create new edge array */
2110         me->pv->old_edges= me->medge;
2111         me->pv->totedge= me->totedge;
2112         me->medge= MEM_mallocN(sizeof(MEdge)*edge_cnt_show, "PMV edges");
2113         me->pv->edge_map= MEM_mallocN(sizeof(int)*me->pv->totedge,"PMV edgemap");
2114         for(i=0; i<me->totedge; ++i) {
2115                 if(me->pv->vert_map[me->pv->old_edges[i].v1] < me->totvert &&
2116                    me->pv->vert_map[me->pv->old_edges[i].v2] < me->totvert) {
2117                         MEdge *cr_e= &me->medge[edge_ndx_show];
2118                         me->pv->edge_map[i]= edge_ndx_show;
2119                         *cr_e= me->pv->old_edges[i];
2120                         cr_e->v1= me->pv->vert_map[me->pv->old_edges[i].v1];
2121                         cr_e->v2= me->pv->vert_map[me->pv->old_edges[i].v2];
2122                         ++edge_ndx_show;
2123                 }
2124                 else me->pv->edge_map[i]= -1;
2125         }
2126         me->totedge= edge_cnt_show;
2127         CustomData_set_layer(&me->edata, CD_MEDGE, me->medge);
2128
2129         DAG_object_flush_update(G.scene, OBACT, OB_RECALC_DATA);
2130 }
2131
2132 rcti sculptmode_pmv_box()
2133 {
2134         short down[2], mouse[2];
2135         rcti ret;
2136
2137         getmouseco_areawin(down);
2138
2139         while((get_mbut()&L_MOUSE) || (get_mbut()&R_MOUSE)) {
2140                 getmouseco_areawin(mouse);
2141
2142                 scrarea_do_windraw(curarea);
2143
2144                 persp(PERSP_WIN);
2145                 glLineWidth(2);
2146                 setlinestyle(2);
2147                 sdrawXORline(down[0],down[1],mouse[0],down[1]);
2148                 sdrawXORline(mouse[0],down[1],mouse[0],mouse[1]);
2149                 sdrawXORline(mouse[0],mouse[1],down[0],mouse[1]);
2150                 sdrawXORline(down[0],mouse[1],down[0],down[1]);
2151                 setlinestyle(0);
2152                 glLineWidth(1);
2153                 persp(PERSP_VIEW);
2154
2155                 screen_swapbuffers();
2156                 backdrawview3d(0);
2157         }
2158
2159         ret.xmin= down[0]<mouse[0]?down[0]:mouse[0];
2160         ret.ymin= down[1]<mouse[1]?down[1]:mouse[1];
2161         ret.xmax= down[0]>mouse[0]?down[0]:mouse[0];
2162         ret.ymax= down[1]>mouse[1]?down[1]:mouse[1];
2163         return ret;
2164 }
2165
2166 void sculptmode_pmv(int mode)
2167 {
2168         Object *ob= OBACT;
2169         rcti hb_2d;
2170
2171         if(ob_get_key(ob)) {
2172                 error("Cannot hide mesh with shape keys enabled");
2173                 return;
2174         }
2175
2176         hb_2d= sculptmode_pmv_box(); /* Get 2D hide box */
2177
2178         sculptmode_correct_state();
2179
2180         waitcursor(1);
2181
2182         if(hb_2d.xmax-hb_2d.xmin > 3 && hb_2d.ymax-hb_2d.ymin > 3) {
2183                 init_sculptmatrices();
2184
2185                 sculptmode_do_pmv(ob,&hb_2d,mode);
2186         }
2187         else sculptmode_pmv_off(get_mesh(ob));
2188
2189         scrarea_do_windraw(curarea);
2190
2191         BIF_undo_push("Partial mesh hide");
2192
2193         waitcursor(0);
2194 }