source/blender/editors/sculpt_paint/paint_vertex.c

   1 /**
   2  * $Id$
   3  *
   4  * ***** BEGIN GPL LICENSE BLOCK *****
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version 2
   9  * of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19  *
  20  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
  21  * All rights reserved.
  22  *
  23  * The Original Code is: all of this file.
  24  *
  25  * Contributor(s): none yet.
  26  *
  27  * ***** END GPL LICENSE BLOCK *****
  28  */
  29
  30 #include <math.h>
  31 #include <string.h>
  32
  33 #ifdef WIN32
  34 #include <io.h>
  35 #else
  36 #include <unistd.h>
  37 #endif
  38
  39 #include "MEM_guardedalloc.h"
  40
  41 #include "IMB_imbuf.h"
  42 #include "IMB_imbuf_types.h"
  43
  44 #include "BLI_blenlib.h"
  45 #include "BLI_arithb.h"
  46 #include "MTC_matrixops.h"
  47
  48 #include "DNA_action_types.h"
  49 #include "DNA_armature_types.h"
  50 #include "DNA_brush_types.h"
  51 #include "DNA_cloth_types.h"
  52 #include "DNA_mesh_types.h"
  53 #include "DNA_meshdata_types.h"
  54 #include "DNA_modifier_types.h"
  55 #include "DNA_object_types.h"
  56 #include "DNA_object_force.h"
  57 #include "DNA_particle_types.h"
  58 #include "DNA_screen_types.h"
  59 #include "DNA_space_types.h"
  60 #include "DNA_scene_types.h"
  61 #include "DNA_view3d_types.h"
  62 #include "DNA_userdef_types.h"
  63
  64 #include "RNA_access.h"
  65
  66 #include "BKE_armature.h"
  67 #include "BKE_DerivedMesh.h"
  68 #include "BKE_cloth.h"
  69 #include "BKE_context.h"
  70 #include "BKE_customdata.h"
  71 #include "BKE_depsgraph.h"
  72 #include "BKE_deform.h"
  73 #include "BKE_displist.h"
  74 #include "BKE_global.h"
  75 #include "BKE_mesh.h"
  76 #include "BKE_modifier.h"
  77 #include "BKE_multires.h"
  78 #include "BKE_object.h"
  79 #include "BKE_utildefines.h"
  80
  81 #include "WM_api.h"
  82 #include "WM_types.h"
  83
  84 #include "BIF_gl.h"
  85 #include "BIF_glutil.h"
  86
  87 #include "ED_mesh.h"
  88 #include "ED_object.h"
  89 #include "ED_screen.h"
  90 #include "ED_util.h"
  91 #include "ED_view3d.h"
  92
  93         /* vp->mode */
  94 #define VP_MIX  0
  95 #define VP_ADD  1
  96 #define VP_SUB  2
  97 #define VP_MUL  3
  98 #define VP_BLUR 4
  99 #define VP_LIGHTEN      5
 100 #define VP_DARKEN       6
 101
 102 #define MAXINDEX        512000
 103
 104 /* XXX */
 105 static void error() {}
 106
 107 /* polling - retrieve whether cursor should be set or operator should be done */
 108
 109 static int vp_poll(bContext *C)
 110 {
 111         if(G.f & G_VERTEXPAINT) {
 112                 ScrArea *sa= CTX_wm_area(C);
 113                 if(sa->spacetype==SPACE_VIEW3D) {
 114                         ARegion *ar= CTX_wm_region(C);
 115                         if(ar->regiontype==RGN_TYPE_WINDOW)
 116                                 return 1;
 117                 }
 118         }
 119         return 0;
 120 }
 121
 122 static int wp_poll(bContext *C)
 123 {
 124         if(G.f & G_WEIGHTPAINT) {
 125                 ScrArea *sa= CTX_wm_area(C);
 126                 if(sa->spacetype==SPACE_VIEW3D) {
 127                         ARegion *ar= CTX_wm_region(C);
 128                         if(ar->regiontype==RGN_TYPE_WINDOW)
 129                                 return 1;
 130                 }
 131         }
 132         return 0;
 133 }
 134
 135
 136 /* Cursors */
 137 static void vp_drawcursor(bContext *C, int x, int y, void *customdata)
 138 {
 139         ToolSettings *ts= CTX_data_tool_settings(C);
 140
 141         glTranslatef((float)x, (float)y, 0.0f);
 142
 143         glColor4ub(255, 255, 255, 128);
 144         glEnable( GL_LINE_SMOOTH );
 145         glEnable(GL_BLEND);
 146         glutil_draw_lined_arc(0.0, M_PI*2.0, ts->vpaint->size, 40);
 147         glDisable(GL_BLEND);
 148         glDisable( GL_LINE_SMOOTH );
 149
 150         glTranslatef((float)-x, (float)-y, 0.0f);
 151 }
 152
 153 static void wp_drawcursor(bContext *C, int x, int y, void *customdata)
 154 {
 155         ToolSettings *ts= CTX_data_tool_settings(C);
 156
 157         glTranslatef((float)x, (float)y, 0.0f);
 158
 159         glColor4ub(200, 200, 255, 128);
 160         glEnable( GL_LINE_SMOOTH );
 161         glEnable(GL_BLEND);
 162         glutil_draw_lined_arc(0.0, M_PI*2.0, ts->wpaint->size, 40);
 163         glDisable(GL_BLEND);
 164         glDisable( GL_LINE_SMOOTH );
 165
 166         glTranslatef((float)-x, (float)-y, 0.0f);
 167 }
 168
 169 static void toggle_paint_cursor(bContext *C, int wpaint)
 170 {
 171         ToolSettings *ts = CTX_data_scene(C)->toolsettings;
 172         VPaint *vp = wpaint ? ts->wpaint : ts->vpaint;
 173
 174         if(vp->paintcursor) {
 175                 WM_paint_cursor_end(CTX_wm_manager(C), vp->paintcursor);
 176                 vp->paintcursor = NULL;
 177         }
 178         else {
 179                 vp->paintcursor = wpaint ?
 180                         WM_paint_cursor_activate(CTX_wm_manager(C), wp_poll, wp_drawcursor, NULL) :
 181                         WM_paint_cursor_activate(CTX_wm_manager(C), vp_poll, vp_drawcursor, NULL);
 182         }
 183 }
 184
 185 static VPaint *new_vpaint(int wpaint)
 186 {
 187         VPaint *vp= MEM_callocN(sizeof(VPaint), "VPaint");
 188
 189         vp->r= 1.0f;
 190         vp->g= 1.0f;
 191         vp->b= 1.0f;
 192         vp->a= 0.2f;
 193         vp->size= 25.0f;
 194         vp->gamma= vp->mul= 1.0f;
 195
 196         vp->flag= VP_AREA+VP_SOFT+VP_SPRAY;
 197
 198         if(wpaint) {
 199                 vp->weight= 1.0f;
 200                 vp->a= 1.0f;
 201                 vp->flag= VP_AREA+VP_SOFT;
 202         }
 203         return vp;
 204 }
 205
 206 static int *get_indexarray(void)
 207 {
 208         return MEM_mallocN(sizeof(int)*MAXINDEX + 2, "vertexpaint");
 209 }
 210
 211
 212 /* in contradiction to cpack drawing colors, the MCOL colors (vpaint colors) are per byte!
 213    so not endian sensitive. Mcol = ABGR!!! so be cautious with cpack calls */
 214
 215 unsigned int rgba_to_mcol(float r, float g, float b, float a)
 216 {
 217         int ir, ig, ib, ia;
 218         unsigned int col;
 219         char *cp;
 220
 221         ir= floor(255.0*r);
 222         if(ir<0) ir= 0; else if(ir>255) ir= 255;
 223         ig= floor(255.0*g);
 224         if(ig<0) ig= 0; else if(ig>255) ig= 255;
 225         ib= floor(255.0*b);
 226         if(ib<0) ib= 0; else if(ib>255) ib= 255;
 227         ia= floor(255.0*a);
 228         if(ia<0) ia= 0; else if(ia>255) ia= 255;
 229
 230         cp= (char *)&col;
 231         cp[0]= ia;
 232         cp[1]= ib;
 233         cp[2]= ig;
 234         cp[3]= ir;
 235
 236         return col;
 237
 238 }
 239
 240 static unsigned int vpaint_get_current_col(VPaint *vp)
 241 {
 242         return rgba_to_mcol(vp->r, vp->g, vp->b, 1.0f);
 243 }
 244
 245 void do_shared_vertexcol(Mesh *me)
 246 {
 247         /* if no mcol: do not do */
 248         /* if tface: only the involved faces, otherwise all */
 249         MFace *mface;
 250         MTFace *tface;
 251         int a;
 252         short *scolmain, *scol;
 253         char *mcol;
 254
 255         if(me->mcol==0 || me->totvert==0 || me->totface==0) return;
 256
 257         scolmain= MEM_callocN(4*sizeof(short)*me->totvert, "colmain");
 258
 259         tface= me->mtface;
 260         mface= me->mface;
 261         mcol= (char *)me->mcol;
 262         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
 263                 if((tface && tface->mode & TF_SHAREDCOL) || (G.f & G_FACESELECT)==0) {
 264                         scol= scolmain+4*mface->v1;
 265                         scol[0]++; scol[1]+= mcol[1]; scol[2]+= mcol[2]; scol[3]+= mcol[3];
 266                         scol= scolmain+4*mface->v2;
 267                         scol[0]++; scol[1]+= mcol[5]; scol[2]+= mcol[6]; scol[3]+= mcol[7];
 268                         scol= scolmain+4*mface->v3;
 269                         scol[0]++; scol[1]+= mcol[9]; scol[2]+= mcol[10]; scol[3]+= mcol[11];
 270                         if(mface->v4) {
 271                                 scol= scolmain+4*mface->v4;
 272                                 scol[0]++; scol[1]+= mcol[13]; scol[2]+= mcol[14]; scol[3]+= mcol[15];
 273                         }
 274                 }
 275                 if(tface) tface++;
 276         }
 277
 278         a= me->totvert;
 279         scol= scolmain;
 280         while(a--) {
 281                 if(scol[0]>1) {
 282                         scol[1]/= scol[0];
 283                         scol[2]/= scol[0];
 284                         scol[3]/= scol[0];
 285                 }
 286                 scol+= 4;
 287         }
 288
 289         tface= me->mtface;
 290         mface= me->mface;
 291         mcol= (char *)me->mcol;
 292         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
 293                 if((tface && tface->mode & TF_SHAREDCOL) || (G.f & G_FACESELECT)==0) {
 294                         scol= scolmain+4*mface->v1;
 295                         mcol[1]= scol[1]; mcol[2]= scol[2]; mcol[3]= scol[3];
 296                         scol= scolmain+4*mface->v2;
 297                         mcol[5]= scol[1]; mcol[6]= scol[2]; mcol[7]= scol[3];
 298                         scol= scolmain+4*mface->v3;
 299                         mcol[9]= scol[1]; mcol[10]= scol[2]; mcol[11]= scol[3];
 300                         if(mface->v4) {
 301                                 scol= scolmain+4*mface->v4;
 302                                 mcol[13]= scol[1]; mcol[14]= scol[2]; mcol[15]= scol[3];
 303                         }
 304                 }
 305                 if(tface) tface++;
 306         }
 307
 308         MEM_freeN(scolmain);
 309 }
 310
 311 void make_vertexcol(Scene *scene, int shade)    /* single ob */
 312 {
 313         Object *ob;
 314         Mesh *me;
 315
 316         if(scene->obedit) {
 317                 error("Unable to perform function in Edit Mode");
 318                 return;
 319         }
 320
 321         ob= OBACT;
 322         if(!ob || ob->id.lib) return;
 323         me= get_mesh(ob);
 324         if(me==0) return;
 325
 326         /* copies from shadedisplist to mcol */
 327         if(!me->mcol) {
 328                 CustomData_add_layer(&me->fdata, CD_MCOL, CD_CALLOC, NULL, me->totface);
 329                 mesh_update_customdata_pointers(me);
 330         }
 331
 332         if(shade)
 333                 shadeMeshMCol(scene, ob, me);
 334         else
 335                 memset(me->mcol, 255, 4*sizeof(MCol)*me->totface);
 336
 337 // XXX  if (me->mr) multires_load_cols(me);
 338
 339         DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
 340
 341 }
 342
 343 static void copy_vpaint_prev(VPaint *vp, unsigned int *mcol, int tot)
 344 {
 345         if(vp->vpaint_prev) {
 346                 MEM_freeN(vp->vpaint_prev);
 347                 vp->vpaint_prev= NULL;
 348         }
 349         vp->tot= tot;
 350
 351         if(mcol==NULL || tot==0) return;
 352
 353         vp->vpaint_prev= MEM_mallocN(4*sizeof(int)*tot, "vpaint_prev");
 354         memcpy(vp->vpaint_prev, mcol, 4*sizeof(int)*tot);
 355
 356 }
 357
 358 static void copy_wpaint_prev (VPaint *wp, MDeformVert *dverts, int dcount)
 359 {
 360         if (wp->wpaint_prev) {
 361                 free_dverts(wp->wpaint_prev, wp->tot);
 362                 wp->wpaint_prev= NULL;
 363         }
 364
 365         if(dverts && dcount) {
 366
 367                 wp->wpaint_prev = MEM_mallocN (sizeof(MDeformVert)*dcount, "wpaint prev");
 368                 wp->tot = dcount;
 369                 copy_dverts (wp->wpaint_prev, dverts, dcount);
 370         }
 371 }
 372
 373
 374 void clear_vpaint(Scene *scene)
 375 {
 376         Mesh *me;
 377         Object *ob;
 378         unsigned int *to, paintcol;
 379         int a;
 380
 381         if((G.f & G_VERTEXPAINT)==0) return;
 382
 383         ob= OBACT;
 384         me= get_mesh(ob);
 385         if(!ob || ob->id.lib) return;
 386
 387         if(me==0 || me->mcol==0 || me->totface==0) return;
 388
 389         paintcol= vpaint_get_current_col(scene->toolsettings->vpaint);
 390
 391         to= (unsigned int *)me->mcol;
 392         a= 4*me->totface;
 393         while(a--) {
 394                 *to= paintcol;
 395                 to++;
 396         }
 397         DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
 398
 399 }
 400
 401 void clear_vpaint_selectedfaces(Scene *scene)
 402 {
 403         Mesh *me;
 404         MFace *mf;
 405         Object *ob;
 406         unsigned int paintcol, *mcol;
 407         int i;
 408
 409         ob= OBACT;
 410         me= get_mesh(ob);
 411         if(me==0 || me->totface==0) return;
 412
 413         if(!me->mcol)
 414                 make_vertexcol(scene, 0);
 415
 416         paintcol= vpaint_get_current_col(scene->toolsettings->vpaint);
 417
 418         mf = me->mface;
 419         mcol = (unsigned int*)me->mcol;
 420         for (i = 0; i < me->totface; i++, mf++, mcol+=4) {
 421                 if (mf->flag & ME_FACE_SEL) {
 422                         mcol[0] = paintcol;
 423                         mcol[1] = paintcol;
 424                         mcol[2] = paintcol;
 425                         mcol[3] = paintcol;
 426                 }
 427         }
 428
 429         DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
 430 }
 431
 432
 433 /* fills in the selected faces with the current weight and vertex group */
 434 void clear_wpaint_selectedfaces(Scene *scene)
 435 {
 436         VPaint *wp= scene->toolsettings->wpaint;
 437         float paintweight= wp->weight;
 438         Mesh *me;
 439         MFace *mface;
 440         Object *ob;
 441         MDeformWeight *dw, *uw;
 442         int *indexar;
 443         int index, vgroup;
 444         unsigned int faceverts[5]={0,0,0,0,0};
 445         unsigned char i;
 446         int vgroup_mirror= -1;
 447
 448         ob= OBACT;
 449         me= ob->data;
 450         if(me==0 || me->totface==0 || me->dvert==0 || !me->mface) return;
 451
 452         indexar= get_indexarray();
 453         for(index=0, mface=me->mface; index<me->totface; index++, mface++) {
 454                 if((mface->flag & ME_FACE_SEL)==0)
 455                         indexar[index]= 0;
 456                 else
 457                         indexar[index]= index+1;
 458         }
 459
 460         vgroup= ob->actdef-1;
 461
 462         /* directly copied from weight_paint, should probaby split into a seperate function */
 463         /* if mirror painting, find the other group */
 464         if(wp->flag & VP_MIRROR_X) {
 465                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
 466                 if(defgroup) {
 467                         bDeformGroup *curdef;
 468                         int actdef= 0;
 469                         char name[32];
 470
 471                         BLI_strncpy(name, defgroup->name, 32);
 472                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
 473
 474                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
 475                                 if (!strcmp(curdef->name, name))
 476                                         break;
 477                         if(curdef==NULL) {
 478                                 int olddef= ob->actdef; /* tsk, add_defgroup sets the active defgroup */
 479                                 curdef= add_defgroup_name (ob, name);
 480                                 ob->actdef= olddef;
 481                         }
 482
 483                         if(curdef && curdef!=defgroup)
 484                                 vgroup_mirror= actdef;
 485                 }
 486         }
 487         /* end copy from weight_paint*/
 488
 489         copy_wpaint_prev(wp, me->dvert, me->totvert);
 490
 491         for(index=0; index<me->totface; index++) {
 492                 if(indexar[index] && indexar[index]<=me->totface) {
 493                         mface= me->mface + (indexar[index]-1);
 494                         /* just so we can loop through the verts */
 495                         faceverts[0]= mface->v1;
 496                         faceverts[1]= mface->v2;
 497                         faceverts[2]= mface->v3;
 498                         faceverts[3]= mface->v4;
 499                         for (i=0; i<3 || faceverts[i]; i++) {
 500                                 if(!((me->dvert+faceverts[i])->flag)) {
 501                                         dw= verify_defweight(me->dvert+faceverts[i], vgroup);
 502                                         if(dw) {
 503                                                 uw= verify_defweight(wp->wpaint_prev+faceverts[i], vgroup);
 504                                                 uw->weight= dw->weight; /* set the undo weight */
 505                                                 dw->weight= paintweight;
 506
 507                                                 if(wp->flag & VP_MIRROR_X) {    /* x mirror painting */
 508                                                         int j= mesh_get_x_mirror_vert(ob, faceverts[i]);
 509                                                         if(j>=0) {
 510                                                                 /* copy, not paint again */
 511                                                                 if(vgroup_mirror != -1) {
 512                                                                         dw= verify_defweight(me->dvert+j, vgroup_mirror);
 513                                                                         uw= verify_defweight(wp->wpaint_prev+j, vgroup_mirror);
 514                                                                 } else {
 515                                                                         dw= verify_defweight(me->dvert+j, vgroup);
 516                                                                         uw= verify_defweight(wp->wpaint_prev+j, vgroup);
 517                                                                 }
 518                                                                 uw->weight= dw->weight; /* set the undo weight */
 519                                                                 dw->weight= paintweight;
 520                                                         }
 521                                                 }
 522                                         }
 523                                         (me->dvert+faceverts[i])->flag= 1;
 524                                 }
 525                         }
 526                 }
 527         }
 528
 529         index=0;
 530         while (index<me->totvert) {
 531                 (me->dvert+index)->flag= 0;
 532                 index++;
 533         }
 534
 535         MEM_freeN(indexar);
 536         copy_wpaint_prev(wp, NULL, 0);
 537
 538         DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
 539 }
 540
 541
 542 void vpaint_dogamma(Scene *scene)
 543 {
 544         VPaint *vp= scene->toolsettings->vpaint;
 545         Mesh *me;
 546         Object *ob;
 547         float igam, fac;
 548         int a, temp;
 549         unsigned char *cp, gamtab[256];
 550
 551         if((G.f & G_VERTEXPAINT)==0) return;
 552
 553         ob= OBACT;
 554         me= get_mesh(ob);
 555         if(me==0 || me->mcol==0 || me->totface==0) return;
 556
 557         igam= 1.0/vp->gamma;
 558         for(a=0; a<256; a++) {
 559
 560                 fac= ((float)a)/255.0;
 561                 fac= vp->mul*pow( fac, igam);
 562
 563                 temp= 255.9*fac;
 564
 565                 if(temp<=0) gamtab[a]= 0;
 566                 else if(temp>=255) gamtab[a]= 255;
 567                 else gamtab[a]= temp;
 568         }
 569
 570         a= 4*me->totface;
 571         cp= (unsigned char *)me->mcol;
 572         while(a--) {
 573
 574                 cp[1]= gamtab[ cp[1] ];
 575                 cp[2]= gamtab[ cp[2] ];
 576                 cp[3]= gamtab[ cp[3] ];
 577
 578                 cp+= 4;
 579         }
 580 }
 581
 582 static unsigned int mcol_blend(unsigned int col1, unsigned int col2, int fac)
 583 {
 584         char *cp1, *cp2, *cp;
 585         int mfac;
 586         unsigned int col=0;
 587
 588         if(fac==0) return col1;
 589         if(fac>=255) return col2;
 590
 591         mfac= 255-fac;
 592
 593         cp1= (char *)&col1;
 594         cp2= (char *)&col2;
 595         cp=  (char *)&col;
 596
 597         cp[0]= 255;
 598         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
 599         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
 600         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
 601
 602         return col;
 603 }
 604
 605 static unsigned int mcol_add(unsigned int col1, unsigned int col2, int fac)
 606 {
 607         char *cp1, *cp2, *cp;
 608         int temp;
 609         unsigned int col=0;
 610
 611         if(fac==0) return col1;
 612
 613         cp1= (char *)&col1;
 614         cp2= (char *)&col2;
 615         cp=  (char *)&col;
 616
 617         cp[0]= 255;
 618         temp= cp1[1] + ((fac*cp2[1])/255);
 619         if(temp>254) cp[1]= 255; else cp[1]= temp;
 620         temp= cp1[2] + ((fac*cp2[2])/255);
 621         if(temp>254) cp[2]= 255; else cp[2]= temp;
 622         temp= cp1[3] + ((fac*cp2[3])/255);
 623         if(temp>254) cp[3]= 255; else cp[3]= temp;
 624
 625         return col;
 626 }
 627
 628 static unsigned int mcol_sub(unsigned int col1, unsigned int col2, int fac)
 629 {
 630         char *cp1, *cp2, *cp;
 631         int temp;
 632         unsigned int col=0;
 633
 634         if(fac==0) return col1;
 635
 636         cp1= (char *)&col1;
 637         cp2= (char *)&col2;
 638         cp=  (char *)&col;
 639
 640         cp[0]= 255;
 641         temp= cp1[1] - ((fac*cp2[1])/255);
 642         if(temp<0) cp[1]= 0; else cp[1]= temp;
 643         temp= cp1[2] - ((fac*cp2[2])/255);
 644         if(temp<0) cp[2]= 0; else cp[2]= temp;
 645         temp= cp1[3] - ((fac*cp2[3])/255);
 646         if(temp<0) cp[3]= 0; else cp[3]= temp;
 647
 648         return col;
 649 }
 650
 651 static unsigned int mcol_mul(unsigned int col1, unsigned int col2, int fac)
 652 {
 653         char *cp1, *cp2, *cp;
 654         int mfac;
 655         unsigned int col=0;
 656
 657         if(fac==0) return col1;
 658
 659         mfac= 255-fac;
 660
 661         cp1= (char *)&col1;
 662         cp2= (char *)&col2;
 663         cp=  (char *)&col;
 664
 665         /* first mul, then blend the fac */
 666         cp[0]= 255;
 667         cp[1]= (mfac*cp1[1] + fac*((cp2[1]*cp1[1])/255)  )/255;
 668         cp[2]= (mfac*cp1[2] + fac*((cp2[2]*cp1[2])/255)  )/255;
 669         cp[3]= (mfac*cp1[3] + fac*((cp2[3]*cp1[3])/255)  )/255;
 670
 671
 672         return col;
 673 }
 674
 675 static unsigned int mcol_lighten(unsigned int col1, unsigned int col2, int fac)
 676 {
 677         char *cp1, *cp2, *cp;
 678         int mfac;
 679         unsigned int col=0;
 680
 681         if(fac==0) return col1;
 682         if(fac>=255) return col2;
 683
 684         mfac= 255-fac;
 685
 686         cp1= (char *)&col1;
 687         cp2= (char *)&col2;
 688         cp=  (char *)&col;
 689
 690         /* See if are lighter, if so mix, else dont do anything.
 691         if the paint col is darker then the original, then ignore */
 692         if (cp1[1]+cp1[2]+cp1[3] > cp2[1]+cp2[2]+cp2[3])
 693                 return col1;
 694
 695         cp[0]= 255;
 696         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
 697         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
 698         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
 699
 700         return col;
 701 }
 702
 703 static unsigned int mcol_darken(unsigned int col1, unsigned int col2, int fac)
 704 {
 705         char *cp1, *cp2, *cp;
 706         int mfac;
 707         unsigned int col=0;
 708
 709         if(fac==0) return col1;
 710         if(fac>=255) return col2;
 711
 712         mfac= 255-fac;
 713
 714         cp1= (char *)&col1;
 715         cp2= (char *)&col2;
 716         cp=  (char *)&col;
 717
 718         /* See if were darker, if so mix, else dont do anything.
 719         if the paint col is brighter then the original, then ignore */
 720         if (cp1[1]+cp1[2]+cp1[3] < cp2[1]+cp2[2]+cp2[3])
 721                 return col1;
 722
 723         cp[0]= 255;
 724         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
 725         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
 726         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
 727         return col;
 728 }
 729
 730 static void vpaint_blend(VPaint *vp, unsigned int *col, unsigned int *colorig, unsigned int paintcol, int alpha)
 731 {
 732
 733         if(vp->mode==VP_MIX || vp->mode==VP_BLUR) *col= mcol_blend( *col, paintcol, alpha);
 734         else if(vp->mode==VP_ADD) *col= mcol_add( *col, paintcol, alpha);
 735         else if(vp->mode==VP_SUB) *col= mcol_sub( *col, paintcol, alpha);
 736         else if(vp->mode==VP_MUL) *col= mcol_mul( *col, paintcol, alpha);
 737         else if(vp->mode==VP_LIGHTEN) *col= mcol_lighten( *col, paintcol, alpha);
 738         else if(vp->mode==VP_DARKEN) *col= mcol_darken( *col, paintcol, alpha);
 739
 740         /* if no spray, clip color adding with colorig & orig alpha */
 741         if((vp->flag & VP_SPRAY)==0) {
 742                 unsigned int testcol=0, a;
 743                 char *cp, *ct, *co;
 744
 745                 alpha= (int)(255.0*vp->a);
 746
 747                 if(vp->mode==VP_MIX || vp->mode==VP_BLUR) testcol= mcol_blend( *colorig, paintcol, alpha);
 748                 else if(vp->mode==VP_ADD) testcol= mcol_add( *colorig, paintcol, alpha);
 749                 else if(vp->mode==VP_SUB) testcol= mcol_sub( *colorig, paintcol, alpha);
 750                 else if(vp->mode==VP_MUL) testcol= mcol_mul( *colorig, paintcol, alpha);
 751                 else if(vp->mode==VP_LIGHTEN)  testcol= mcol_lighten( *colorig, paintcol, alpha);
 752                 else if(vp->mode==VP_DARKEN)   testcol= mcol_darken( *colorig, paintcol, alpha);
 753
 754                 cp= (char *)col;
 755                 ct= (char *)&testcol;
 756                 co= (char *)colorig;
 757
 758                 for(a=0; a<4; a++) {
 759                         if( ct[a]<co[a] ) {
 760                                 if( cp[a]<ct[a] ) cp[a]= ct[a];
 761                                 else if( cp[a]>co[a] ) cp[a]= co[a];
 762                         }
 763                         else {
 764                                 if( cp[a]<co[a] ) cp[a]= co[a];
 765                                 else if( cp[a]>ct[a] ) cp[a]= ct[a];
 766                         }
 767                 }
 768         }
 769 }
 770
 771
 772 static int sample_backbuf_area(ViewContext *vc, int *indexar, int totface, int x, int y, float size)
 773 {
 774         struct ImBuf *ibuf;
 775         int a, tot=0, index;
 776
 777         if(totface+4>=MAXINDEX) return 0;
 778
 779         if(size>64.0) size= 64.0;
 780
 781         ibuf= view3d_read_backbuf(vc, x-size, y-size, x+size, y+size);
 782         if(ibuf) {
 783                 unsigned int *rt= ibuf->rect;
 784
 785                 memset(indexar, 0, sizeof(int)*totface+4);      /* plus 2! first element is total, +2 was giving valgrind errors, +4 seems ok */
 786
 787                 size= ibuf->x*ibuf->y;
 788                 while(size--) {
 789
 790                         if(*rt) {
 791                                 index= WM_framebuffer_to_index(*rt);
 792                                 if(index>0 && index<=totface)
 793                                         indexar[index] = 1;
 794                         }
 795
 796                         rt++;
 797                 }
 798
 799                 for(a=1; a<=totface; a++) {
 800                         if(indexar[a]) indexar[tot++]= a;
 801                 }
 802
 803                 IMB_freeImBuf(ibuf);
 804         }
 805
 806         return tot;
 807 }
 808
 809 static int calc_vp_alpha_dl(VPaint *vp, ViewContext *vc, float vpimat[][3], float *vert_nor, short *mval)
 810 {
 811         float fac, dx, dy;
 812         int alpha;
 813         short vertco[2];
 814
 815         if(vp->flag & VP_SOFT) {
 816                 project_short_noclip(vc->ar, vert_nor, vertco);
 817                 dx= mval[0]-vertco[0];
 818                 dy= mval[1]-vertco[1];
 819
 820                 fac= sqrt(dx*dx + dy*dy);
 821                 if(fac > vp->size) return 0;
 822                 if(vp->flag & VP_HARD)
 823                         alpha= 255;
 824                 else
 825                         alpha= 255.0*vp->a*(1.0-fac/vp->size);
 826         }
 827         else {
 828                 alpha= 255.0*vp->a;
 829         }
 830
 831         if(vp->flag & VP_NORMALS) {
 832                 float *no= vert_nor+3;
 833
 834                         /* transpose ! */
 835                 fac= vpimat[2][0]*no[0]+vpimat[2][1]*no[1]+vpimat[2][2]*no[2];
 836                 if(fac>0.0) {
 837                         dx= vpimat[0][0]*no[0]+vpimat[0][1]*no[1]+vpimat[0][2]*no[2];
 838                         dy= vpimat[1][0]*no[0]+vpimat[1][1]*no[1]+vpimat[1][2]*no[2];
 839
 840                         alpha*= fac/sqrt(dx*dx + dy*dy + fac*fac);
 841                 }
 842                 else return 0;
 843         }
 844
 845         return alpha;
 846 }
 847
 848 static void wpaint_blend(VPaint *wp, MDeformWeight *dw, MDeformWeight *uw, float alpha, float paintval)
 849 {
 850
 851         if(dw==NULL || uw==NULL) return;
 852
 853         if(wp->mode==VP_MIX || wp->mode==VP_BLUR)
 854                 dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
 855         else if(wp->mode==VP_ADD)
 856                 dw->weight += paintval*alpha;
 857         else if(wp->mode==VP_SUB)
 858                 dw->weight -= paintval*alpha;
 859         else if(wp->mode==VP_MUL)
 860                 /* first mul, then blend the fac */
 861                 dw->weight = ((1.0-alpha) + alpha*paintval)*dw->weight;
 862         else if(wp->mode==VP_LIGHTEN) {
 863                 if (dw->weight < paintval)
 864                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
 865         } else if(wp->mode==VP_DARKEN) {
 866                 if (dw->weight > paintval)
 867                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
 868         }
 869         CLAMP(dw->weight, 0.0f, 1.0f);
 870
 871         /* if no spray, clip result with orig weight & orig alpha */
 872         if((wp->flag & VP_SPRAY)==0) {
 873                 float testw=0.0f;
 874
 875                 alpha= wp->a;
 876                 if(wp->mode==VP_MIX || wp->mode==VP_BLUR)
 877                         testw = paintval*alpha + uw->weight*(1.0-alpha);
 878                 else if(wp->mode==VP_ADD)
 879                         testw = uw->weight + paintval*alpha;
 880                 else if(wp->mode==VP_SUB)
 881                         testw = uw->weight - paintval*alpha;
 882                 else if(wp->mode==VP_MUL)
 883                         /* first mul, then blend the fac */
 884                         testw = ((1.0-alpha) + alpha*paintval)*uw->weight;
 885                 else if(wp->mode==VP_LIGHTEN) {
 886                         if (uw->weight < paintval)
 887                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
 888                         else
 889                                 testw = uw->weight;
 890                 } else if(wp->mode==VP_DARKEN) {
 891                         if (uw->weight > paintval)
 892                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
 893                         else
 894                                 testw = uw->weight;
 895                 }
 896                 CLAMP(testw, 0.0f, 1.0f);
 897
 898                 if( testw<uw->weight ) {
 899                         if(dw->weight < testw) dw->weight= testw;
 900                         else if(dw->weight > uw->weight) dw->weight= uw->weight;
 901                 }
 902                 else {
 903                         if(dw->weight > testw) dw->weight= testw;
 904                         else if(dw->weight < uw->weight) dw->weight= uw->weight;
 905                 }
 906         }
 907
 908 }
 909
 910 /* ----------------------------------------------------- */
 911
 912 /* used for 3d view, on active object, assumes me->dvert exists */
 913 /* if mode==1: */
 914 /*     samples cursor location, and gives menu with vertex groups to activate */
 915 /* else */
 916 /*     sets wp->weight to the closest weight value to vertex */
 917 /*     note: we cant sample frontbuf, weight colors are interpolated too unpredictable */
 918 void sample_wpaint(Scene *scene, ARegion *ar, View3D *v3d, int mode)
 919 {
 920         ViewContext vc;
 921         VPaint *wp= scene->toolsettings->wpaint;
 922         Object *ob= OBACT;
 923         Mesh *me= get_mesh(ob);
 924         int index;
 925         short mval[2], sco[2];
 926
 927         if (!me) return;
 928
 929 //      getmouseco_areawin(mval);
 930         index= view3d_sample_backbuf(&vc, mval[0], mval[1]);
 931
 932         if(index && index<=me->totface) {
 933                 MFace *mface;
 934
 935                 mface= ((MFace *)me->mface) + index-1;
 936
 937                 if(mode==1) {   /* sampe which groups are in here */
 938                         MDeformVert *dv;
 939                         int a, totgroup;
 940
 941                         totgroup= BLI_countlist(&ob->defbase);
 942                         if(totgroup) {
 943                                 int totmenu=0;
 944                                 int *groups=MEM_callocN(totgroup*sizeof(int), "groups");
 945
 946                                 dv= me->dvert+mface->v1;
 947                                 for(a=0; a<dv->totweight; a++) {
 948                                         if (dv->dw[a].def_nr<totgroup)
 949                                                 groups[dv->dw[a].def_nr]= 1;
 950                                 }
 951                                 dv= me->dvert+mface->v2;
 952                                 for(a=0; a<dv->totweight; a++) {
 953                                         if (dv->dw[a].def_nr<totgroup)
 954                                                 groups[dv->dw[a].def_nr]= 1;
 955                                 }
 956                                 dv= me->dvert+mface->v3;
 957                                 for(a=0; a<dv->totweight; a++) {
 958                                         if (dv->dw[a].def_nr<totgroup)
 959                                                 groups[dv->dw[a].def_nr]= 1;
 960                                 }
 961                                 if(mface->v4) {
 962                                         dv= me->dvert+mface->v4;
 963                                         for(a=0; a<dv->totweight; a++) {
 964                                                 if (dv->dw[a].def_nr<totgroup)
 965                                                         groups[dv->dw[a].def_nr]= 1;
 966                                         }
 967                                 }
 968                                 for(a=0; a<totgroup; a++)
 969                                         if(groups[a]) totmenu++;
 970
 971                                 if(totmenu==0) {
 972                                         //notice("No Vertex Group Selected");
 973                                 }
 974                                 else {
 975                                         bDeformGroup *dg;
 976                                         short val;
 977                                         char item[40], *str= MEM_mallocN(40*totmenu+40, "menu");
 978
 979                                         strcpy(str, "Vertex Groups %t");
 980                                         for(a=0, dg=ob->defbase.first; dg && a<totgroup; a++, dg= dg->next) {
 981                                                 if(groups[a]) {
 982                                                         sprintf(item, "|%s %%x%d", dg->name, a);
 983                                                         strcat(str, item);
 984                                                 }
 985                                         }
 986
 987                                         val= 0; // XXX pupmenu(str);
 988                                         if(val>=0) {
 989                                                 ob->actdef= val+1;
 990                                                 DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
 991                                         }
 992                                         MEM_freeN(str);
 993                                 }
 994                                 MEM_freeN(groups);
 995                         }
 996 //                      else notice("No Vertex Groups in Object");
 997                 }
 998                 else {
 999                         DerivedMesh *dm;
1000                         MDeformWeight *dw;
1001                         float w1, w2, w3, w4, co[3], fac;
1002
1003                         dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
1004                         if(dm->getVertCo==NULL) {
1005                                 //notice("Not supported yet");
1006                         }
1007                         else {
1008                                 /* calc 3 or 4 corner weights */
1009                                 dm->getVertCo(dm, mface->v1, co);
1010                                 project_short_noclip(ar, co, sco);
1011                                 w1= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
1012
1013                                 dm->getVertCo(dm, mface->v2, co);
1014                                 project_short_noclip(ar, co, sco);
1015                                 w2= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
1016
1017                                 dm->getVertCo(dm, mface->v3, co);
1018                                 project_short_noclip(ar, co, sco);
1019                                 w3= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
1020
1021                                 if(mface->v4) {
1022                                         dm->getVertCo(dm, mface->v4, co);
1023                                         project_short_noclip(ar, co, sco);
1024                                         w4= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
1025                                 }
1026                                 else w4= 1.0e10;
1027
1028                                 fac= MIN4(w1, w2, w3, w4);
1029                                 if(w1==fac) {
1030                                         dw= get_defweight(me->dvert+mface->v1, ob->actdef-1);
1031                                         if(dw) wp->weight= dw->weight; else wp->weight= 0.0f;
1032                                 }
1033                                 else if(w2==fac) {
1034                                         dw= get_defweight(me->dvert+mface->v2, ob->actdef-1);
1035                                         if(dw) wp->weight= dw->weight; else wp->weight= 0.0f;
1036                                 }
1037                                 else if(w3==fac) {
1038                                         dw= get_defweight(me->dvert+mface->v3, ob->actdef-1);
1039                                         if(dw) wp->weight= dw->weight; else wp->weight= 0.0f;
1040                                 }
1041                                 else if(w4==fac) {
1042                                         if(mface->v4) {
1043                                                 dw= get_defweight(me->dvert+mface->v4, ob->actdef-1);
1044                                                 if(dw) wp->weight= dw->weight; else wp->weight= 0.0f;
1045                                         }
1046                                 }
1047                         }
1048                         dm->release(dm);
1049                 }
1050
1051         }
1052
1053 }
1054
1055 static void do_weight_paint_vertex(VPaint *wp, Object *ob, int index, int alpha, float paintweight, int vgroup_mirror)
1056 {
1057         Mesh *me= ob->data;
1058         MDeformWeight *dw, *uw;
1059         int vgroup= ob->actdef-1;
1060
1061         if(wp->flag & VP_ONLYVGROUP) {
1062                 dw= get_defweight(me->dvert+index, vgroup);
1063                 uw= get_defweight(wp->wpaint_prev+index, vgroup);
1064         }
1065         else {
1066                 dw= verify_defweight(me->dvert+index, vgroup);
1067                 uw= verify_defweight(wp->wpaint_prev+index, vgroup);
1068         }
1069         if(dw==NULL || uw==NULL)
1070                 return;
1071
1072         wpaint_blend(wp, dw, uw, (float)alpha/255.0, paintweight);
1073
1074         if(wp->flag & VP_MIRROR_X) {    /* x mirror painting */
1075                 int j= mesh_get_x_mirror_vert(ob, index);
1076                 if(j>=0) {
1077                         /* copy, not paint again */
1078                         if(vgroup_mirror != -1)
1079                                 uw= verify_defweight(me->dvert+j, vgroup_mirror);
1080                         else
1081                                 uw= verify_defweight(me->dvert+j, vgroup);
1082
1083                         uw->weight= dw->weight;
1084                 }
1085         }
1086 }
1087
1088
1089 /* *************** set wpaint operator ****************** */
1090
1091 static int set_wpaint(bContext *C, wmOperator *op)              /* toggle */
1092 {
1093         Object *ob= CTX_data_active_object(C);
1094         Scene *scene= CTX_data_scene(C);
1095         VPaint *wp= scene->toolsettings->wpaint;
1096         Mesh *me;
1097
1098         me= get_mesh(ob);
1099         if(ob->id.lib || me==NULL) return OPERATOR_CANCELLED;
1100
1101         if(me && me->totface>=MAXINDEX) {
1102                 error("Maximum number of faces: %d", MAXINDEX-1);
1103                 G.f &= ~G_WEIGHTPAINT;
1104                 return OPERATOR_CANCELLED;
1105         }
1106
1107         if(G.f & G_WEIGHTPAINT) G.f &= ~G_WEIGHTPAINT;
1108         else G.f |= G_WEIGHTPAINT;
1109
1110
1111         /* Weightpaint works by overriding colors in mesh,
1112                 * so need to make sure we recalc on enter and
1113                 * exit (exit needs doing regardless because we
1114                                 * should redeform).
1115                 */
1116         DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
1117
1118         if(G.f & G_WEIGHTPAINT) {
1119                 Object *par;
1120
1121                 if(wp==NULL)
1122                         wp= scene->toolsettings->wpaint= new_vpaint(1);
1123
1124                 toggle_paint_cursor(C, 1);
1125
1126                 mesh_octree_table(ob, NULL, NULL, 's');
1127
1128                 /* verify if active weight group is also active bone */
1129                 par= modifiers_isDeformedByArmature(ob);
1130                 if(par && (par->flag & OB_POSEMODE)) {
1131                         bPoseChannel *pchan;
1132                         for(pchan= par->pose->chanbase.first; pchan; pchan= pchan->next)
1133                                 if(pchan->bone->flag & BONE_ACTIVE)
1134                                         break;
1135                                 if(pchan)
1136                                         vertexgroup_select_by_name(ob, pchan->name);
1137                 }
1138         }
1139         else {
1140                 if(wp)
1141                         toggle_paint_cursor(C, 1);
1142
1143                 mesh_octree_table(ob, NULL, NULL, 'e');
1144         }
1145
1146         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1147
1148         return OPERATOR_FINISHED;
1149 }
1150
1151 /* for switching to/from mode */
1152 static int paint_poll_test(bContext *C)
1153 {
1154         if(ED_operator_view3d_active(C)==0)
1155                 return 0;
1156         if(CTX_data_edit_object(C))
1157                 return 0;
1158         if(CTX_data_active_object(C)==NULL)
1159                 return 0;
1160         return 1;
1161 }
1162
1163 void PAINT_OT_weight_paint_toggle(wmOperatorType *ot)
1164 {
1165
1166         /* identifiers */
1167         ot->name= "Weight Paint Mode";
1168         ot->idname= "PAINT_OT_weight_paint_toggle";
1169
1170         /* api callbacks */
1171         ot->exec= set_wpaint;
1172         ot->poll= paint_poll_test;
1173
1174         /* flags */
1175         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1176
1177 }
1178
1179 /* ************ paint radial controls *************/
1180
1181 void paint_radial_control_invoke(wmOperator *op, VPaint *vp)
1182 {
1183         int mode = RNA_int_get(op->ptr, "mode");
1184         float original_value;
1185
1186         if(mode == WM_RADIALCONTROL_SIZE)
1187                 original_value = vp->size;
1188         else if(mode == WM_RADIALCONTROL_STRENGTH)
1189                 original_value = vp->a;
1190
1191         RNA_float_set(op->ptr, "initial_value", original_value);
1192 }
1193
1194 static int paint_radial_control_exec(wmOperator *op, VPaint *vp)
1195 {
1196         int mode = RNA_int_get(op->ptr, "mode");
1197         float new_value = RNA_float_get(op->ptr, "new_value");
1198
1199         if(mode == WM_RADIALCONTROL_SIZE)
1200                 vp->size = new_value;
1201         else if(mode == WM_RADIALCONTROL_STRENGTH)
1202                 vp->a = new_value;
1203
1204         return OPERATOR_FINISHED;
1205 }
1206
1207 static int vpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1208 {
1209         toggle_paint_cursor(C, 0);
1210         paint_radial_control_invoke(op, CTX_data_scene(C)->toolsettings->vpaint);
1211         return WM_radial_control_invoke(C, op, event);
1212 }
1213
1214 static int vpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1215 {
1216         int ret = WM_radial_control_modal(C, op, event);
1217         if(ret != OPERATOR_RUNNING_MODAL)
1218                 toggle_paint_cursor(C, 0);
1219         return ret;
1220 }
1221
1222 static int vpaint_radial_control_exec(bContext *C, wmOperator *op)
1223 {
1224         int ret = paint_radial_control_exec(op, CTX_data_scene(C)->toolsettings->vpaint);
1225         char str[256];
1226         WM_radial_control_string(op, str, 256);
1227         return ret;
1228 }
1229
1230 static int wpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1231 {
1232         toggle_paint_cursor(C, 1);
1233         paint_radial_control_invoke(op, CTX_data_scene(C)->toolsettings->wpaint);
1234         return WM_radial_control_invoke(C, op, event);
1235 }
1236
1237 static int wpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1238 {
1239         int ret = WM_radial_control_modal(C, op, event);
1240         if(ret != OPERATOR_RUNNING_MODAL)
1241                 toggle_paint_cursor(C, 1);
1242         return ret;
1243 }
1244
1245 static int wpaint_radial_control_exec(bContext *C, wmOperator *op)
1246 {
1247         int ret = paint_radial_control_exec(op, CTX_data_scene(C)->toolsettings->wpaint);
1248         char str[256];
1249         WM_radial_control_string(op, str, 256);
1250         return ret;
1251 }
1252
1253 void PAINT_OT_weight_paint_radial_control(wmOperatorType *ot)
1254 {
1255         WM_OT_radial_control_partial(ot);
1256
1257         ot->name= "Weight Paint Radial Control";
1258         ot->idname= "PAINT_OT_weight_paint_radial_control";
1259
1260         ot->invoke= wpaint_radial_control_invoke;
1261         ot->modal= wpaint_radial_control_modal;
1262         ot->exec= wpaint_radial_control_exec;
1263         ot->poll= wp_poll;
1264
1265         /* flags */
1266         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1267 }
1268
1269 void PAINT_OT_vertex_paint_radial_control(wmOperatorType *ot)
1270 {
1271         WM_OT_radial_control_partial(ot);
1272
1273         ot->name= "Vertex Paint Radial Control";
1274         ot->idname= "PAINT_OT_vertex_paint_radial_control";
1275
1276         ot->invoke= vpaint_radial_control_invoke;
1277         ot->modal= vpaint_radial_control_modal;
1278         ot->exec= vpaint_radial_control_exec;
1279         ot->poll= vp_poll;
1280
1281         /* flags */
1282         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1283 }
1284
1285 /* ************ weight paint operator ********** */
1286
1287 struct WPaintData {
1288         ViewContext vc;
1289         int *indexar;
1290         int vgroup_mirror;
1291         float *vertexcosnos;
1292         float wpimat[3][3];
1293 };
1294
1295 static void wpaint_exit(bContext *C, wmOperator *op)
1296 {
1297         ToolSettings *ts= CTX_data_tool_settings(C);
1298         Object *ob= CTX_data_active_object(C);
1299         struct WPaintData *wpd= op->customdata;
1300
1301         if(wpd->vertexcosnos)
1302                 MEM_freeN(wpd->vertexcosnos);
1303         MEM_freeN(wpd->indexar);
1304
1305         /* frees prev buffer */
1306         copy_wpaint_prev(ts->wpaint, NULL, 0);
1307
1308         /* and particles too */
1309         if(ob->particlesystem.first) {
1310                 ParticleSystem *psys;
1311                 int i;
1312
1313                 for(psys= ob->particlesystem.first; psys; psys= psys->next) {
1314                         for(i=0; i<PSYS_TOT_VG; i++) {
1315                                 if(psys->vgroup[i]==ob->actdef) {
1316                                         psys->recalc |= PSYS_RECALC_HAIR;
1317                                         break;
1318                                 }
1319                         }
1320                 }
1321         }
1322
1323         DAG_object_flush_update(CTX_data_scene(C), ob, OB_RECALC_DATA);
1324
1325         MEM_freeN(wpd);
1326         op->customdata= NULL;
1327 }
1328
1329
1330 static int wpaint_modal(bContext *C, wmOperator *op, wmEvent *event)
1331 {
1332         ToolSettings *ts= CTX_data_tool_settings(C);
1333         VPaint *wp= ts->wpaint;
1334
1335         switch(event->type) {
1336                 case LEFTMOUSE:
1337                         if(event->val==0) { /* release */
1338                                 wpaint_exit(C, op);
1339                                 return OPERATOR_FINISHED;
1340                         }
1341                         /* pass on, first press gets painted too */
1342
1343                 case MOUSEMOVE:
1344                 {
1345                         struct WPaintData *wpd= op->customdata;
1346                         ViewContext *vc= &wpd->vc;
1347                         Object *ob= vc->obact;
1348                         Mesh *me= ob->data;
1349                         float mat[4][4];
1350                         float paintweight= wp->weight;
1351                         int *indexar= wpd->indexar;
1352                         int totindex, index, alpha, totw;
1353                         short mval[2];
1354
1355                         view3d_operator_needs_opengl(C);
1356
1357                         /* load projection matrix */
1358                         wmMultMatrix(ob->obmat);
1359                         wmGetSingleMatrix(mat);
1360                         wmLoadMatrix(wpd->vc.rv3d->viewmat);
1361
1362                         MTC_Mat4SwapMat4(wpd->vc.rv3d->persmat, mat);
1363
1364                         mval[0]= event->x - vc->ar->winrct.xmin;
1365                         mval[1]= event->y - vc->ar->winrct.ymin;
1366
1367                         /* which faces are involved */
1368                         if(wp->flag & VP_AREA) {
1369                                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], wp->size);
1370                         }
1371                         else {
1372                                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1373                                 if(indexar[0]) totindex= 1;
1374                                 else totindex= 0;
1375                         }
1376
1377                         if(wp->flag & VP_COLINDEX) {
1378                                 for(index=0; index<totindex; index++) {
1379                                         if(indexar[index] && indexar[index]<=me->totface) {
1380                                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1381
1382                                                 if(mface->mat_nr!=ob->actcol-1) {
1383                                                         indexar[index]= 0;
1384                                                 }
1385                                         }
1386                                 }
1387                         }
1388
1389                         if((G.f & G_FACESELECT) && me->mface) {
1390                                 for(index=0; index<totindex; index++) {
1391                                         if(indexar[index] && indexar[index]<=me->totface) {
1392                                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1393
1394                                                 if((mface->flag & ME_FACE_SEL)==0) {
1395                                                         indexar[index]= 0;
1396                                                 }
1397                                         }
1398                                 }
1399                         }
1400
1401                         /* make sure each vertex gets treated only once */
1402                         /* and calculate filter weight */
1403                         totw= 0;
1404                         if(wp->mode==VP_BLUR)
1405                                 paintweight= 0.0f;
1406                         else
1407                                 paintweight= wp->weight;
1408
1409                         for(index=0; index<totindex; index++) {
1410                                 if(indexar[index] && indexar[index]<=me->totface) {
1411                                         MFace *mface= me->mface + (indexar[index]-1);
1412
1413                                         (me->dvert+mface->v1)->flag= 1;
1414                                         (me->dvert+mface->v2)->flag= 1;
1415                                         (me->dvert+mface->v3)->flag= 1;
1416                                         if(mface->v4) (me->dvert+mface->v4)->flag= 1;
1417
1418                                         if(wp->mode==VP_BLUR) {
1419                                                 MDeformWeight *dw, *(*dw_func)(MDeformVert *, int) = verify_defweight;
1420
1421                                                 if(wp->flag & VP_ONLYVGROUP)
1422                                                         dw_func= get_defweight;
1423
1424                                                 dw= dw_func(me->dvert+mface->v1, ob->actdef-1);
1425                                                 if(dw) {paintweight+= dw->weight; totw++;}
1426                                                 dw= dw_func(me->dvert+mface->v2, ob->actdef-1);
1427                                                 if(dw) {paintweight+= dw->weight; totw++;}
1428                                                 dw= dw_func(me->dvert+mface->v3, ob->actdef-1);
1429                                                 if(dw) {paintweight+= dw->weight; totw++;}
1430                                                 if(mface->v4) {
1431                                                         dw= dw_func(me->dvert+mface->v4, ob->actdef-1);
1432                                                         if(dw) {paintweight+= dw->weight; totw++;}
1433                                                 }
1434                                         }
1435                                 }
1436                         }
1437
1438                         if(wp->mode==VP_BLUR)
1439                                 paintweight/= (float)totw;
1440
1441                         for(index=0; index<totindex; index++) {
1442
1443                                 if(indexar[index] && indexar[index]<=me->totface) {
1444                                         MFace *mface= me->mface + (indexar[index]-1);
1445
1446                                         if((me->dvert+mface->v1)->flag) {
1447                                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v1, mval);
1448                                                 if(alpha) {
1449                                                         do_weight_paint_vertex(wp, ob, mface->v1, alpha, paintweight, wpd->vgroup_mirror);
1450                                                 }
1451                                                 (me->dvert+mface->v1)->flag= 0;
1452                                         }
1453
1454                                         if((me->dvert+mface->v2)->flag) {
1455                                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v2, mval);
1456                                                 if(alpha) {
1457                                                         do_weight_paint_vertex(wp, ob, mface->v2, alpha, paintweight, wpd->vgroup_mirror);
1458                                                 }
1459                                                 (me->dvert+mface->v2)->flag= 0;
1460                                         }
1461
1462                                         if((me->dvert+mface->v3)->flag) {
1463                                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v3, mval);
1464                                                 if(alpha) {
1465                                                         do_weight_paint_vertex(wp, ob, mface->v3, alpha, paintweight, wpd->vgroup_mirror);
1466                                                 }
1467                                                 (me->dvert+mface->v3)->flag= 0;
1468                                         }
1469
1470                                         if((me->dvert+mface->v4)->flag) {
1471                                                 if(mface->v4) {
1472                                                         alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v4, mval);
1473                                                         if(alpha) {
1474                                                                 do_weight_paint_vertex(wp, ob, mface->v4, alpha, paintweight, wpd->vgroup_mirror);
1475                                                         }
1476                                                         (me->dvert+mface->v4)->flag= 0;
1477                                                 }
1478                                         }
1479                                 }
1480                         }
1481
1482                         MTC_Mat4SwapMat4(vc->rv3d->persmat, mat);
1483
1484                         DAG_object_flush_update(vc->scene, ob, OB_RECALC_DATA);
1485                         ED_region_tag_redraw(vc->ar);
1486                 }
1487         }
1488
1489         return OPERATOR_RUNNING_MODAL;
1490 }
1491
1492 static int wpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1493 {
1494         Scene *scene= CTX_data_scene(C);
1495         ToolSettings *ts= CTX_data_tool_settings(C);
1496         VPaint *wp= ts->wpaint;
1497         Object *ob= CTX_data_active_object(C);
1498         struct WPaintData *wpd;
1499         Mesh *me;
1500         float mat[4][4], imat[4][4];
1501
1502         if(scene->obedit) return OPERATOR_CANCELLED;
1503         // XXX  if(multires_level1_test()) return;
1504
1505         me= get_mesh(ob);
1506         if(me==NULL || me->totface==0) return OPERATOR_CANCELLED;
1507
1508         /* if nothing was added yet, we make dverts and a vertex deform group */
1509         if (!me->dvert)
1510                 create_dverts(&me->id);
1511
1512         /* make customdata storage */
1513         op->customdata= wpd= MEM_callocN(sizeof(struct WPaintData), "WPaintData");
1514         view3d_set_viewcontext(C, &wpd->vc);
1515         wpd->vgroup_mirror= -1;
1516
1517         //      if(qual & LR_CTRLKEY) {
1518         //              sample_wpaint(scene, ar, v3d, 0);
1519         //              return;
1520         //      }
1521         //      if(qual & LR_SHIFTKEY) {
1522         //              sample_wpaint(scene, ar, v3d, 1);
1523         //              return;
1524         //      }
1525
1526         /* ALLOCATIONS! no return after this line */
1527         /* painting on subsurfs should give correct points too, this returns me->totvert amount */
1528         wpd->vertexcosnos= mesh_get_mapped_verts_nors(scene, ob);
1529         wpd->indexar= get_indexarray();
1530         copy_wpaint_prev(wp, me->dvert, me->totvert);
1531
1532         /* this happens on a Bone select, when no vgroup existed yet */
1533         if(ob->actdef<=0) {
1534                 Object *modob;
1535                 if((modob = modifiers_isDeformedByArmature(ob))) {
1536                         bPoseChannel *pchan;
1537                         for(pchan= modob->pose->chanbase.first; pchan; pchan= pchan->next)
1538                                 if(pchan->bone->flag & SELECT)
1539                                         break;
1540                         if(pchan) {
1541                                 bDeformGroup *dg= get_named_vertexgroup(ob, pchan->name);
1542                                 if(dg==NULL)
1543                                         dg= add_defgroup_name(ob, pchan->name); /* sets actdef */
1544                                 else
1545                                         ob->actdef= get_defgroup_num(ob, dg);
1546                         }
1547                 }
1548         }
1549         if(ob->defbase.first==NULL) {
1550                 add_defgroup(ob);
1551         }
1552
1553         //      if(ob->lay & v3d->lay); else error("Active object is not in this layer");
1554
1555         /* imat for normals */
1556         Mat4MulMat4(mat, ob->obmat, wpd->vc.rv3d->viewmat);
1557         Mat4Invert(imat, mat);
1558         Mat3CpyMat4(wpd->wpimat, imat);
1559
1560         /* if mirror painting, find the other group */
1561         if(wp->flag & VP_MIRROR_X) {
1562                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
1563                 if(defgroup) {
1564                         bDeformGroup *curdef;
1565                         int actdef= 0;
1566                         char name[32];
1567
1568                         BLI_strncpy(name, defgroup->name, 32);
1569                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
1570
1571                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
1572                                 if (!strcmp(curdef->name, name))
1573                                         break;
1574                         if(curdef==NULL) {
1575                                 int olddef= ob->actdef; /* tsk, add_defgroup sets the active defgroup */
1576                                 curdef= add_defgroup_name (ob, name);
1577                                 ob->actdef= olddef;
1578                         }
1579
1580                         if(curdef && curdef!=defgroup)
1581                                 wpd->vgroup_mirror= actdef;
1582                 }
1583         }
1584
1585         /* do paint once for click only paint */
1586         wpaint_modal(C, op, event);
1587
1588         /* add modal handler */
1589         WM_event_add_modal_handler(C, &CTX_wm_window(C)->handlers, op);
1590
1591         return OPERATOR_RUNNING_MODAL;
1592 }
1593
1594 void PAINT_OT_weight_paint(wmOperatorType *ot)
1595 {
1596
1597         /* identifiers */
1598         ot->name= "Weight Paint";
1599         ot->idname= "PAINT_OT_weight_paint";
1600
1601         /* api callbacks */
1602         ot->invoke= wpaint_invoke;
1603         ot->modal= wpaint_modal;
1604         /* ot->exec= vpaint_exec; <-- needs stroke property */
1605         ot->poll= wp_poll;
1606
1607         /* flags */
1608         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1609
1610 }
1611
1612 /* ************ set / clear vertex paint mode ********** */
1613
1614
1615 static int set_vpaint(bContext *C, wmOperator *op)              /* toggle */
1616 {
1617         Object *ob= CTX_data_active_object(C);
1618         Scene *scene= CTX_data_scene(C);
1619         VPaint *vp= scene->toolsettings->vpaint;
1620         Mesh *me;
1621
1622         me= get_mesh(ob);
1623
1624         if(me==NULL || object_data_is_libdata(ob)) {
1625                 G.f &= ~G_VERTEXPAINT;
1626                 return OPERATOR_FINISHED;
1627         }
1628
1629         if(me && me->totface>=MAXINDEX) {
1630                 error("Maximum number of faces: %d", MAXINDEX-1);
1631                 G.f &= ~G_VERTEXPAINT;
1632                 return OPERATOR_FINISHED;
1633         }
1634
1635         if(me && me->mcol==NULL) make_vertexcol(scene, 0);
1636
1637         /* toggle: end vpaint */
1638         if(G.f & G_VERTEXPAINT) {
1639
1640                 G.f &= ~G_VERTEXPAINT;
1641
1642                 if(vp) {
1643                         toggle_paint_cursor(C, 0);
1644                         vp->paintcursor= NULL;
1645                 }
1646         }
1647         else {
1648
1649                 G.f |= G_VERTEXPAINT;
1650                 /* Turn off weight painting */
1651                 if (G.f & G_WEIGHTPAINT)
1652                         set_wpaint(C, op);
1653
1654                 if(vp==NULL)
1655                         vp= scene->toolsettings->vpaint= new_vpaint(0);
1656
1657                 toggle_paint_cursor(C, 0);
1658         }
1659
1660         if (me)
1661                 /* update modifier stack for mapping requirements */
1662                 DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
1663
1664         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1665
1666         return OPERATOR_FINISHED;
1667 }
1668
1669 void PAINT_OT_vertex_paint_toggle(wmOperatorType *ot)
1670 {
1671
1672         /* identifiers */
1673         ot->name= "Vertex Paint Mode";
1674         ot->idname= "PAINT_OT_vertex_paint_toggle";
1675
1676         /* api callbacks */
1677         ot->exec= set_vpaint;
1678         ot->poll= paint_poll_test;
1679
1680         /* flags */
1681         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1682
1683 }
1684
1685
1686
1687 /* ********************** vertex paint operator ******************* */
1688
1689 /* Implementation notes:
1690
1691 Operator->invoke()
1692   - validate context (add mcol)
1693   - create customdata storage
1694   - call paint once (mouse click)
1695   - add modal handler
1696
1697 Operator->modal()
1698   - for every mousemove, apply vertex paint
1699   - exit on mouse release, free customdata
1700     (return OPERATOR_FINISHED also removes handler and operator)
1701
1702 For future:
1703   - implement a stroke event (or mousemove with past positons)
1704   - revise whether op->customdata should be added in object, in set_vpaint
1705
1706 */
1707
1708 struct VPaintData {
1709         ViewContext vc;
1710         unsigned int paintcol;
1711         int *indexar;
1712         float *vertexcosnos;
1713         float vpimat[3][3];
1714 };
1715
1716 static void vpaint_exit(bContext *C, wmOperator *op)
1717 {
1718         ToolSettings *ts= CTX_data_tool_settings(C);
1719         struct VPaintData *vpd= op->customdata;
1720
1721         if(vpd->vertexcosnos)
1722                 MEM_freeN(vpd->vertexcosnos);
1723         MEM_freeN(vpd->indexar);
1724
1725         /* frees prev buffer */
1726         copy_vpaint_prev(ts->vpaint, NULL, 0);
1727
1728         MEM_freeN(vpd);
1729         op->customdata= NULL;
1730 }
1731
1732 static int vpaint_modal(bContext *C, wmOperator *op, wmEvent *event)
1733 {
1734         ToolSettings *ts= CTX_data_tool_settings(C);
1735         VPaint *vp= ts->vpaint;
1736
1737         switch(event->type) {
1738                 case LEFTMOUSE:
1739                         if(event->val==0) { /* release */
1740                                 vpaint_exit(C, op);
1741                                 return OPERATOR_FINISHED;
1742                         }
1743                         /* pass on, first press gets painted too */
1744
1745                 case MOUSEMOVE:
1746                 {
1747                         struct VPaintData *vpd= op->customdata;
1748                         ViewContext *vc= &vpd->vc;
1749                         Object *ob= vc->obact;
1750                         Mesh *me= ob->data;
1751                         float mat[4][4];
1752                         int *indexar= vpd->indexar;
1753                         int totindex, index;
1754                         short mval[2];
1755
1756                         view3d_operator_needs_opengl(C);
1757
1758                         /* load projection matrix */
1759                         wmMultMatrix(ob->obmat);
1760                         wmGetSingleMatrix(mat);
1761                         wmLoadMatrix(vc->rv3d->viewmat);
1762
1763                         mval[0]= event->x - vc->ar->winrct.xmin;
1764                         mval[1]= event->y - vc->ar->winrct.ymin;
1765
1766                         /* which faces are involved */
1767                         if(vp->flag & VP_AREA) {
1768                                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], vp->size);
1769                         }
1770                         else {
1771                                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1772                                 if(indexar[0]) totindex= 1;
1773                                 else totindex= 0;
1774                         }
1775
1776                         MTC_Mat4SwapMat4(vc->rv3d->persmat, mat);
1777
1778                         if(vp->flag & VP_COLINDEX) {
1779                                 for(index=0; index<totindex; index++) {
1780                                         if(indexar[index] && indexar[index]<=me->totface) {
1781                                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1782
1783                                                 if(mface->mat_nr!=ob->actcol-1) {
1784                                                         indexar[index]= 0;
1785                                                 }
1786                                         }
1787                                 }
1788                         }
1789                         if((G.f & G_FACESELECT) && me->mface) {
1790                                 for(index=0; index<totindex; index++) {
1791                                         if(indexar[index] && indexar[index]<=me->totface) {
1792                                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1793
1794                                                 if((mface->flag & ME_FACE_SEL)==0)
1795                                                         indexar[index]= 0;
1796                                         }
1797                                 }
1798                         }
1799
1800                         for(index=0; index<totindex; index++) {
1801
1802                                 if(indexar[index] && indexar[index]<=me->totface) {
1803                                         MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1804                                         unsigned int *mcol=       ( (unsigned int *)me->mcol) + 4*(indexar[index]-1);
1805                                         unsigned int *mcolorig= ( (unsigned int *)vp->vpaint_prev) + 4*(indexar[index]-1);
1806                                         int alpha;
1807
1808                                         if(vp->mode==VP_BLUR) {
1809                                                 unsigned int fcol1= mcol_blend( mcol[0], mcol[1], 128);
1810                                                 if(mface->v4) {
1811                                                         unsigned int fcol2= mcol_blend( mcol[2], mcol[3], 128);
1812                                                         vpd->paintcol= mcol_blend( fcol1, fcol2, 128);
1813                                                 }
1814                                                 else {
1815                                                         vpd->paintcol= mcol_blend( mcol[2], fcol1, 170);
1816                                                 }
1817
1818                                         }
1819
1820                                         alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*mface->v1, mval);
1821                                         if(alpha) vpaint_blend(vp, mcol, mcolorig, vpd->paintcol, alpha);
1822
1823                                         alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*mface->v2, mval);
1824                                         if(alpha) vpaint_blend(vp, mcol+1, mcolorig+1, vpd->paintcol, alpha);
1825
1826                                         alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*mface->v3, mval);
1827                                         if(alpha) vpaint_blend(vp, mcol+2, mcolorig+2, vpd->paintcol, alpha);
1828
1829                                         if(mface->v4) {
1830                                                 alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*mface->v4, mval);
1831                                                 if(alpha) vpaint_blend(vp, mcol+3, mcolorig+3, vpd->paintcol, alpha);
1832                                         }
1833                                 }
1834                         }
1835
1836                         MTC_Mat4SwapMat4(vc->rv3d->persmat, mat);
1837
1838                         do_shared_vertexcol(me);
1839
1840                         ED_region_tag_redraw(vc->ar);
1841
1842                         DAG_object_flush_update(vc->scene, ob, OB_RECALC_DATA);
1843                 }
1844                 break;
1845         }
1846
1847         return OPERATOR_RUNNING_MODAL;
1848 }
1849
1850 static int vpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1851 {
1852         ToolSettings *ts= CTX_data_tool_settings(C);
1853         VPaint *vp= ts->vpaint;
1854         struct VPaintData *vpd;
1855         Object *ob= CTX_data_active_object(C);
1856         Mesh *me;
1857         float mat[4][4], imat[4][4];
1858
1859         /* context checks could be a poll() */
1860         me= get_mesh(ob);
1861         if(me==NULL || me->totface==0) return OPERATOR_CANCELLED;
1862
1863         if(me->mcol==NULL) make_vertexcol(CTX_data_scene(C), 0);
1864         if(me->mcol==NULL) return OPERATOR_CANCELLED;
1865
1866         /* make customdata storage */
1867         op->customdata= vpd= MEM_callocN(sizeof(struct VPaintData), "VPaintData");
1868         view3d_set_viewcontext(C, &vpd->vc);
1869
1870         vpd->vertexcosnos= mesh_get_mapped_verts_nors(vpd->vc.scene, ob);
1871         vpd->indexar= get_indexarray();
1872         vpd->paintcol= vpaint_get_current_col(vp);
1873
1874         /* for filtering */
1875         copy_vpaint_prev(vp, (unsigned int *)me->mcol, me->totface);
1876
1877         /* some old cruft to sort out later */
1878         Mat4MulMat4(mat, ob->obmat, vpd->vc.rv3d->viewmat);
1879         Mat4Invert(imat, mat);
1880         Mat3CpyMat4(vpd->vpimat, imat);
1881
1882         /* do paint once for click only paint */
1883         vpaint_modal(C, op, event);
1884
1885         /* add modal handler */
1886         WM_event_add_modal_handler(C, &CTX_wm_window(C)->handlers, op);
1887
1888         return OPERATOR_RUNNING_MODAL;
1889 }
1890
1891 void PAINT_OT_vertex_paint(wmOperatorType *ot)
1892 {
1893         /* identifiers */
1894         ot->name= "Vertex Paint";
1895         ot->idname= "PAINT_OT_vertex_paint";
1896
1897         /* api callbacks */
1898         ot->invoke= vpaint_invoke;
1899         ot->modal= vpaint_modal;
1900         /* ot->exec= vpaint_exec; <-- needs stroke property */
1901         ot->poll= vp_poll;
1902
1903         /* flags */
1904         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1905 }
1906