Bugfix: weight paint combined with pose mode was not working correct
[blender.git] / source / blender / editors / sculpt_paint / paint_vertex.c
1 /**
2  * $Id$
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  */
29
30 #include <math.h>
31 #include <string.h>
32
33 #ifdef WIN32
34 #include <io.h>
35 #else
36 #include <unistd.h>
37 #endif   
38
39 #include "MEM_guardedalloc.h"
40
41 #include "IMB_imbuf.h"
42 #include "IMB_imbuf_types.h"
43
44 #include "BLI_blenlib.h"
45 #include "BLI_math.h"
46 #include "BLI_ghash.h"
47
48 #include "DNA_anim_types.h"
49 #include "DNA_action_types.h"
50 #include "DNA_armature_types.h"
51 #include "DNA_brush_types.h"
52 #include "DNA_cloth_types.h"
53 #include "DNA_mesh_types.h"
54 #include "DNA_meshdata_types.h"
55 #include "DNA_modifier_types.h"
56 #include "DNA_object_types.h"
57 #include "DNA_object_force.h"
58 #include "DNA_particle_types.h"
59 #include "DNA_screen_types.h"
60 #include "DNA_space_types.h"
61 #include "DNA_scene_types.h"
62 #include "DNA_view3d_types.h"
63 #include "DNA_userdef_types.h"
64
65 #include "RNA_access.h"
66 #include "RNA_define.h"
67
68 #include "BKE_armature.h"
69 #include "BKE_brush.h"
70 #include "BKE_DerivedMesh.h"
71 #include "BKE_cloth.h"
72 #include "BKE_context.h"
73 #include "BKE_customdata.h"
74 #include "BKE_depsgraph.h"
75 #include "BKE_deform.h"
76 #include "BKE_displist.h"
77 #include "BKE_global.h"
78 #include "BKE_mesh.h"
79 #include "BKE_modifier.h"
80 #include "BKE_object.h"
81 #include "BKE_paint.h"
82 #include "BKE_utildefines.h"
83
84 #include "WM_api.h"
85 #include "WM_types.h"
86
87 #include "BIF_gl.h"
88 #include "BIF_glutil.h"
89
90 #include "ED_mesh.h"
91 #include "ED_object.h"
92 #include "ED_screen.h"
93 #include "ED_util.h"
94 #include "ED_view3d.h"
95
96 #include "paint_intern.h"
97
98         /* vp->mode */
99 #define VP_MIX  0
100 #define VP_ADD  1
101 #define VP_SUB  2
102 #define VP_MUL  3
103 #define VP_BLUR 4
104 #define VP_LIGHTEN      5
105 #define VP_DARKEN       6
106
107 #define MAXINDEX        512000
108
109 /* XXX */
110 static void error() {}
111
112 /* polling - retrieve whether cursor should be set or operator should be done */
113
114
115 /* Returns true if vertex paint mode is active */
116 int vertex_paint_mode_poll(bContext *C)
117 {
118         Object *ob = CTX_data_active_object(C);
119
120         return ob && ob->mode == OB_MODE_VERTEX_PAINT;
121 }
122
123 int vertex_paint_poll(bContext *C)
124 {
125         if(vertex_paint_mode_poll(C) && 
126            paint_brush(&CTX_data_tool_settings(C)->vpaint->paint)) {
127                 ScrArea *sa= CTX_wm_area(C);
128                 if(sa->spacetype==SPACE_VIEW3D) {
129                         ARegion *ar= CTX_wm_region(C);
130                         if(ar->regiontype==RGN_TYPE_WINDOW)
131                                 return 1;
132                 }
133         }
134         return 0;
135 }
136
137 int weight_paint_poll(bContext *C)
138 {
139         Object *ob = CTX_data_active_object(C);
140
141         if(ob && ob->mode & OB_MODE_WEIGHT_PAINT &&
142            paint_brush(&CTX_data_tool_settings(C)->wpaint->paint)) {
143                 ScrArea *sa= CTX_wm_area(C);
144                 if(sa->spacetype==SPACE_VIEW3D) {
145                         ARegion *ar= CTX_wm_region(C);
146                         if(ar->regiontype==RGN_TYPE_WINDOW)
147                                 return 1;
148                 }
149         }
150         return 0;
151 }
152
153 static VPaint *new_vpaint(int wpaint)
154 {
155         VPaint *vp= MEM_callocN(sizeof(VPaint), "VPaint");
156         
157         vp->flag= VP_AREA+VP_SPRAY;
158         
159         if(wpaint)
160                 vp->flag= VP_AREA;
161
162         return vp;
163 }
164
165 static int *get_indexarray(void)
166 {
167         return MEM_mallocN(sizeof(int)*MAXINDEX + 2, "vertexpaint");
168 }
169
170
171 /* in contradiction to cpack drawing colors, the MCOL colors (vpaint colors) are per byte! 
172    so not endian sensitive. Mcol = ABGR!!! so be cautious with cpack calls */
173
174 unsigned int rgba_to_mcol(float r, float g, float b, float a)
175 {
176         int ir, ig, ib, ia;
177         unsigned int col;
178         char *cp;
179         
180         ir= floor(255.0*r);
181         if(ir<0) ir= 0; else if(ir>255) ir= 255;
182         ig= floor(255.0*g);
183         if(ig<0) ig= 0; else if(ig>255) ig= 255;
184         ib= floor(255.0*b);
185         if(ib<0) ib= 0; else if(ib>255) ib= 255;
186         ia= floor(255.0*a);
187         if(ia<0) ia= 0; else if(ia>255) ia= 255;
188         
189         cp= (char *)&col;
190         cp[0]= ia;
191         cp[1]= ib;
192         cp[2]= ig;
193         cp[3]= ir;
194         
195         return col;
196         
197 }
198
199 unsigned int vpaint_get_current_col(VPaint *vp)
200 {
201         Brush *brush = paint_brush(&vp->paint);
202         return rgba_to_mcol(brush->rgb[0], brush->rgb[1], brush->rgb[2], 1.0f);
203 }
204
205 static void do_shared_vertexcol(Mesh *me)
206 {
207         /* if no mcol: do not do */
208         /* if tface: only the involved faces, otherwise all */
209         MFace *mface;
210         MTFace *tface;
211         int a;
212         short *scolmain, *scol;
213         char *mcol;
214         
215         if(me->mcol==0 || me->totvert==0 || me->totface==0) return;
216         
217         scolmain= MEM_callocN(4*sizeof(short)*me->totvert, "colmain");
218         
219         tface= me->mtface;
220         mface= me->mface;
221         mcol= (char *)me->mcol;
222         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
223                 if((tface && tface->mode & TF_SHAREDCOL) || (me->editflag & ME_EDIT_PAINT_MASK)==0) {
224                         scol= scolmain+4*mface->v1;
225                         scol[0]++; scol[1]+= mcol[1]; scol[2]+= mcol[2]; scol[3]+= mcol[3];
226                         scol= scolmain+4*mface->v2;
227                         scol[0]++; scol[1]+= mcol[5]; scol[2]+= mcol[6]; scol[3]+= mcol[7];
228                         scol= scolmain+4*mface->v3;
229                         scol[0]++; scol[1]+= mcol[9]; scol[2]+= mcol[10]; scol[3]+= mcol[11];
230                         if(mface->v4) {
231                                 scol= scolmain+4*mface->v4;
232                                 scol[0]++; scol[1]+= mcol[13]; scol[2]+= mcol[14]; scol[3]+= mcol[15];
233                         }
234                 }
235                 if(tface) tface++;
236         }
237         
238         a= me->totvert;
239         scol= scolmain;
240         while(a--) {
241                 if(scol[0]>1) {
242                         scol[1]/= scol[0];
243                         scol[2]/= scol[0];
244                         scol[3]/= scol[0];
245                 }
246                 scol+= 4;
247         }
248         
249         tface= me->mtface;
250         mface= me->mface;
251         mcol= (char *)me->mcol;
252         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
253                 if((tface && tface->mode & TF_SHAREDCOL) || (me->editflag & ME_EDIT_PAINT_MASK)==0) {
254                         scol= scolmain+4*mface->v1;
255                         mcol[1]= scol[1]; mcol[2]= scol[2]; mcol[3]= scol[3];
256                         scol= scolmain+4*mface->v2;
257                         mcol[5]= scol[1]; mcol[6]= scol[2]; mcol[7]= scol[3];
258                         scol= scolmain+4*mface->v3;
259                         mcol[9]= scol[1]; mcol[10]= scol[2]; mcol[11]= scol[3];
260                         if(mface->v4) {
261                                 scol= scolmain+4*mface->v4;
262                                 mcol[13]= scol[1]; mcol[14]= scol[2]; mcol[15]= scol[3];
263                         }
264                 }
265                 if(tface) tface++;
266         }
267
268         MEM_freeN(scolmain);
269 }
270
271 static void make_vertexcol(Object *ob)  /* single ob */
272 {
273         Mesh *me;
274         if(!ob || ob->id.lib) return;
275         me= get_mesh(ob);
276         if(me==0) return;
277         if(me->edit_mesh) return;
278
279         /* copies from shadedisplist to mcol */
280         if(!me->mcol) {
281                 CustomData_add_layer(&me->fdata, CD_MCOL, CD_CALLOC, NULL, me->totface);
282                 mesh_update_customdata_pointers(me);
283         }
284
285         //if(shade)
286         //      shadeMeshMCol(scene, ob, me);
287         //else
288
289         memset(me->mcol, 255, 4*sizeof(MCol)*me->totface);
290         
291         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
292         
293 }
294
295 static void copy_vpaint_prev(VPaint *vp, unsigned int *mcol, int tot)
296 {
297         if(vp->vpaint_prev) {
298                 MEM_freeN(vp->vpaint_prev);
299                 vp->vpaint_prev= NULL;
300         }
301         vp->tot= tot;   
302         
303         if(mcol==NULL || tot==0) return;
304         
305         vp->vpaint_prev= MEM_mallocN(4*sizeof(int)*tot, "vpaint_prev");
306         memcpy(vp->vpaint_prev, mcol, 4*sizeof(int)*tot);
307         
308 }
309
310 static void copy_wpaint_prev (VPaint *wp, MDeformVert *dverts, int dcount)
311 {
312         if (wp->wpaint_prev) {
313                 free_dverts(wp->wpaint_prev, wp->tot);
314                 wp->wpaint_prev= NULL;
315         }
316         
317         if(dverts && dcount) {
318                 
319                 wp->wpaint_prev = MEM_mallocN (sizeof(MDeformVert)*dcount, "wpaint prev");
320                 wp->tot = dcount;
321                 copy_dverts (wp->wpaint_prev, dverts, dcount);
322         }
323 }
324
325
326 void vpaint_fill(Object *ob, unsigned int paintcol)
327 {
328         Mesh *me;
329         MFace *mf;
330         unsigned int *mcol;
331         int i, selected;
332
333         me= get_mesh(ob);
334         if(me==0 || me->totface==0) return;
335
336         if(!me->mcol)
337                 make_vertexcol(ob);
338
339         selected= (me->editflag & ME_EDIT_PAINT_MASK);
340
341         mf = me->mface;
342         mcol = (unsigned int*)me->mcol;
343         for (i = 0; i < me->totface; i++, mf++, mcol+=4) {
344                 if (!selected || mf->flag & ME_FACE_SEL) {
345                         mcol[0] = paintcol;
346                         mcol[1] = paintcol;
347                         mcol[2] = paintcol;
348                         mcol[3] = paintcol;
349                 }
350         }
351         
352         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
353 }
354
355
356 /* fills in the selected faces with the current weight and vertex group */
357 void wpaint_fill(VPaint *wp, Object *ob, float paintweight)
358 {
359         Mesh *me;
360         MFace *mface;
361         MDeformWeight *dw, *uw;
362         int *indexar;
363         int index, vgroup;
364         unsigned int faceverts[5]={0,0,0,0,0};
365         unsigned char i;
366         int vgroup_mirror= -1;
367         int selected;
368         
369         me= ob->data;
370         if(me==0 || me->totface==0 || me->dvert==0 || !me->mface) return;
371         
372         selected= (me->editflag & ME_EDIT_PAINT_MASK);
373
374         indexar= get_indexarray();
375         if(selected) {
376                 for(index=0, mface=me->mface; index<me->totface; index++, mface++) {
377                         if((mface->flag & ME_FACE_SEL)==0)
378                                 indexar[index]= 0;
379                         else
380                                 indexar[index]= index+1;
381                 }
382         }
383         else {
384                 for(index=0; index<me->totface; index++)
385                         indexar[index]= index+1;
386         }
387         
388         vgroup= ob->actdef-1;
389         
390         /* directly copied from weight_paint, should probaby split into a separate function */
391         /* if mirror painting, find the other group */          
392         if(me->editflag & ME_EDIT_MIRROR_X) {
393                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
394                 if(defgroup) {
395                         bDeformGroup *curdef;
396                         int actdef= 0;
397                         char name[32];
398
399                         BLI_strncpy(name, defgroup->name, 32);
400                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
401                         
402                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
403                                 if (!strcmp(curdef->name, name))
404                                         break;
405                         if(curdef==NULL) {
406                                 int olddef= ob->actdef; /* tsk, ED_vgroup_add sets the active defgroup */
407                                 curdef= ED_vgroup_add_name (ob, name);
408                                 ob->actdef= olddef;
409                         }
410                         
411                         if(curdef && curdef!=defgroup)
412                                 vgroup_mirror= actdef;
413                 }
414         }
415         /* end copy from weight_paint*/
416         
417         copy_wpaint_prev(wp, me->dvert, me->totvert);
418         
419         for(index=0; index<me->totface; index++) {
420                 if(indexar[index] && indexar[index]<=me->totface) {
421                         mface= me->mface + (indexar[index]-1);
422                         /* just so we can loop through the verts */
423                         faceverts[0]= mface->v1;
424                         faceverts[1]= mface->v2;
425                         faceverts[2]= mface->v3;
426                         faceverts[3]= mface->v4;
427                         for (i=0; i<3 || faceverts[i]; i++) {
428                                 if(!((me->dvert+faceverts[i])->flag)) {
429                                         dw= ED_vgroup_weight_verify(me->dvert+faceverts[i], vgroup);
430                                         if(dw) {
431                                                 uw= ED_vgroup_weight_verify(wp->wpaint_prev+faceverts[i], vgroup);
432                                                 uw->weight= dw->weight; /* set the undo weight */
433                                                 dw->weight= paintweight;
434                                                 
435                                                 if(me->editflag & ME_EDIT_MIRROR_X) {   /* x mirror painting */
436                                                         int j= mesh_get_x_mirror_vert(ob, faceverts[i]);
437                                                         if(j>=0) {
438                                                                 /* copy, not paint again */
439                                                                 if(vgroup_mirror != -1) {
440                                                                         dw= ED_vgroup_weight_verify(me->dvert+j, vgroup_mirror);
441                                                                         uw= ED_vgroup_weight_verify(wp->wpaint_prev+j, vgroup_mirror);
442                                                                 } else {
443                                                                         dw= ED_vgroup_weight_verify(me->dvert+j, vgroup);
444                                                                         uw= ED_vgroup_weight_verify(wp->wpaint_prev+j, vgroup);
445                                                                 }
446                                                                 uw->weight= dw->weight; /* set the undo weight */
447                                                                 dw->weight= paintweight;
448                                                         }
449                                                 }
450                                         }
451                                         (me->dvert+faceverts[i])->flag= 1;
452                                 }
453                         }
454                 }
455         }
456         
457         index=0;
458         while (index<me->totvert) {
459                 (me->dvert+index)->flag= 0;
460                 index++;
461         }
462         
463         MEM_freeN(indexar);
464         copy_wpaint_prev(wp, NULL, 0);
465
466         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
467 }
468
469 /* XXX: should be re-implemented as a vertex/weight paint 'colour correct' operator
470  
471 void vpaint_dogamma(Scene *scene)
472 {
473         VPaint *vp= scene->toolsettings->vpaint;
474         Mesh *me;
475         Object *ob;
476         float igam, fac;
477         int a, temp;
478         unsigned char *cp, gamtab[256];
479
480         ob= OBACT;
481         me= get_mesh(ob);
482
483         if(!(ob->mode & OB_MODE_VERTEX_PAINT)) return;
484         if(me==0 || me->mcol==0 || me->totface==0) return;
485
486         igam= 1.0/vp->gamma;
487         for(a=0; a<256; a++) {
488                 
489                 fac= ((float)a)/255.0;
490                 fac= vp->mul*pow( fac, igam);
491                 
492                 temp= 255.9*fac;
493                 
494                 if(temp<=0) gamtab[a]= 0;
495                 else if(temp>=255) gamtab[a]= 255;
496                 else gamtab[a]= temp;
497         }
498
499         a= 4*me->totface;
500         cp= (unsigned char *)me->mcol;
501         while(a--) {
502                 
503                 cp[1]= gamtab[ cp[1] ];
504                 cp[2]= gamtab[ cp[2] ];
505                 cp[3]= gamtab[ cp[3] ];
506                 
507                 cp+= 4;
508         }
509 }
510  */
511
512 static unsigned int mcol_blend(unsigned int col1, unsigned int col2, int fac)
513 {
514         char *cp1, *cp2, *cp;
515         int mfac;
516         unsigned int col=0;
517         
518         if(fac==0) return col1;
519         if(fac>=255) return col2;
520
521         mfac= 255-fac;
522         
523         cp1= (char *)&col1;
524         cp2= (char *)&col2;
525         cp=  (char *)&col;
526         
527         cp[0]= 255;
528         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
529         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
530         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
531         
532         return col;
533 }
534
535 static unsigned int mcol_add(unsigned int col1, unsigned int col2, int fac)
536 {
537         char *cp1, *cp2, *cp;
538         int temp;
539         unsigned int col=0;
540         
541         if(fac==0) return col1;
542         
543         cp1= (char *)&col1;
544         cp2= (char *)&col2;
545         cp=  (char *)&col;
546         
547         cp[0]= 255;
548         temp= cp1[1] + ((fac*cp2[1])/255);
549         if(temp>254) cp[1]= 255; else cp[1]= temp;
550         temp= cp1[2] + ((fac*cp2[2])/255);
551         if(temp>254) cp[2]= 255; else cp[2]= temp;
552         temp= cp1[3] + ((fac*cp2[3])/255);
553         if(temp>254) cp[3]= 255; else cp[3]= temp;
554         
555         return col;
556 }
557
558 static unsigned int mcol_sub(unsigned int col1, unsigned int col2, int fac)
559 {
560         char *cp1, *cp2, *cp;
561         int temp;
562         unsigned int col=0;
563         
564         if(fac==0) return col1;
565         
566         cp1= (char *)&col1;
567         cp2= (char *)&col2;
568         cp=  (char *)&col;
569         
570         cp[0]= 255;
571         temp= cp1[1] - ((fac*cp2[1])/255);
572         if(temp<0) cp[1]= 0; else cp[1]= temp;
573         temp= cp1[2] - ((fac*cp2[2])/255);
574         if(temp<0) cp[2]= 0; else cp[2]= temp;
575         temp= cp1[3] - ((fac*cp2[3])/255);
576         if(temp<0) cp[3]= 0; else cp[3]= temp;
577         
578         return col;
579 }
580
581 static unsigned int mcol_mul(unsigned int col1, unsigned int col2, int fac)
582 {
583         char *cp1, *cp2, *cp;
584         int mfac;
585         unsigned int col=0;
586         
587         if(fac==0) return col1;
588
589         mfac= 255-fac;
590         
591         cp1= (char *)&col1;
592         cp2= (char *)&col2;
593         cp=  (char *)&col;
594         
595         /* first mul, then blend the fac */
596         cp[0]= 255;
597         cp[1]= (mfac*cp1[1] + fac*((cp2[1]*cp1[1])/255)  )/255;
598         cp[2]= (mfac*cp1[2] + fac*((cp2[2]*cp1[2])/255)  )/255;
599         cp[3]= (mfac*cp1[3] + fac*((cp2[3]*cp1[3])/255)  )/255;
600
601         
602         return col;
603 }
604
605 static unsigned int mcol_lighten(unsigned int col1, unsigned int col2, int fac)
606 {
607         char *cp1, *cp2, *cp;
608         int mfac;
609         unsigned int col=0;
610         
611         if(fac==0) return col1;
612         if(fac>=255) return col2;
613
614         mfac= 255-fac;
615         
616         cp1= (char *)&col1;
617         cp2= (char *)&col2;
618         cp=  (char *)&col;
619         
620         /* See if are lighter, if so mix, else dont do anything.
621         if the paint col is darker then the original, then ignore */
622         if (cp1[1]+cp1[2]+cp1[3] > cp2[1]+cp2[2]+cp2[3])
623                 return col1;
624         
625         cp[0]= 255;
626         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
627         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
628         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
629         
630         return col;
631 }
632
633 static unsigned int mcol_darken(unsigned int col1, unsigned int col2, int fac)
634 {
635         char *cp1, *cp2, *cp;
636         int mfac;
637         unsigned int col=0;
638         
639         if(fac==0) return col1;
640         if(fac>=255) return col2;
641
642         mfac= 255-fac;
643         
644         cp1= (char *)&col1;
645         cp2= (char *)&col2;
646         cp=  (char *)&col;
647         
648         /* See if were darker, if so mix, else dont do anything.
649         if the paint col is brighter then the original, then ignore */
650         if (cp1[1]+cp1[2]+cp1[3] < cp2[1]+cp2[2]+cp2[3])
651                 return col1;
652         
653         cp[0]= 255;
654         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
655         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
656         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
657         return col;
658 }
659
660 static void vpaint_blend(VPaint *vp, unsigned int *col, unsigned int *colorig, unsigned int paintcol, int alpha)
661 {
662         Brush *brush = paint_brush(&vp->paint);
663
664         if(vp->mode==VP_MIX || vp->mode==VP_BLUR) *col= mcol_blend( *col, paintcol, alpha);
665         else if(vp->mode==VP_ADD) *col= mcol_add( *col, paintcol, alpha);
666         else if(vp->mode==VP_SUB) *col= mcol_sub( *col, paintcol, alpha);
667         else if(vp->mode==VP_MUL) *col= mcol_mul( *col, paintcol, alpha);
668         else if(vp->mode==VP_LIGHTEN) *col= mcol_lighten( *col, paintcol, alpha);
669         else if(vp->mode==VP_DARKEN) *col= mcol_darken( *col, paintcol, alpha);
670         
671         /* if no spray, clip color adding with colorig & orig alpha */
672         if((vp->flag & VP_SPRAY)==0) {
673                 unsigned int testcol=0, a;
674                 char *cp, *ct, *co;
675                 
676                 alpha= (int)(255.0*brush->alpha);
677                 
678                 if(vp->mode==VP_MIX || vp->mode==VP_BLUR) testcol= mcol_blend( *colorig, paintcol, alpha);
679                 else if(vp->mode==VP_ADD) testcol= mcol_add( *colorig, paintcol, alpha);
680                 else if(vp->mode==VP_SUB) testcol= mcol_sub( *colorig, paintcol, alpha);
681                 else if(vp->mode==VP_MUL) testcol= mcol_mul( *colorig, paintcol, alpha);
682                 else if(vp->mode==VP_LIGHTEN)  testcol= mcol_lighten( *colorig, paintcol, alpha);
683                 else if(vp->mode==VP_DARKEN)   testcol= mcol_darken( *colorig, paintcol, alpha);
684                 
685                 cp= (char *)col;
686                 ct= (char *)&testcol;
687                 co= (char *)colorig;
688                 
689                 for(a=0; a<4; a++) {
690                         if( ct[a]<co[a] ) {
691                                 if( cp[a]<ct[a] ) cp[a]= ct[a];
692                                 else if( cp[a]>co[a] ) cp[a]= co[a];
693                         }
694                         else {
695                                 if( cp[a]<co[a] ) cp[a]= co[a];
696                                 else if( cp[a]>ct[a] ) cp[a]= ct[a];
697                         }
698                 }
699         }
700 }
701
702
703 static int sample_backbuf_area(ViewContext *vc, int *indexar, int totface, int x, int y, float size)
704 {
705         struct ImBuf *ibuf;
706         int a, tot=0, index;
707         
708         if(totface+4>=MAXINDEX) return 0;
709         
710         /* brecht: disabled this because it obviously failes for
711            brushes with size > 64, why is this here? */
712         /*if(size>64.0) size= 64.0;*/
713         
714         ibuf= view3d_read_backbuf(vc, x-size, y-size, x+size, y+size);
715         if(ibuf) {
716                 unsigned int *rt= ibuf->rect;
717
718                 memset(indexar, 0, sizeof(int)*totface+4);      /* plus 2! first element is total, +2 was giving valgrind errors, +4 seems ok */
719                 
720                 size= ibuf->x*ibuf->y;
721                 while(size--) {
722                                 
723                         if(*rt) {
724                                 index= WM_framebuffer_to_index(*rt);
725                                 if(index>0 && index<=totface)
726                                         indexar[index] = 1;
727                         }
728                 
729                         rt++;
730                 }
731                 
732                 for(a=1; a<=totface; a++) {
733                         if(indexar[a]) indexar[tot++]= a;
734                 }
735
736                 IMB_freeImBuf(ibuf);
737         }
738         
739         return tot;
740 }
741
742 static float calc_vp_alpha_dl(VPaint *vp, ViewContext *vc, float vpimat[][3], float *vert_nor, float *mval, float pressure)
743 {
744         Brush *brush = paint_brush(&vp->paint);
745         float fac, fac_2, size, dx, dy;
746         float alpha;
747         short vertco[2];
748         
749         project_short_noclip(vc->ar, vert_nor, vertco);
750         dx= mval[0]-vertco[0];
751         dy= mval[1]-vertco[1];
752         
753         if (brush->flag & BRUSH_SIZE_PRESSURE)
754                 size = pressure * brush->size;
755         else
756                 size = brush->size;
757         
758         fac_2= dx*dx + dy*dy;
759         if(fac_2 > size*size) return 0.f;
760         fac = sqrtf(fac_2);
761         
762         alpha= brush->alpha * brush_curve_strength_clamp(brush, fac, size);
763         
764         if (brush->flag & BRUSH_ALPHA_PRESSURE)
765                 alpha *= pressure;
766                 
767         if(vp->flag & VP_NORMALS) {
768                 float *no= vert_nor+3;
769                 
770                 /* transpose ! */
771                 fac= vpimat[2][0]*no[0]+vpimat[2][1]*no[1]+vpimat[2][2]*no[2];
772                 if(fac>0.0) {
773                         dx= vpimat[0][0]*no[0]+vpimat[0][1]*no[1]+vpimat[0][2]*no[2];
774                         dy= vpimat[1][0]*no[0]+vpimat[1][1]*no[1]+vpimat[1][2]*no[2];
775                         
776                         alpha*= fac/sqrtf(dx*dx + dy*dy + fac*fac);
777                 }
778                 else return 0.f;
779         }
780         
781         return alpha;
782 }
783
784 static void wpaint_blend(VPaint *wp, MDeformWeight *dw, MDeformWeight *uw, float alpha, float paintval, int flip)
785 {
786         Brush *brush = paint_brush(&wp->paint);
787         int mode = wp->mode;
788         
789         if(dw==NULL || uw==NULL) return;
790         
791         if (flip) {
792                 switch(mode) {
793                         case VP_MIX:
794                                 paintval = 1.f - paintval; break;
795                         case VP_ADD:
796                                 mode= VP_SUB; break;
797                         case VP_SUB:
798                                 mode= VP_ADD; break;
799                         case VP_LIGHTEN:
800                                 mode= VP_DARKEN; break;
801                         case VP_DARKEN:
802                                 mode= VP_LIGHTEN; break;
803                 }
804         }
805         
806         if(mode==VP_MIX || mode==VP_BLUR)
807                 dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
808         else if(mode==VP_ADD)
809                 dw->weight += paintval*alpha;
810         else if(mode==VP_SUB) 
811                 dw->weight -= paintval*alpha;
812         else if(mode==VP_MUL) 
813                 /* first mul, then blend the fac */
814                 dw->weight = ((1.0-alpha) + alpha*paintval)*dw->weight;
815         else if(mode==VP_LIGHTEN) {
816                 if (dw->weight < paintval)
817                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
818         } else if(mode==VP_DARKEN) {
819                 if (dw->weight > paintval)
820                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
821         }
822         CLAMP(dw->weight, 0.0f, 1.0f);
823         
824         /* if no spray, clip result with orig weight & orig alpha */
825         if((wp->flag & VP_SPRAY)==0) {
826                 float testw=0.0f;
827                 
828                 alpha= brush->alpha;
829                 if(mode==VP_MIX || mode==VP_BLUR)
830                         testw = paintval*alpha + uw->weight*(1.0-alpha);
831                 else if(mode==VP_ADD)
832                         testw = uw->weight + paintval*alpha;
833                 else if(mode==VP_SUB) 
834                         testw = uw->weight - paintval*alpha;
835                 else if(mode==VP_MUL) 
836                         /* first mul, then blend the fac */
837                         testw = ((1.0-alpha) + alpha*paintval)*uw->weight;              
838                 else if(mode==VP_LIGHTEN) {
839                         if (uw->weight < paintval)
840                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
841                         else
842                                 testw = uw->weight;
843                 } else if(mode==VP_DARKEN) {
844                         if (uw->weight > paintval)
845                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
846                         else
847                                 testw = uw->weight;
848                 }
849                 CLAMP(testw, 0.0f, 1.0f);
850                 
851                 if( testw<uw->weight ) {
852                         if(dw->weight < testw) dw->weight= testw;
853                         else if(dw->weight > uw->weight) dw->weight= uw->weight;
854                 }
855                 else {
856                         if(dw->weight > testw) dw->weight= testw;
857                         else if(dw->weight < uw->weight) dw->weight= uw->weight;
858                 }
859         }
860         
861 }
862
863 /* ----------------------------------------------------- */
864
865 /* used for 3d view, on active object, assumes me->dvert exists */
866 /* if mode==1: */
867 /*     samples cursor location, and gives menu with vertex groups to activate */
868 /* else */
869 /*     sets wp->weight to the closest weight value to vertex */
870 /*     note: we cant sample frontbuf, weight colors are interpolated too unpredictable */
871 void sample_wpaint(Scene *scene, ARegion *ar, View3D *v3d, int mode)
872 {
873         ViewContext vc;
874         ToolSettings *ts= scene->toolsettings;
875         Object *ob= OBACT;
876         Mesh *me= get_mesh(ob);
877         int index;
878         short mval[2] = {0, 0}, sco[2];
879
880         if (!me) return;
881         
882 //      getmouseco_areawin(mval);
883         index= view3d_sample_backbuf(&vc, mval[0], mval[1]);
884         
885         if(index && index<=me->totface) {
886                 MFace *mface;
887                 
888                 mface= ((MFace *)me->mface) + index-1;
889                 
890                 if(mode==1) {   /* sampe which groups are in here */
891                         MDeformVert *dv;
892                         int a, totgroup;
893                         
894                         totgroup= BLI_countlist(&ob->defbase);
895                         if(totgroup) {
896                                 int totmenu=0;
897                                 int *groups=MEM_callocN(totgroup*sizeof(int), "groups");
898                                 
899                                 dv= me->dvert+mface->v1;
900                                 for(a=0; a<dv->totweight; a++) {
901                                         if (dv->dw[a].def_nr<totgroup)
902                                                 groups[dv->dw[a].def_nr]= 1;
903                                 }
904                                 dv= me->dvert+mface->v2;
905                                 for(a=0; a<dv->totweight; a++) {
906                                         if (dv->dw[a].def_nr<totgroup)
907                                                 groups[dv->dw[a].def_nr]= 1;
908                                 }
909                                 dv= me->dvert+mface->v3;
910                                 for(a=0; a<dv->totweight; a++) {
911                                         if (dv->dw[a].def_nr<totgroup)
912                                                 groups[dv->dw[a].def_nr]= 1;
913                                 }
914                                 if(mface->v4) {
915                                         dv= me->dvert+mface->v4;
916                                         for(a=0; a<dv->totweight; a++) {
917                                                 if (dv->dw[a].def_nr<totgroup)
918                                                         groups[dv->dw[a].def_nr]= 1;
919                                         }
920                                 }
921                                 for(a=0; a<totgroup; a++)
922                                         if(groups[a]) totmenu++;
923                                 
924                                 if(totmenu==0) {
925                                         //notice("No Vertex Group Selected");
926                                 }
927                                 else {
928                                         bDeformGroup *dg;
929                                         short val;
930                                         char item[40], *str= MEM_mallocN(40*totmenu+40, "menu");
931                                         
932                                         strcpy(str, "Vertex Groups %t");
933                                         for(a=0, dg=ob->defbase.first; dg && a<totgroup; a++, dg= dg->next) {
934                                                 if(groups[a]) {
935                                                         sprintf(item, "|%s %%x%d", dg->name, a);
936                                                         strcat(str, item);
937                                                 }
938                                         }
939                                         
940                                         val= 0; // XXX pupmenu(str);
941                                         if(val>=0) {
942                                                 ob->actdef= val+1;
943                                                 DAG_id_flush_update(&me->id, OB_RECALC_DATA);
944                                         }
945                                         MEM_freeN(str);
946                                 }
947                                 MEM_freeN(groups);
948                         }
949 //                      else notice("No Vertex Groups in Object");
950                 }
951                 else {
952                         DerivedMesh *dm;
953                         MDeformWeight *dw;
954                         float w1, w2, w3, w4, co[3], fac;
955                         
956                         dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
957                         if(dm->getVertCo==NULL) {
958                                 //notice("Not supported yet");
959                         }
960                         else {
961                                 /* calc 3 or 4 corner weights */
962                                 dm->getVertCo(dm, mface->v1, co);
963                                 project_short_noclip(ar, co, sco);
964                                 w1= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
965                                 
966                                 dm->getVertCo(dm, mface->v2, co);
967                                 project_short_noclip(ar, co, sco);
968                                 w2= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
969                                 
970                                 dm->getVertCo(dm, mface->v3, co);
971                                 project_short_noclip(ar, co, sco);
972                                 w3= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
973                                 
974                                 if(mface->v4) {
975                                         dm->getVertCo(dm, mface->v4, co);
976                                         project_short_noclip(ar, co, sco);
977                                         w4= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
978                                 }
979                                 else w4= 1.0e10;
980                                 
981                                 fac= MIN4(w1, w2, w3, w4);
982                                 if(w1==fac) {
983                                         dw= ED_vgroup_weight_get(me->dvert+mface->v1, ob->actdef-1);
984                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
985                                 }
986                                 else if(w2==fac) {
987                                         dw= ED_vgroup_weight_get(me->dvert+mface->v2, ob->actdef-1);
988                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
989                                 }
990                                 else if(w3==fac) {
991                                         dw= ED_vgroup_weight_get(me->dvert+mface->v3, ob->actdef-1);
992                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
993                                 }
994                                 else if(w4==fac) {
995                                         if(mface->v4) {
996                                                 dw= ED_vgroup_weight_get(me->dvert+mface->v4, ob->actdef-1);
997                                                 if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
998                                         }
999                                 }
1000                         }
1001                         dm->release(dm);
1002                 }               
1003                 
1004         }
1005         
1006 }
1007
1008 static void do_weight_paint_auto_normalize(MDeformVert *dvert, 
1009                                            int paint_nr, char *map)
1010 {
1011 //      MDeformWeight *dw = dvert->dw;
1012         float sum=0.0f, fac=0.0f, paintw=0.0f;
1013         int i, tot=0;
1014
1015         if (!map)
1016                 return;
1017
1018         for (i=0; i<dvert->totweight; i++) {
1019                 if (dvert->dw[i].def_nr == paint_nr)
1020                         paintw = dvert->dw[i].weight;
1021
1022                 if (map[dvert->dw[i].def_nr]) {
1023                         tot += 1;
1024                         if (dvert->dw[i].def_nr != paint_nr)
1025                                 sum += dvert->dw[i].weight;
1026                 }
1027         }
1028         
1029         if (!tot || sum <= (1.0f - paintw))
1030                 return;
1031
1032         fac = sum / (1.0f - paintw);
1033         fac = fac==0.0f ? 1.0f : 1.0f / fac;
1034
1035         for (i=0; i<dvert->totweight; i++) {
1036                 if (map[dvert->dw[i].def_nr]) {
1037                         if (dvert->dw[i].def_nr != paint_nr)
1038                                 dvert->dw[i].weight *= fac;
1039                 }
1040         }
1041 }
1042
1043 static void do_weight_paint_vertex(VPaint *wp, Object *ob, int index, 
1044                                    float alpha, float paintweight, int flip, 
1045                                    int vgroup_mirror, char *validmap)
1046 {
1047         Mesh *me= ob->data;
1048         MDeformWeight *dw, *uw;
1049         int vgroup= ob->actdef-1;
1050         
1051         if(wp->flag & VP_ONLYVGROUP) {
1052                 dw= ED_vgroup_weight_get(me->dvert+index, vgroup);
1053                 uw= ED_vgroup_weight_get(wp->wpaint_prev+index, vgroup);
1054         }
1055         else {
1056                 dw= ED_vgroup_weight_verify(me->dvert+index, vgroup);
1057                 uw= ED_vgroup_weight_verify(wp->wpaint_prev+index, vgroup);
1058         }
1059         if(dw==NULL || uw==NULL)
1060                 return;
1061         
1062         wpaint_blend(wp, dw, uw, alpha, paintweight, flip);
1063         do_weight_paint_auto_normalize(me->dvert+index, vgroup, validmap);
1064
1065         if(me->editflag & ME_EDIT_MIRROR_X) {   /* x mirror painting */
1066                 int j= mesh_get_x_mirror_vert(ob, index);
1067                 if(j>=0) {
1068                         /* copy, not paint again */
1069                         if(vgroup_mirror != -1)
1070                                 uw= ED_vgroup_weight_verify(me->dvert+j, vgroup_mirror);
1071                         else
1072                                 uw= ED_vgroup_weight_verify(me->dvert+j, vgroup);
1073                                 
1074                         uw->weight= dw->weight;
1075
1076                         do_weight_paint_auto_normalize(me->dvert+j, vgroup, validmap);
1077                 }
1078         }
1079 }
1080
1081
1082 /* *************** set wpaint operator ****************** */
1083
1084 static int set_wpaint(bContext *C, wmOperator *op)              /* toggle */
1085 {               
1086         Object *ob= CTX_data_active_object(C);
1087         Scene *scene= CTX_data_scene(C);
1088         VPaint *wp= scene->toolsettings->wpaint;
1089         Mesh *me;
1090         
1091         me= get_mesh(ob);
1092         if(ob->id.lib || me==NULL) return OPERATOR_PASS_THROUGH;
1093         
1094         if(me && me->totface>=MAXINDEX) {
1095                 error("Maximum number of faces: %d", MAXINDEX-1);
1096                 ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1097                 return OPERATOR_CANCELLED;
1098         }
1099         
1100         if(ob->mode & OB_MODE_WEIGHT_PAINT) ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1101         else ob->mode |= OB_MODE_WEIGHT_PAINT;
1102         
1103         
1104         /* Weightpaint works by overriding colors in mesh,
1105                 * so need to make sure we recalc on enter and
1106                 * exit (exit needs doing regardless because we
1107                                 * should redeform).
1108                 */
1109         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
1110         
1111         if(ob->mode & OB_MODE_WEIGHT_PAINT) {
1112                 Object *par;
1113                 
1114                 if(wp==NULL)
1115                         wp= scene->toolsettings->wpaint= new_vpaint(1);
1116
1117                 paint_init(&wp->paint, PAINT_CURSOR_WEIGHT_PAINT);
1118                 paint_cursor_start(C, weight_paint_poll);
1119                 
1120                 mesh_octree_table(ob, NULL, NULL, 's');
1121                 
1122                 /* verify if active weight group is also active bone */
1123                 par= modifiers_isDeformedByArmature(ob);
1124                 if(par && (par->mode & OB_MODE_POSE)) {
1125                         bArmature *arm= par->data;
1126
1127                         if(arm->act_bone)
1128                                 ED_vgroup_select_by_name(ob, arm->act_bone->name);
1129                 }
1130         }
1131         else {
1132                 mesh_octree_table(ob, NULL, NULL, 'e');
1133         }
1134         
1135         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1136         
1137         return OPERATOR_FINISHED;
1138 }
1139
1140 /* for switching to/from mode */
1141 static int paint_poll_test(bContext *C)
1142 {
1143         if(ED_operator_view3d_active(C)==0)
1144                 return 0;
1145         if(CTX_data_edit_object(C))
1146                 return 0;
1147         if(CTX_data_active_object(C)==NULL)
1148                 return 0;
1149         return 1;
1150 }
1151
1152 void PAINT_OT_weight_paint_toggle(wmOperatorType *ot)
1153 {
1154         
1155         /* identifiers */
1156         ot->name= "Weight Paint Mode";
1157         ot->idname= "PAINT_OT_weight_paint_toggle";
1158         
1159         /* api callbacks */
1160         ot->exec= set_wpaint;
1161         ot->poll= paint_poll_test;
1162         
1163         /* flags */
1164         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1165         
1166 }
1167
1168 /* ************ paint radial controls *************/
1169
1170 static int vpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1171 {
1172         Paint *p = paint_get_active(CTX_data_scene(C));
1173         Brush *brush = paint_brush(p);
1174         
1175         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1176         p->paint_cursor = NULL;
1177         brush_radial_control_invoke(op, brush, 1);
1178         return WM_radial_control_invoke(C, op, event);
1179 }
1180
1181 static int vpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1182 {
1183         int ret = WM_radial_control_modal(C, op, event);
1184         if(ret != OPERATOR_RUNNING_MODAL)
1185                 paint_cursor_start(C, vertex_paint_poll);
1186         return ret;
1187 }
1188
1189 static int vpaint_radial_control_exec(bContext *C, wmOperator *op)
1190 {
1191         Brush *brush = paint_brush(&CTX_data_scene(C)->toolsettings->vpaint->paint);
1192         int ret = brush_radial_control_exec(op, brush, 1);
1193         
1194         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1195         
1196         return ret;
1197 }
1198
1199 static int wpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1200 {
1201         Paint *p = paint_get_active(CTX_data_scene(C));
1202         Brush *brush = paint_brush(p);
1203         
1204         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1205         p->paint_cursor = NULL;
1206         brush_radial_control_invoke(op, brush, 1);
1207         return WM_radial_control_invoke(C, op, event);
1208 }
1209
1210 static int wpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1211 {
1212         int ret = WM_radial_control_modal(C, op, event);
1213         if(ret != OPERATOR_RUNNING_MODAL)
1214                 paint_cursor_start(C, weight_paint_poll);
1215         return ret;
1216 }
1217
1218 static int wpaint_radial_control_exec(bContext *C, wmOperator *op)
1219 {
1220         Brush *brush = paint_brush(&CTX_data_scene(C)->toolsettings->wpaint->paint);
1221         int ret = brush_radial_control_exec(op, brush, 1);
1222         
1223         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1224         
1225         return ret;
1226 }
1227
1228 void PAINT_OT_weight_paint_radial_control(wmOperatorType *ot)
1229 {
1230         WM_OT_radial_control_partial(ot);
1231
1232         ot->name= "Weight Paint Radial Control";
1233         ot->idname= "PAINT_OT_weight_paint_radial_control";
1234
1235         ot->invoke= wpaint_radial_control_invoke;
1236         ot->modal= wpaint_radial_control_modal;
1237         ot->exec= wpaint_radial_control_exec;
1238         ot->poll= weight_paint_poll;
1239         
1240         /* flags */
1241         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1242 }
1243
1244 void PAINT_OT_vertex_paint_radial_control(wmOperatorType *ot)
1245 {
1246         WM_OT_radial_control_partial(ot);
1247
1248         ot->name= "Vertex Paint Radial Control";
1249         ot->idname= "PAINT_OT_vertex_paint_radial_control";
1250
1251         ot->invoke= vpaint_radial_control_invoke;
1252         ot->modal= vpaint_radial_control_modal;
1253         ot->exec= vpaint_radial_control_exec;
1254         ot->poll= vertex_paint_poll;
1255         
1256         /* flags */
1257         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1258 }
1259
1260 /* ************ weight paint operator ********** */
1261
1262 struct WPaintData {
1263         ViewContext vc;
1264         int *indexar;
1265         int vgroup_mirror;
1266         float *vertexcosnos;
1267         float wpimat[3][3];
1268         
1269         /*variables for auto normalize*/
1270         int auto_normalize;
1271         char *vgroup_validmap; /*stores if vgroups tie to deforming bones or not*/
1272 };
1273
1274 static char *wpaint_make_validmap(Mesh *me, Object *ob)
1275 {
1276         bDeformGroup *dg;
1277         ModifierData *md;
1278         char *validmap;
1279         bPose *pose;
1280         bPoseChannel *chan;
1281         ArmatureModifierData *amd;
1282         GHash *gh = BLI_ghash_new(BLI_ghashutil_strhash, BLI_ghashutil_strcmp);
1283         int i = 0, step1=1;
1284
1285         /*add all names to a hash table*/
1286         for (dg=ob->defbase.first, i=0; dg; dg=dg->next, i++) {
1287                 BLI_ghash_insert(gh, dg->name, NULL);
1288         }
1289
1290         if (!i)
1291                 return NULL;
1292
1293         validmap = MEM_callocN(i, "wpaint valid map");
1294
1295         /*now loop through the armature modifiers and identify deform bones*/
1296         for (md = ob->modifiers.first; md; md= !md->next && step1 ? (step1=0), modifiers_getVirtualModifierList(ob) : md->next) {
1297                 if (!(md->mode & (eModifierMode_Realtime|eModifierMode_Virtual)))
1298                         continue;
1299
1300                 if (md->type == eModifierType_Armature) 
1301                 {
1302                         amd = (ArmatureModifierData*) md;
1303
1304                         if(amd->object && amd->object->pose) {
1305                                 pose = amd->object->pose;
1306                                 
1307                                 for (chan=pose->chanbase.first; chan; chan=chan->next) {
1308                                         if (chan->bone->flag & BONE_NO_DEFORM)
1309                                                 continue;
1310
1311                                         if (BLI_ghash_haskey(gh, chan->name)) {
1312                                                 BLI_ghash_remove(gh, chan->name, NULL, NULL);
1313                                                 BLI_ghash_insert(gh, chan->name, SET_INT_IN_POINTER(1));
1314                                         }
1315                                 }
1316                         }
1317                 }
1318         }
1319         
1320         /*add all names to a hash table*/
1321         for (dg=ob->defbase.first, i=0; dg; dg=dg->next, i++) {
1322                 if (BLI_ghash_lookup(gh, dg->name) != NULL) {
1323                         validmap[i] = 1;
1324                 }
1325         }
1326
1327         BLI_ghash_free(gh, NULL, NULL);
1328
1329         return validmap;
1330 }
1331
1332 static int wpaint_stroke_test_start(bContext *C, wmOperator *op, wmEvent *event)
1333 {
1334         Scene *scene= CTX_data_scene(C);
1335         struct PaintStroke *stroke = op->customdata;
1336         ToolSettings *ts= CTX_data_tool_settings(C);
1337         VPaint *wp= ts->wpaint;
1338         Object *ob= CTX_data_active_object(C);
1339         struct WPaintData *wpd;
1340         Mesh *me;
1341         float mat[4][4], imat[4][4];
1342         
1343         if(scene->obedit) return OPERATOR_CANCELLED;
1344         
1345         me= get_mesh(ob);
1346         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1347         
1348         /* if nothing was added yet, we make dverts and a vertex deform group */
1349         if (!me->dvert)
1350                 ED_vgroup_data_create(&me->id);
1351         
1352         /* make mode data storage */
1353         wpd= MEM_callocN(sizeof(struct WPaintData), "WPaintData");
1354         paint_stroke_set_mode_data(stroke, wpd);
1355         view3d_set_viewcontext(C, &wpd->vc);
1356         wpd->vgroup_mirror= -1;
1357         
1358         /*set up auto-normalize, and generate map for detecting which
1359           vgroups affect deform bones*/
1360         wpd->auto_normalize = ts->auto_normalize;
1361         if (wpd->auto_normalize)
1362                 wpd->vgroup_validmap = wpaint_make_validmap(me, ob);
1363         
1364         //      if(qual & LR_CTRLKEY) {
1365         //              sample_wpaint(scene, ar, v3d, 0);
1366         //              return;
1367         //      }
1368         //      if(qual & LR_SHIFTKEY) {
1369         //              sample_wpaint(scene, ar, v3d, 1);
1370         //              return;
1371         //      }
1372         
1373         /* ALLOCATIONS! no return after this line */
1374         /* painting on subsurfs should give correct points too, this returns me->totvert amount */
1375         wpd->vertexcosnos= mesh_get_mapped_verts_nors(scene, ob);
1376         wpd->indexar= get_indexarray();
1377         copy_wpaint_prev(wp, me->dvert, me->totvert);
1378         
1379         /* this happens on a Bone select, when no vgroup existed yet */
1380         if(ob->actdef<=0) {
1381                 Object *modob;
1382                 if((modob = modifiers_isDeformedByArmature(ob))) {
1383                         bPoseChannel *pchan;
1384                         for(pchan= modob->pose->chanbase.first; pchan; pchan= pchan->next)
1385                                 if(pchan->bone->flag & SELECT)
1386                                         break;
1387                         if(pchan) {
1388                                 bDeformGroup *dg= get_named_vertexgroup(ob, pchan->name);
1389                                 if(dg==NULL)
1390                                         dg= ED_vgroup_add_name(ob, pchan->name);        /* sets actdef */
1391                                 else
1392                                         ob->actdef= get_defgroup_num(ob, dg);
1393                         }
1394                 }
1395         }
1396         if(ob->defbase.first==NULL) {
1397                 ED_vgroup_add(ob);
1398         }       
1399         
1400         //      if(ob->lay & v3d->lay); else error("Active object is not in this layer");
1401         
1402         /* imat for normals */
1403         mul_m4_m4m4(mat, ob->obmat, wpd->vc.rv3d->viewmat);
1404         invert_m4_m4(imat, mat);
1405         copy_m3_m4(wpd->wpimat, imat);
1406         
1407         /* if mirror painting, find the other group */
1408         if(me->editflag & ME_EDIT_MIRROR_X) {
1409                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
1410                 if(defgroup) {
1411                         bDeformGroup *curdef;
1412                         int actdef= 0;
1413                         char name[32];
1414                         
1415                         BLI_strncpy(name, defgroup->name, 32);
1416                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
1417                         
1418                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
1419                                 if (!strcmp(curdef->name, name))
1420                                         break;
1421                         if(curdef==NULL) {
1422                                 int olddef= ob->actdef; /* tsk, ED_vgroup_add sets the active defgroup */
1423                                 curdef= ED_vgroup_add_name (ob, name);
1424                                 ob->actdef= olddef;
1425                         }
1426                         
1427                         if(curdef && curdef!=defgroup)
1428                                 wpd->vgroup_mirror= actdef;
1429                 }
1430         }
1431         
1432         return 1;
1433 }
1434
1435 static void wpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1436 {
1437         ToolSettings *ts= CTX_data_tool_settings(C);
1438         VPaint *wp= ts->wpaint;
1439         Brush *brush = paint_brush(&wp->paint);
1440         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1441         ViewContext *vc= &wpd->vc;
1442         Object *ob= vc->obact;
1443         Mesh *me= ob->data;
1444         float mat[4][4];
1445         float paintweight= ts->vgroup_weight;
1446         int *indexar= wpd->indexar;
1447         int totindex, index, totw, flip;
1448         float alpha;
1449         float mval[2], pressure;
1450
1451         view3d_operator_needs_opengl(C);
1452                         
1453         /* load projection matrix */
1454         wmMultMatrix(ob->obmat);
1455         wmGetSingleMatrix(mat);
1456         wmLoadMatrix(wpd->vc.rv3d->viewmat);
1457
1458         flip = RNA_boolean_get(itemptr, "flip");
1459         pressure = RNA_float_get(itemptr, "pressure");
1460         RNA_float_get_array(itemptr, "mouse", mval);
1461         mval[0]-= vc->ar->winrct.xmin;
1462         mval[1]-= vc->ar->winrct.ymin;
1463                         
1464         swap_m4m4(wpd->vc.rv3d->persmat, mat);
1465                         
1466         /* which faces are involved */
1467         if(wp->flag & VP_AREA) {
1468                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush->size);
1469         }
1470         else {
1471                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1472                 if(indexar[0]) totindex= 1;
1473                 else totindex= 0;
1474         }
1475                         
1476         if(wp->flag & VP_COLINDEX) {
1477                 for(index=0; index<totindex; index++) {
1478                         if(indexar[index] && indexar[index]<=me->totface) {
1479                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1480                                                 
1481                                 if(mface->mat_nr!=ob->actcol-1) {
1482                                         indexar[index]= 0;
1483                                 }
1484                         }
1485                 }
1486         }
1487                         
1488         if((me->editflag & ME_EDIT_PAINT_MASK) && me->mface) {
1489                 for(index=0; index<totindex; index++) {
1490                         if(indexar[index] && indexar[index]<=me->totface) {
1491                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1492                                                 
1493                                 if((mface->flag & ME_FACE_SEL)==0) {
1494                                         indexar[index]= 0;
1495                                 }
1496                         }                                       
1497                 }
1498         }
1499                         
1500         /* make sure each vertex gets treated only once */
1501         /* and calculate filter weight */
1502         totw= 0;
1503         if(wp->mode==VP_BLUR) 
1504                 paintweight= 0.0f;
1505         else
1506                 paintweight= ts->vgroup_weight;
1507                         
1508         for(index=0; index<totindex; index++) {
1509                 if(indexar[index] && indexar[index]<=me->totface) {
1510                         MFace *mface= me->mface + (indexar[index]-1);
1511                                         
1512                         (me->dvert+mface->v1)->flag= 1;
1513                         (me->dvert+mface->v2)->flag= 1;
1514                         (me->dvert+mface->v3)->flag= 1;
1515                         if(mface->v4) (me->dvert+mface->v4)->flag= 1;
1516                                         
1517                         if(wp->mode==VP_BLUR) {
1518                                 MDeformWeight *dw, *(*dw_func)(MDeformVert *, int) = ED_vgroup_weight_verify;
1519                                                 
1520                                 if(wp->flag & VP_ONLYVGROUP)
1521                                         dw_func= ED_vgroup_weight_get;
1522                                                 
1523                                 dw= dw_func(me->dvert+mface->v1, ob->actdef-1);
1524                                 if(dw) {paintweight+= dw->weight; totw++;}
1525                                 dw= dw_func(me->dvert+mface->v2, ob->actdef-1);
1526                                 if(dw) {paintweight+= dw->weight; totw++;}
1527                                 dw= dw_func(me->dvert+mface->v3, ob->actdef-1);
1528                                 if(dw) {paintweight+= dw->weight; totw++;}
1529                                 if(mface->v4) {
1530                                         dw= dw_func(me->dvert+mface->v4, ob->actdef-1);
1531                                         if(dw) {paintweight+= dw->weight; totw++;}
1532                                 }
1533                         }
1534                 }
1535         }
1536                         
1537         if(wp->mode==VP_BLUR) 
1538                 paintweight/= (float)totw;
1539                         
1540         for(index=0; index<totindex; index++) {
1541                                 
1542                 if(indexar[index] && indexar[index]<=me->totface) {
1543                         MFace *mface= me->mface + (indexar[index]-1);
1544                                         
1545                         if((me->dvert+mface->v1)->flag) {
1546                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v1, mval, pressure);
1547                                 if(alpha) {
1548                                         do_weight_paint_vertex(wp, ob, mface->v1, 
1549                                                 alpha, paintweight, flip, wpd->vgroup_mirror, 
1550                                                 wpd->vgroup_validmap);
1551                                 }
1552                                 (me->dvert+mface->v1)->flag= 0;
1553                         }
1554                                         
1555                         if((me->dvert+mface->v2)->flag) {
1556                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v2, mval, pressure);
1557                                 if(alpha) {
1558                                         do_weight_paint_vertex(wp, ob, mface->v2, 
1559                                                 alpha, paintweight, flip, wpd->vgroup_mirror, 
1560                                                 wpd->vgroup_validmap);
1561                                 }
1562                                 (me->dvert+mface->v2)->flag= 0;
1563                         }
1564                                         
1565                         if((me->dvert+mface->v3)->flag) {
1566                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v3, mval, pressure);
1567                                 if(alpha) {
1568                                         do_weight_paint_vertex(wp, ob, mface->v3, 
1569                                                 alpha, paintweight, flip, wpd->vgroup_mirror, 
1570                                                 wpd->vgroup_validmap);
1571                                 }
1572                                 (me->dvert+mface->v3)->flag= 0;
1573                         }
1574                                         
1575                         if((me->dvert+mface->v4)->flag) {
1576                                 if(mface->v4) {
1577                                         alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v4, mval, pressure);
1578                                         if(alpha) {
1579                                                 do_weight_paint_vertex(wp, ob, mface->v4, 
1580                                                         alpha, paintweight, flip, wpd->vgroup_mirror,
1581                                                         wpd->vgroup_validmap);
1582                                         }
1583                                         (me->dvert+mface->v4)->flag= 0;
1584                                 }
1585                         }
1586                 }
1587         }
1588                         
1589         swap_m4m4(vc->rv3d->persmat, mat);
1590                         
1591         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1592         ED_region_tag_redraw(vc->ar);
1593 }
1594
1595 static void wpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
1596 {
1597         ToolSettings *ts= CTX_data_tool_settings(C);
1598         Object *ob= CTX_data_active_object(C);
1599         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1600         
1601         if(wpd->vertexcosnos)
1602                 MEM_freeN(wpd->vertexcosnos);
1603         MEM_freeN(wpd->indexar);
1604         
1605         if (wpd->vgroup_validmap)
1606                 MEM_freeN(wpd->vgroup_validmap);
1607
1608         /* frees prev buffer */
1609         copy_wpaint_prev(ts->wpaint, NULL, 0);
1610         
1611         /* and particles too */
1612         if(ob->particlesystem.first) {
1613                 ParticleSystem *psys;
1614                 int i;
1615                 
1616                 for(psys= ob->particlesystem.first; psys; psys= psys->next) {
1617                         for(i=0; i<PSYS_TOT_VG; i++) {
1618                                 if(psys->vgroup[i]==ob->actdef) {
1619                                         psys->recalc |= PSYS_RECALC_RESET;
1620                                         break;
1621                                 }
1622                         }
1623                 }
1624         }
1625         
1626         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1627         
1628         MEM_freeN(wpd);
1629 }
1630
1631
1632 static int wpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1633 {
1634         
1635         op->customdata = paint_stroke_new(C, NULL, wpaint_stroke_test_start,
1636                                           wpaint_stroke_update_step,
1637                                           wpaint_stroke_done);
1638         
1639         /* add modal handler */
1640         WM_event_add_modal_handler(C, op);
1641
1642         op->type->modal(C, op, event);
1643         
1644         return OPERATOR_RUNNING_MODAL;
1645 }
1646
1647 void PAINT_OT_weight_paint(wmOperatorType *ot)
1648 {
1649         
1650         /* identifiers */
1651         ot->name= "Weight Paint";
1652         ot->idname= "PAINT_OT_weight_paint";
1653         
1654         /* api callbacks */
1655         ot->invoke= wpaint_invoke;
1656         ot->modal= paint_stroke_modal;
1657         /* ot->exec= vpaint_exec; <-- needs stroke property */
1658         ot->poll= weight_paint_poll;
1659         
1660         /* flags */
1661         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1662
1663         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1664 }
1665
1666 static int weight_paint_set_exec(bContext *C, wmOperator *op)
1667 {
1668         struct Scene *scene= CTX_data_scene(C);
1669         Object *obact = CTX_data_active_object(C);
1670
1671         wpaint_fill(scene->toolsettings->wpaint, obact, scene->toolsettings->vgroup_weight);
1672         ED_region_tag_redraw(CTX_wm_region(C)); // XXX - should redraw all 3D views
1673         return OPERATOR_FINISHED;
1674 }
1675
1676 void PAINT_OT_weight_set(wmOperatorType *ot)
1677 {
1678         /* identifiers */
1679         ot->name= "Weight Set";
1680         ot->idname= "PAINT_OT_weight_set";
1681
1682         /* api callbacks */
1683         ot->exec= weight_paint_set_exec;
1684         ot->poll= facemask_paint_poll;
1685
1686         /* flags */
1687         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1688
1689         RNA_def_boolean(ot->srna, "extend", 0, "Extend", "Extend selection instead of deselecting everything first.");
1690 }
1691
1692 /* ************ set / clear vertex paint mode ********** */
1693
1694
1695 static int set_vpaint(bContext *C, wmOperator *op)              /* toggle */
1696 {       
1697         Object *ob= CTX_data_active_object(C);
1698         Scene *scene= CTX_data_scene(C);
1699         VPaint *vp= scene->toolsettings->vpaint;
1700         Mesh *me;
1701         
1702         me= get_mesh(ob);
1703         
1704         if(me==NULL || object_data_is_libdata(ob)) {
1705                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1706                 return OPERATOR_PASS_THROUGH;
1707         }
1708         
1709         if(me && me->totface>=MAXINDEX) {
1710                 error("Maximum number of faces: %d", MAXINDEX-1);
1711                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1712                 return OPERATOR_FINISHED;
1713         }
1714         
1715         if(me && me->mcol==NULL) make_vertexcol(ob);
1716         
1717         /* toggle: end vpaint */
1718         if(ob->mode & OB_MODE_VERTEX_PAINT) {
1719                 
1720                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1721         }
1722         else {
1723                 ob->mode |= OB_MODE_VERTEX_PAINT;
1724                 /* Turn off weight painting */
1725                 if (ob->mode & OB_MODE_WEIGHT_PAINT)
1726                         set_wpaint(C, op);
1727                 
1728                 if(vp==NULL)
1729                         vp= scene->toolsettings->vpaint= new_vpaint(0);
1730                 
1731                 paint_cursor_start(C, vertex_paint_poll);
1732
1733                 paint_init(&vp->paint, PAINT_CURSOR_VERTEX_PAINT);
1734         }
1735         
1736         if (me)
1737                 /* update modifier stack for mapping requirements */
1738                 DAG_id_flush_update(&me->id, OB_RECALC_DATA);
1739         
1740         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1741         
1742         return OPERATOR_FINISHED;
1743 }
1744
1745 void PAINT_OT_vertex_paint_toggle(wmOperatorType *ot)
1746 {
1747         
1748         /* identifiers */
1749         ot->name= "Vertex Paint Mode";
1750         ot->idname= "PAINT_OT_vertex_paint_toggle";
1751         
1752         /* api callbacks */
1753         ot->exec= set_vpaint;
1754         ot->poll= paint_poll_test;
1755         
1756         /* flags */
1757         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1758         
1759 }
1760
1761
1762
1763 /* ********************** vertex paint operator ******************* */
1764
1765 /* Implementation notes:
1766
1767 Operator->invoke()
1768   - validate context (add mcol)
1769   - create customdata storage
1770   - call paint once (mouse click)
1771   - add modal handler 
1772
1773 Operator->modal()
1774   - for every mousemove, apply vertex paint
1775   - exit on mouse release, free customdata
1776     (return OPERATOR_FINISHED also removes handler and operator)
1777
1778 For future:
1779   - implement a stroke event (or mousemove with past positons)
1780   - revise whether op->customdata should be added in object, in set_vpaint
1781
1782 */
1783
1784 typedef struct VPaintData {
1785         ViewContext vc;
1786         unsigned int paintcol;
1787         int *indexar;
1788         float *vertexcosnos;
1789         float vpimat[3][3];
1790 } VPaintData;
1791
1792 static int vpaint_stroke_test_start(bContext *C, struct wmOperator *op, wmEvent *event)
1793 {
1794         ToolSettings *ts= CTX_data_tool_settings(C);
1795         struct PaintStroke *stroke = op->customdata;
1796         VPaint *vp= ts->vpaint;
1797         struct VPaintData *vpd;
1798         Object *ob= CTX_data_active_object(C);
1799         Mesh *me;
1800         float mat[4][4], imat[4][4];
1801
1802         /* context checks could be a poll() */
1803         me= get_mesh(ob);
1804         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1805         
1806         if(me->mcol==NULL) make_vertexcol(ob);
1807         if(me->mcol==NULL) return OPERATOR_CANCELLED;
1808         
1809         /* make mode data storage */
1810         vpd= MEM_callocN(sizeof(struct VPaintData), "VPaintData");
1811         paint_stroke_set_mode_data(stroke, vpd);
1812         view3d_set_viewcontext(C, &vpd->vc);
1813         
1814         vpd->vertexcosnos= mesh_get_mapped_verts_nors(vpd->vc.scene, ob);
1815         vpd->indexar= get_indexarray();
1816         vpd->paintcol= vpaint_get_current_col(vp);
1817         
1818         /* for filtering */
1819         copy_vpaint_prev(vp, (unsigned int *)me->mcol, me->totface);
1820         
1821         /* some old cruft to sort out later */
1822         mul_m4_m4m4(mat, ob->obmat, vpd->vc.rv3d->viewmat);
1823         invert_m4_m4(imat, mat);
1824         copy_m3_m4(vpd->vpimat, imat);
1825
1826         return 1;
1827 }
1828
1829 static void vpaint_paint_face(VPaint *vp, VPaintData *vpd, Object *ob, int index, float mval[2], float pressure, int flip)
1830 {
1831         ViewContext *vc = &vpd->vc;
1832         Mesh *me = get_mesh(ob);
1833         MFace *mface= ((MFace*)me->mface) + index;
1834         unsigned int *mcol= ((unsigned int*)me->mcol) + 4*index;
1835         unsigned int *mcolorig= ((unsigned int*)vp->vpaint_prev) + 4*index;
1836         float alpha;
1837         int i;
1838         
1839         if((vp->flag & VP_COLINDEX && mface->mat_nr!=ob->actcol-1) ||
1840            ((me->editflag & ME_EDIT_PAINT_MASK) && !(mface->flag & ME_FACE_SEL)))
1841                 return;
1842
1843         if(vp->mode==VP_BLUR) {
1844                 unsigned int fcol1= mcol_blend( mcol[0], mcol[1], 128);
1845                 if(mface->v4) {
1846                         unsigned int fcol2= mcol_blend( mcol[2], mcol[3], 128);
1847                         vpd->paintcol= mcol_blend( fcol1, fcol2, 128);
1848                 }
1849                 else {
1850                         vpd->paintcol= mcol_blend( mcol[2], fcol1, 170);
1851                 }
1852                 
1853         }
1854
1855         for(i = 0; i < (mface->v4 ? 4 : 3); ++i) {
1856                 alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*(&mface->v1)[i], mval, pressure);
1857                 if(alpha)
1858                         vpaint_blend(vp, mcol+i, mcolorig+i, vpd->paintcol, (int)(alpha*255.0));
1859         }
1860 }
1861
1862 static void vpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1863 {
1864         ToolSettings *ts= CTX_data_tool_settings(C);
1865         struct VPaintData *vpd = paint_stroke_mode_data(stroke);
1866         VPaint *vp= ts->vpaint;
1867         Brush *brush = paint_brush(&vp->paint);
1868         ViewContext *vc= &vpd->vc;
1869         Object *ob= vc->obact;
1870         Mesh *me= ob->data;
1871         float mat[4][4];
1872         int *indexar= vpd->indexar;
1873         int totindex, index, flip;
1874         float pressure, mval[2];
1875
1876         RNA_float_get_array(itemptr, "mouse", mval);
1877         flip = RNA_boolean_get(itemptr, "flip");
1878         pressure = RNA_float_get(itemptr, "pressure");
1879                         
1880         view3d_operator_needs_opengl(C);
1881                         
1882         /* load projection matrix */
1883         wmMultMatrix(ob->obmat);
1884         wmGetSingleMatrix(mat);
1885         wmLoadMatrix(vc->rv3d->viewmat);
1886
1887         mval[0]-= vc->ar->winrct.xmin;
1888         mval[1]-= vc->ar->winrct.ymin;
1889
1890                         
1891         /* which faces are involved */
1892         if(vp->flag & VP_AREA) {
1893                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush->size);
1894         }
1895         else {
1896                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1897                 if(indexar[0]) totindex= 1;
1898                 else totindex= 0;
1899         }
1900                         
1901         swap_m4m4(vc->rv3d->persmat, mat);
1902                         
1903         for(index=0; index<totindex; index++) {                         
1904                 if(indexar[index] && indexar[index]<=me->totface)
1905                         vpaint_paint_face(vp, vpd, ob, indexar[index]-1, mval, pressure, flip);
1906         }
1907                                                 
1908         swap_m4m4(vc->rv3d->persmat, mat);
1909
1910         /* was disabled because it is slow, but necessary for blur */
1911         if(vp->mode == VP_BLUR)
1912                 do_shared_vertexcol(me);
1913                         
1914         ED_region_tag_redraw(vc->ar);
1915                         
1916         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1917 }
1918
1919 static void vpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
1920 {
1921         ToolSettings *ts= CTX_data_tool_settings(C);
1922         struct VPaintData *vpd= paint_stroke_mode_data(stroke);
1923         
1924         if(vpd->vertexcosnos)
1925                 MEM_freeN(vpd->vertexcosnos);
1926         MEM_freeN(vpd->indexar);
1927         
1928         /* frees prev buffer */
1929         copy_vpaint_prev(ts->vpaint, NULL, 0);
1930         
1931         MEM_freeN(vpd);
1932 }
1933
1934 static int vpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1935 {
1936         
1937         op->customdata = paint_stroke_new(C, NULL, vpaint_stroke_test_start,
1938                                           vpaint_stroke_update_step,
1939                                           vpaint_stroke_done);
1940         
1941         /* add modal handler */
1942         WM_event_add_modal_handler(C, op);
1943
1944         op->type->modal(C, op, event);
1945         
1946         return OPERATOR_RUNNING_MODAL;
1947 }
1948
1949 void PAINT_OT_vertex_paint(wmOperatorType *ot)
1950 {
1951         /* identifiers */
1952         ot->name= "Vertex Paint";
1953         ot->idname= "PAINT_OT_vertex_paint";
1954         
1955         /* api callbacks */
1956         ot->invoke= vpaint_invoke;
1957         ot->modal= paint_stroke_modal;
1958         /* ot->exec= vpaint_exec; <-- needs stroke property */
1959         ot->poll= vertex_paint_poll;
1960         
1961         /* flags */
1962         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1963
1964         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1965 }
1966