5dd69cdf03055993b6371b447868b802efd753a9
[blender-staging.git] / source / blender / editors / sculpt_paint / paint_vertex.c
1 /**
2  * $Id$
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  */
29
30 #include <math.h>
31 #include <string.h>
32
33 #ifdef WIN32
34 #include <io.h>
35 #else
36 #include <unistd.h>
37 #endif   
38
39 #include "MEM_guardedalloc.h"
40
41 #include "IMB_imbuf.h"
42 #include "IMB_imbuf_types.h"
43
44 #include "BLI_blenlib.h"
45 #include "BLI_arithb.h"
46
47
48 #include "DNA_action_types.h"
49 #include "DNA_armature_types.h"
50 #include "DNA_brush_types.h"
51 #include "DNA_cloth_types.h"
52 #include "DNA_mesh_types.h"
53 #include "DNA_meshdata_types.h"
54 #include "DNA_modifier_types.h"
55 #include "DNA_object_types.h"
56 #include "DNA_object_force.h"
57 #include "DNA_particle_types.h"
58 #include "DNA_screen_types.h"
59 #include "DNA_space_types.h"
60 #include "DNA_scene_types.h"
61 #include "DNA_view3d_types.h"
62 #include "DNA_userdef_types.h"
63
64 #include "RNA_access.h"
65 #include "RNA_define.h"
66
67 #include "BKE_armature.h"
68 #include "BKE_brush.h"
69 #include "BKE_DerivedMesh.h"
70 #include "BKE_cloth.h"
71 #include "BKE_context.h"
72 #include "BKE_customdata.h"
73 #include "BKE_depsgraph.h"
74 #include "BKE_deform.h"
75 #include "BKE_displist.h"
76 #include "BKE_global.h"
77 #include "BKE_mesh.h"
78 #include "BKE_modifier.h"
79 #include "BKE_object.h"
80 #include "BKE_paint.h"
81 #include "BKE_utildefines.h"
82
83 #include "WM_api.h"
84 #include "WM_types.h"
85
86 #include "BIF_gl.h"
87 #include "BIF_glutil.h"
88
89 #include "ED_mesh.h"
90 #include "ED_object.h"
91 #include "ED_screen.h"
92 #include "ED_util.h"
93 #include "ED_view3d.h"
94
95 #include "paint_intern.h"
96
97         /* vp->mode */
98 #define VP_MIX  0
99 #define VP_ADD  1
100 #define VP_SUB  2
101 #define VP_MUL  3
102 #define VP_BLUR 4
103 #define VP_LIGHTEN      5
104 #define VP_DARKEN       6
105
106 #define MAXINDEX        512000
107
108 /* XXX */
109 static void error() {}
110
111 /* polling - retrieve whether cursor should be set or operator should be done */
112
113
114 /* Returns true if vertex paint mode is active */
115 int vertex_paint_mode_poll(bContext *C)
116 {
117         Object *ob = CTX_data_active_object(C);
118
119         return ob && ob->mode == OB_MODE_VERTEX_PAINT;
120 }
121
122 int vertex_paint_poll(bContext *C)
123 {
124         if(vertex_paint_mode_poll(C) && 
125            paint_brush(&CTX_data_tool_settings(C)->vpaint->paint)) {
126                 ScrArea *sa= CTX_wm_area(C);
127                 if(sa->spacetype==SPACE_VIEW3D) {
128                         ARegion *ar= CTX_wm_region(C);
129                         if(ar->regiontype==RGN_TYPE_WINDOW)
130                                 return 1;
131                 }
132         }
133         return 0;
134 }
135
136 int weight_paint_poll(bContext *C)
137 {
138         Object *ob = CTX_data_active_object(C);
139
140         if(ob && ob->mode & OB_MODE_WEIGHT_PAINT &&
141            paint_brush(&CTX_data_tool_settings(C)->wpaint->paint)) {
142                 ScrArea *sa= CTX_wm_area(C);
143                 if(sa->spacetype==SPACE_VIEW3D) {
144                         ARegion *ar= CTX_wm_region(C);
145                         if(ar->regiontype==RGN_TYPE_WINDOW)
146                                 return 1;
147                 }
148         }
149         return 0;
150 }
151
152 static VPaint *new_vpaint(int wpaint)
153 {
154         VPaint *vp= MEM_callocN(sizeof(VPaint), "VPaint");
155         
156         vp->gamma= vp->mul= 1.0f;
157         
158         vp->flag= VP_AREA+VP_SOFT+VP_SPRAY;
159         
160         if(wpaint)
161                 vp->flag= VP_AREA+VP_SOFT;
162
163         return vp;
164 }
165
166 static int *get_indexarray(void)
167 {
168         return MEM_mallocN(sizeof(int)*MAXINDEX + 2, "vertexpaint");
169 }
170
171
172 /* in contradiction to cpack drawing colors, the MCOL colors (vpaint colors) are per byte! 
173    so not endian sensitive. Mcol = ABGR!!! so be cautious with cpack calls */
174
175 unsigned int rgba_to_mcol(float r, float g, float b, float a)
176 {
177         int ir, ig, ib, ia;
178         unsigned int col;
179         char *cp;
180         
181         ir= floor(255.0*r);
182         if(ir<0) ir= 0; else if(ir>255) ir= 255;
183         ig= floor(255.0*g);
184         if(ig<0) ig= 0; else if(ig>255) ig= 255;
185         ib= floor(255.0*b);
186         if(ib<0) ib= 0; else if(ib>255) ib= 255;
187         ia= floor(255.0*a);
188         if(ia<0) ia= 0; else if(ia>255) ia= 255;
189         
190         cp= (char *)&col;
191         cp[0]= ia;
192         cp[1]= ib;
193         cp[2]= ig;
194         cp[3]= ir;
195         
196         return col;
197         
198 }
199
200 static unsigned int vpaint_get_current_col(VPaint *vp)
201 {
202         Brush *brush = paint_brush(&vp->paint);
203         return rgba_to_mcol(brush->rgb[0], brush->rgb[1], brush->rgb[2], 1.0f);
204 }
205
206 static void do_shared_vertexcol(Mesh *me)
207 {
208         /* if no mcol: do not do */
209         /* if tface: only the involved faces, otherwise all */
210         MFace *mface;
211         MTFace *tface;
212         int a;
213         short *scolmain, *scol;
214         char *mcol;
215         
216         if(me->mcol==0 || me->totvert==0 || me->totface==0) return;
217         
218         scolmain= MEM_callocN(4*sizeof(short)*me->totvert, "colmain");
219         
220         tface= me->mtface;
221         mface= me->mface;
222         mcol= (char *)me->mcol;
223         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
224                 if((tface && tface->mode & TF_SHAREDCOL) || (me->editflag & ME_EDIT_PAINT_MASK)==0) {
225                         scol= scolmain+4*mface->v1;
226                         scol[0]++; scol[1]+= mcol[1]; scol[2]+= mcol[2]; scol[3]+= mcol[3];
227                         scol= scolmain+4*mface->v2;
228                         scol[0]++; scol[1]+= mcol[5]; scol[2]+= mcol[6]; scol[3]+= mcol[7];
229                         scol= scolmain+4*mface->v3;
230                         scol[0]++; scol[1]+= mcol[9]; scol[2]+= mcol[10]; scol[3]+= mcol[11];
231                         if(mface->v4) {
232                                 scol= scolmain+4*mface->v4;
233                                 scol[0]++; scol[1]+= mcol[13]; scol[2]+= mcol[14]; scol[3]+= mcol[15];
234                         }
235                 }
236                 if(tface) tface++;
237         }
238         
239         a= me->totvert;
240         scol= scolmain;
241         while(a--) {
242                 if(scol[0]>1) {
243                         scol[1]/= scol[0];
244                         scol[2]/= scol[0];
245                         scol[3]/= scol[0];
246                 }
247                 scol+= 4;
248         }
249         
250         tface= me->mtface;
251         mface= me->mface;
252         mcol= (char *)me->mcol;
253         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
254                 if((tface && tface->mode & TF_SHAREDCOL) || (me->editflag & ME_EDIT_PAINT_MASK)==0) {
255                         scol= scolmain+4*mface->v1;
256                         mcol[1]= scol[1]; mcol[2]= scol[2]; mcol[3]= scol[3];
257                         scol= scolmain+4*mface->v2;
258                         mcol[5]= scol[1]; mcol[6]= scol[2]; mcol[7]= scol[3];
259                         scol= scolmain+4*mface->v3;
260                         mcol[9]= scol[1]; mcol[10]= scol[2]; mcol[11]= scol[3];
261                         if(mface->v4) {
262                                 scol= scolmain+4*mface->v4;
263                                 mcol[13]= scol[1]; mcol[14]= scol[2]; mcol[15]= scol[3];
264                         }
265                 }
266                 if(tface) tface++;
267         }
268
269         MEM_freeN(scolmain);
270 }
271
272 void make_vertexcol(Scene *scene, int shade)    /* single ob */
273 {
274         Object *ob;
275         Mesh *me;
276
277         if(scene->obedit) {
278                 error("Unable to perform function in Edit Mode");
279                 return;
280         }
281         
282         ob= OBACT;
283         if(!ob || ob->id.lib) return;
284         me= get_mesh(ob);
285         if(me==0) return;
286
287         /* copies from shadedisplist to mcol */
288         if(!me->mcol) {
289                 CustomData_add_layer(&me->fdata, CD_MCOL, CD_CALLOC, NULL, me->totface);
290                 mesh_update_customdata_pointers(me);
291         }
292
293         if(shade)
294                 shadeMeshMCol(scene, ob, me);
295         else
296                 memset(me->mcol, 255, 4*sizeof(MCol)*me->totface);
297         
298         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
299         
300 }
301
302 static void copy_vpaint_prev(VPaint *vp, unsigned int *mcol, int tot)
303 {
304         if(vp->vpaint_prev) {
305                 MEM_freeN(vp->vpaint_prev);
306                 vp->vpaint_prev= NULL;
307         }
308         vp->tot= tot;   
309         
310         if(mcol==NULL || tot==0) return;
311         
312         vp->vpaint_prev= MEM_mallocN(4*sizeof(int)*tot, "vpaint_prev");
313         memcpy(vp->vpaint_prev, mcol, 4*sizeof(int)*tot);
314         
315 }
316
317 static void copy_wpaint_prev (VPaint *wp, MDeformVert *dverts, int dcount)
318 {
319         if (wp->wpaint_prev) {
320                 free_dverts(wp->wpaint_prev, wp->tot);
321                 wp->wpaint_prev= NULL;
322         }
323         
324         if(dverts && dcount) {
325                 
326                 wp->wpaint_prev = MEM_mallocN (sizeof(MDeformVert)*dcount, "wpaint prev");
327                 wp->tot = dcount;
328                 copy_dverts (wp->wpaint_prev, dverts, dcount);
329         }
330 }
331
332
333 void clear_vpaint(Scene *scene, int selected)
334 {
335         Mesh *me;
336         MFace *mf;
337         Object *ob;
338         unsigned int paintcol, *mcol;
339         int i;
340
341         ob= OBACT;
342         me= get_mesh(ob);
343         if(me==0 || me->totface==0) return;
344
345         if(!me->mcol)
346                 make_vertexcol(scene, 0);
347
348         paintcol= vpaint_get_current_col(scene->toolsettings->vpaint);
349
350         mf = me->mface;
351         mcol = (unsigned int*)me->mcol;
352         for (i = 0; i < me->totface; i++, mf++, mcol+=4) {
353                 if (!selected || mf->flag & ME_FACE_SEL) {
354                         mcol[0] = paintcol;
355                         mcol[1] = paintcol;
356                         mcol[2] = paintcol;
357                         mcol[3] = paintcol;
358                 }
359         }
360         
361         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
362 }
363
364
365 /* fills in the selected faces with the current weight and vertex group */
366 void clear_wpaint_selectedfaces(Scene *scene)
367 {
368         ToolSettings *ts= scene->toolsettings;
369         VPaint *wp= ts->wpaint;
370         float paintweight= ts->vgroup_weight;
371         Mesh *me;
372         MFace *mface;
373         Object *ob;
374         MDeformWeight *dw, *uw;
375         int *indexar;
376         int index, vgroup;
377         unsigned int faceverts[5]={0,0,0,0,0};
378         unsigned char i;
379         int vgroup_mirror= -1;
380         
381         ob= OBACT;
382         me= ob->data;
383         if(me==0 || me->totface==0 || me->dvert==0 || !me->mface) return;
384         
385         indexar= get_indexarray();
386         for(index=0, mface=me->mface; index<me->totface; index++, mface++) {
387                 if((mface->flag & ME_FACE_SEL)==0)
388                         indexar[index]= 0;
389                 else
390                         indexar[index]= index+1;
391         }
392         
393         vgroup= ob->actdef-1;
394         
395         /* directly copied from weight_paint, should probaby split into a seperate function */
396         /* if mirror painting, find the other group */          
397         if(me->editflag & ME_EDIT_MIRROR_X) {
398                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
399                 if(defgroup) {
400                         bDeformGroup *curdef;
401                         int actdef= 0;
402                         char name[32];
403
404                         BLI_strncpy(name, defgroup->name, 32);
405                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
406                         
407                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
408                                 if (!strcmp(curdef->name, name))
409                                         break;
410                         if(curdef==NULL) {
411                                 int olddef= ob->actdef; /* tsk, ED_vgroup_add sets the active defgroup */
412                                 curdef= ED_vgroup_add_name (ob, name);
413                                 ob->actdef= olddef;
414                         }
415                         
416                         if(curdef && curdef!=defgroup)
417                                 vgroup_mirror= actdef;
418                 }
419         }
420         /* end copy from weight_paint*/
421         
422         copy_wpaint_prev(wp, me->dvert, me->totvert);
423         
424         for(index=0; index<me->totface; index++) {
425                 if(indexar[index] && indexar[index]<=me->totface) {
426                         mface= me->mface + (indexar[index]-1);
427                         /* just so we can loop through the verts */
428                         faceverts[0]= mface->v1;
429                         faceverts[1]= mface->v2;
430                         faceverts[2]= mface->v3;
431                         faceverts[3]= mface->v4;
432                         for (i=0; i<3 || faceverts[i]; i++) {
433                                 if(!((me->dvert+faceverts[i])->flag)) {
434                                         dw= ED_vgroup_weight_verify(me->dvert+faceverts[i], vgroup);
435                                         if(dw) {
436                                                 uw= ED_vgroup_weight_verify(wp->wpaint_prev+faceverts[i], vgroup);
437                                                 uw->weight= dw->weight; /* set the undo weight */
438                                                 dw->weight= paintweight;
439                                                 
440                                                 if(me->editflag & ME_EDIT_MIRROR_X) {   /* x mirror painting */
441                                                         int j= mesh_get_x_mirror_vert(ob, faceverts[i]);
442                                                         if(j>=0) {
443                                                                 /* copy, not paint again */
444                                                                 if(vgroup_mirror != -1) {
445                                                                         dw= ED_vgroup_weight_verify(me->dvert+j, vgroup_mirror);
446                                                                         uw= ED_vgroup_weight_verify(wp->wpaint_prev+j, vgroup_mirror);
447                                                                 } else {
448                                                                         dw= ED_vgroup_weight_verify(me->dvert+j, vgroup);
449                                                                         uw= ED_vgroup_weight_verify(wp->wpaint_prev+j, vgroup);
450                                                                 }
451                                                                 uw->weight= dw->weight; /* set the undo weight */
452                                                                 dw->weight= paintweight;
453                                                         }
454                                                 }
455                                         }
456                                         (me->dvert+faceverts[i])->flag= 1;
457                                 }
458                         }
459                 }
460         }
461         
462         index=0;
463         while (index<me->totvert) {
464                 (me->dvert+index)->flag= 0;
465                 index++;
466         }
467         
468         MEM_freeN(indexar);
469         copy_wpaint_prev(wp, NULL, 0);
470
471         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
472 }
473
474
475 void vpaint_dogamma(Scene *scene)
476 {
477         VPaint *vp= scene->toolsettings->vpaint;
478         Mesh *me;
479         Object *ob;
480         float igam, fac;
481         int a, temp;
482         unsigned char *cp, gamtab[256];
483
484         ob= OBACT;
485         me= get_mesh(ob);
486
487         if(!(ob->mode & OB_MODE_VERTEX_PAINT)) return;
488         if(me==0 || me->mcol==0 || me->totface==0) return;
489
490         igam= 1.0/vp->gamma;
491         for(a=0; a<256; a++) {
492                 
493                 fac= ((float)a)/255.0;
494                 fac= vp->mul*pow( fac, igam);
495                 
496                 temp= 255.9*fac;
497                 
498                 if(temp<=0) gamtab[a]= 0;
499                 else if(temp>=255) gamtab[a]= 255;
500                 else gamtab[a]= temp;
501         }
502
503         a= 4*me->totface;
504         cp= (unsigned char *)me->mcol;
505         while(a--) {
506                 
507                 cp[1]= gamtab[ cp[1] ];
508                 cp[2]= gamtab[ cp[2] ];
509                 cp[3]= gamtab[ cp[3] ];
510                 
511                 cp+= 4;
512         }
513 }
514
515 static unsigned int mcol_blend(unsigned int col1, unsigned int col2, int fac)
516 {
517         char *cp1, *cp2, *cp;
518         int mfac;
519         unsigned int col=0;
520         
521         if(fac==0) return col1;
522         if(fac>=255) return col2;
523
524         mfac= 255-fac;
525         
526         cp1= (char *)&col1;
527         cp2= (char *)&col2;
528         cp=  (char *)&col;
529         
530         cp[0]= 255;
531         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
532         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
533         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
534         
535         return col;
536 }
537
538 static unsigned int mcol_add(unsigned int col1, unsigned int col2, int fac)
539 {
540         char *cp1, *cp2, *cp;
541         int temp;
542         unsigned int col=0;
543         
544         if(fac==0) return col1;
545         
546         cp1= (char *)&col1;
547         cp2= (char *)&col2;
548         cp=  (char *)&col;
549         
550         cp[0]= 255;
551         temp= cp1[1] + ((fac*cp2[1])/255);
552         if(temp>254) cp[1]= 255; else cp[1]= temp;
553         temp= cp1[2] + ((fac*cp2[2])/255);
554         if(temp>254) cp[2]= 255; else cp[2]= temp;
555         temp= cp1[3] + ((fac*cp2[3])/255);
556         if(temp>254) cp[3]= 255; else cp[3]= temp;
557         
558         return col;
559 }
560
561 static unsigned int mcol_sub(unsigned int col1, unsigned int col2, int fac)
562 {
563         char *cp1, *cp2, *cp;
564         int temp;
565         unsigned int col=0;
566         
567         if(fac==0) return col1;
568         
569         cp1= (char *)&col1;
570         cp2= (char *)&col2;
571         cp=  (char *)&col;
572         
573         cp[0]= 255;
574         temp= cp1[1] - ((fac*cp2[1])/255);
575         if(temp<0) cp[1]= 0; else cp[1]= temp;
576         temp= cp1[2] - ((fac*cp2[2])/255);
577         if(temp<0) cp[2]= 0; else cp[2]= temp;
578         temp= cp1[3] - ((fac*cp2[3])/255);
579         if(temp<0) cp[3]= 0; else cp[3]= temp;
580         
581         return col;
582 }
583
584 static unsigned int mcol_mul(unsigned int col1, unsigned int col2, int fac)
585 {
586         char *cp1, *cp2, *cp;
587         int mfac;
588         unsigned int col=0;
589         
590         if(fac==0) return col1;
591
592         mfac= 255-fac;
593         
594         cp1= (char *)&col1;
595         cp2= (char *)&col2;
596         cp=  (char *)&col;
597         
598         /* first mul, then blend the fac */
599         cp[0]= 255;
600         cp[1]= (mfac*cp1[1] + fac*((cp2[1]*cp1[1])/255)  )/255;
601         cp[2]= (mfac*cp1[2] + fac*((cp2[2]*cp1[2])/255)  )/255;
602         cp[3]= (mfac*cp1[3] + fac*((cp2[3]*cp1[3])/255)  )/255;
603
604         
605         return col;
606 }
607
608 static unsigned int mcol_lighten(unsigned int col1, unsigned int col2, int fac)
609 {
610         char *cp1, *cp2, *cp;
611         int mfac;
612         unsigned int col=0;
613         
614         if(fac==0) return col1;
615         if(fac>=255) return col2;
616
617         mfac= 255-fac;
618         
619         cp1= (char *)&col1;
620         cp2= (char *)&col2;
621         cp=  (char *)&col;
622         
623         /* See if are lighter, if so mix, else dont do anything.
624         if the paint col is darker then the original, then ignore */
625         if (cp1[1]+cp1[2]+cp1[3] > cp2[1]+cp2[2]+cp2[3])
626                 return col1;
627         
628         cp[0]= 255;
629         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
630         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
631         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
632         
633         return col;
634 }
635
636 static unsigned int mcol_darken(unsigned int col1, unsigned int col2, int fac)
637 {
638         char *cp1, *cp2, *cp;
639         int mfac;
640         unsigned int col=0;
641         
642         if(fac==0) return col1;
643         if(fac>=255) return col2;
644
645         mfac= 255-fac;
646         
647         cp1= (char *)&col1;
648         cp2= (char *)&col2;
649         cp=  (char *)&col;
650         
651         /* See if were darker, if so mix, else dont do anything.
652         if the paint col is brighter then the original, then ignore */
653         if (cp1[1]+cp1[2]+cp1[3] < cp2[1]+cp2[2]+cp2[3])
654                 return col1;
655         
656         cp[0]= 255;
657         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
658         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
659         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
660         return col;
661 }
662
663 static void vpaint_blend(VPaint *vp, unsigned int *col, unsigned int *colorig, unsigned int paintcol, int alpha)
664 {
665         Brush *brush = paint_brush(&vp->paint);
666
667         if(vp->mode==VP_MIX || vp->mode==VP_BLUR) *col= mcol_blend( *col, paintcol, alpha);
668         else if(vp->mode==VP_ADD) *col= mcol_add( *col, paintcol, alpha);
669         else if(vp->mode==VP_SUB) *col= mcol_sub( *col, paintcol, alpha);
670         else if(vp->mode==VP_MUL) *col= mcol_mul( *col, paintcol, alpha);
671         else if(vp->mode==VP_LIGHTEN) *col= mcol_lighten( *col, paintcol, alpha);
672         else if(vp->mode==VP_DARKEN) *col= mcol_darken( *col, paintcol, alpha);
673         
674         /* if no spray, clip color adding with colorig & orig alpha */
675         if((vp->flag & VP_SPRAY)==0) {
676                 unsigned int testcol=0, a;
677                 char *cp, *ct, *co;
678                 
679                 alpha= (int)(255.0*brush->alpha);
680                 
681                 if(vp->mode==VP_MIX || vp->mode==VP_BLUR) testcol= mcol_blend( *colorig, paintcol, alpha);
682                 else if(vp->mode==VP_ADD) testcol= mcol_add( *colorig, paintcol, alpha);
683                 else if(vp->mode==VP_SUB) testcol= mcol_sub( *colorig, paintcol, alpha);
684                 else if(vp->mode==VP_MUL) testcol= mcol_mul( *colorig, paintcol, alpha);
685                 else if(vp->mode==VP_LIGHTEN)  testcol= mcol_lighten( *colorig, paintcol, alpha);
686                 else if(vp->mode==VP_DARKEN)   testcol= mcol_darken( *colorig, paintcol, alpha);
687                 
688                 cp= (char *)col;
689                 ct= (char *)&testcol;
690                 co= (char *)colorig;
691                 
692                 for(a=0; a<4; a++) {
693                         if( ct[a]<co[a] ) {
694                                 if( cp[a]<ct[a] ) cp[a]= ct[a];
695                                 else if( cp[a]>co[a] ) cp[a]= co[a];
696                         }
697                         else {
698                                 if( cp[a]<co[a] ) cp[a]= co[a];
699                                 else if( cp[a]>ct[a] ) cp[a]= ct[a];
700                         }
701                 }
702         }
703 }
704
705
706 static int sample_backbuf_area(ViewContext *vc, int *indexar, int totface, int x, int y, float size)
707 {
708         struct ImBuf *ibuf;
709         int a, tot=0, index;
710         
711         if(totface+4>=MAXINDEX) return 0;
712         
713         /* brecht: disabled this because it obviously failes for
714            brushes with size > 64, why is this here? */
715         /*if(size>64.0) size= 64.0;*/
716         
717         ibuf= view3d_read_backbuf(vc, x-size, y-size, x+size, y+size);
718         if(ibuf) {
719                 unsigned int *rt= ibuf->rect;
720
721                 memset(indexar, 0, sizeof(int)*totface+4);      /* plus 2! first element is total, +2 was giving valgrind errors, +4 seems ok */
722                 
723                 size= ibuf->x*ibuf->y;
724                 while(size--) {
725                                 
726                         if(*rt) {
727                                 index= WM_framebuffer_to_index(*rt);
728                                 if(index>0 && index<=totface)
729                                         indexar[index] = 1;
730                         }
731                 
732                         rt++;
733                 }
734                 
735                 for(a=1; a<=totface; a++) {
736                         if(indexar[a]) indexar[tot++]= a;
737                 }
738
739                 IMB_freeImBuf(ibuf);
740         }
741         
742         return tot;
743 }
744
745 static int calc_vp_alpha_dl(VPaint *vp, ViewContext *vc, float vpimat[][3], float *vert_nor, float *mval)
746 {
747         Brush *brush = paint_brush(&vp->paint);
748         float fac, dx, dy;
749         int alpha;
750         short vertco[2];
751         
752         if(vp->flag & VP_SOFT) {
753                 project_short_noclip(vc->ar, vert_nor, vertco);
754                 dx= mval[0]-vertco[0];
755                 dy= mval[1]-vertco[1];
756                 
757                 fac= sqrt(dx*dx + dy*dy);
758                 if(fac > brush->size) return 0;
759                 if(vp->flag & VP_HARD)
760                         alpha= 255;
761                 else
762                         alpha= 255.0*brush->alpha*(1.0-fac/brush->size);
763         }
764         else {
765                 alpha= 255.0*brush->alpha;
766         }
767
768         if(vp->flag & VP_NORMALS) {
769                 float *no= vert_nor+3;
770                 
771                         /* transpose ! */
772                 fac= vpimat[2][0]*no[0]+vpimat[2][1]*no[1]+vpimat[2][2]*no[2];
773                 if(fac>0.0) {
774                         dx= vpimat[0][0]*no[0]+vpimat[0][1]*no[1]+vpimat[0][2]*no[2];
775                         dy= vpimat[1][0]*no[0]+vpimat[1][1]*no[1]+vpimat[1][2]*no[2];
776                         
777                         alpha*= fac/sqrt(dx*dx + dy*dy + fac*fac);
778                 }
779                 else return 0;
780         }
781         
782         return alpha;
783 }
784
785 static void wpaint_blend(VPaint *wp, MDeformWeight *dw, MDeformWeight *uw, float alpha, float paintval)
786 {
787         Brush *brush = paint_brush(&wp->paint);
788         
789         if(dw==NULL || uw==NULL) return;
790         
791         if(wp->mode==VP_MIX || wp->mode==VP_BLUR)
792                 dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
793         else if(wp->mode==VP_ADD)
794                 dw->weight += paintval*alpha;
795         else if(wp->mode==VP_SUB) 
796                 dw->weight -= paintval*alpha;
797         else if(wp->mode==VP_MUL) 
798                 /* first mul, then blend the fac */
799                 dw->weight = ((1.0-alpha) + alpha*paintval)*dw->weight;
800         else if(wp->mode==VP_LIGHTEN) {
801                 if (dw->weight < paintval)
802                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
803         } else if(wp->mode==VP_DARKEN) {
804                 if (dw->weight > paintval)
805                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
806         }
807         CLAMP(dw->weight, 0.0f, 1.0f);
808         
809         /* if no spray, clip result with orig weight & orig alpha */
810         if((wp->flag & VP_SPRAY)==0) {
811                 float testw=0.0f;
812                 
813                 alpha= brush->alpha;
814                 if(wp->mode==VP_MIX || wp->mode==VP_BLUR)
815                         testw = paintval*alpha + uw->weight*(1.0-alpha);
816                 else if(wp->mode==VP_ADD)
817                         testw = uw->weight + paintval*alpha;
818                 else if(wp->mode==VP_SUB) 
819                         testw = uw->weight - paintval*alpha;
820                 else if(wp->mode==VP_MUL) 
821                         /* first mul, then blend the fac */
822                         testw = ((1.0-alpha) + alpha*paintval)*uw->weight;              
823                 else if(wp->mode==VP_LIGHTEN) {
824                         if (uw->weight < paintval)
825                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
826                         else
827                                 testw = uw->weight;
828                 } else if(wp->mode==VP_DARKEN) {
829                         if (uw->weight > paintval)
830                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
831                         else
832                                 testw = uw->weight;
833                 }
834                 CLAMP(testw, 0.0f, 1.0f);
835                 
836                 if( testw<uw->weight ) {
837                         if(dw->weight < testw) dw->weight= testw;
838                         else if(dw->weight > uw->weight) dw->weight= uw->weight;
839                 }
840                 else {
841                         if(dw->weight > testw) dw->weight= testw;
842                         else if(dw->weight < uw->weight) dw->weight= uw->weight;
843                 }
844         }
845         
846 }
847
848 /* ----------------------------------------------------- */
849
850 /* used for 3d view, on active object, assumes me->dvert exists */
851 /* if mode==1: */
852 /*     samples cursor location, and gives menu with vertex groups to activate */
853 /* else */
854 /*     sets wp->weight to the closest weight value to vertex */
855 /*     note: we cant sample frontbuf, weight colors are interpolated too unpredictable */
856 void sample_wpaint(Scene *scene, ARegion *ar, View3D *v3d, int mode)
857 {
858         ViewContext vc;
859         ToolSettings *ts= scene->toolsettings;
860         Object *ob= OBACT;
861         Mesh *me= get_mesh(ob);
862         int index;
863         short mval[2], sco[2];
864
865         if (!me) return;
866         
867 //      getmouseco_areawin(mval);
868         index= view3d_sample_backbuf(&vc, mval[0], mval[1]);
869         
870         if(index && index<=me->totface) {
871                 MFace *mface;
872                 
873                 mface= ((MFace *)me->mface) + index-1;
874                 
875                 if(mode==1) {   /* sampe which groups are in here */
876                         MDeformVert *dv;
877                         int a, totgroup;
878                         
879                         totgroup= BLI_countlist(&ob->defbase);
880                         if(totgroup) {
881                                 int totmenu=0;
882                                 int *groups=MEM_callocN(totgroup*sizeof(int), "groups");
883                                 
884                                 dv= me->dvert+mface->v1;
885                                 for(a=0; a<dv->totweight; a++) {
886                                         if (dv->dw[a].def_nr<totgroup)
887                                                 groups[dv->dw[a].def_nr]= 1;
888                                 }
889                                 dv= me->dvert+mface->v2;
890                                 for(a=0; a<dv->totweight; a++) {
891                                         if (dv->dw[a].def_nr<totgroup)
892                                                 groups[dv->dw[a].def_nr]= 1;
893                                 }
894                                 dv= me->dvert+mface->v3;
895                                 for(a=0; a<dv->totweight; a++) {
896                                         if (dv->dw[a].def_nr<totgroup)
897                                                 groups[dv->dw[a].def_nr]= 1;
898                                 }
899                                 if(mface->v4) {
900                                         dv= me->dvert+mface->v4;
901                                         for(a=0; a<dv->totweight; a++) {
902                                                 if (dv->dw[a].def_nr<totgroup)
903                                                         groups[dv->dw[a].def_nr]= 1;
904                                         }
905                                 }
906                                 for(a=0; a<totgroup; a++)
907                                         if(groups[a]) totmenu++;
908                                 
909                                 if(totmenu==0) {
910                                         //notice("No Vertex Group Selected");
911                                 }
912                                 else {
913                                         bDeformGroup *dg;
914                                         short val;
915                                         char item[40], *str= MEM_mallocN(40*totmenu+40, "menu");
916                                         
917                                         strcpy(str, "Vertex Groups %t");
918                                         for(a=0, dg=ob->defbase.first; dg && a<totgroup; a++, dg= dg->next) {
919                                                 if(groups[a]) {
920                                                         sprintf(item, "|%s %%x%d", dg->name, a);
921                                                         strcat(str, item);
922                                                 }
923                                         }
924                                         
925                                         val= 0; // XXX pupmenu(str);
926                                         if(val>=0) {
927                                                 ob->actdef= val+1;
928                                                 DAG_id_flush_update(&me->id, OB_RECALC_DATA);
929                                         }
930                                         MEM_freeN(str);
931                                 }
932                                 MEM_freeN(groups);
933                         }
934 //                      else notice("No Vertex Groups in Object");
935                 }
936                 else {
937                         DerivedMesh *dm;
938                         MDeformWeight *dw;
939                         float w1, w2, w3, w4, co[3], fac;
940                         
941                         dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
942                         if(dm->getVertCo==NULL) {
943                                 //notice("Not supported yet");
944                         }
945                         else {
946                                 /* calc 3 or 4 corner weights */
947                                 dm->getVertCo(dm, mface->v1, co);
948                                 project_short_noclip(ar, co, sco);
949                                 w1= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
950                                 
951                                 dm->getVertCo(dm, mface->v2, co);
952                                 project_short_noclip(ar, co, sco);
953                                 w2= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
954                                 
955                                 dm->getVertCo(dm, mface->v3, co);
956                                 project_short_noclip(ar, co, sco);
957                                 w3= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
958                                 
959                                 if(mface->v4) {
960                                         dm->getVertCo(dm, mface->v4, co);
961                                         project_short_noclip(ar, co, sco);
962                                         w4= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
963                                 }
964                                 else w4= 1.0e10;
965                                 
966                                 fac= MIN4(w1, w2, w3, w4);
967                                 if(w1==fac) {
968                                         dw= ED_vgroup_weight_get(me->dvert+mface->v1, ob->actdef-1);
969                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
970                                 }
971                                 else if(w2==fac) {
972                                         dw= ED_vgroup_weight_get(me->dvert+mface->v2, ob->actdef-1);
973                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
974                                 }
975                                 else if(w3==fac) {
976                                         dw= ED_vgroup_weight_get(me->dvert+mface->v3, ob->actdef-1);
977                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
978                                 }
979                                 else if(w4==fac) {
980                                         if(mface->v4) {
981                                                 dw= ED_vgroup_weight_get(me->dvert+mface->v4, ob->actdef-1);
982                                                 if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
983                                         }
984                                 }
985                         }
986                         dm->release(dm);
987                 }               
988                 
989         }
990         
991 }
992
993 static void do_weight_paint_vertex(VPaint *wp, Object *ob, int index, int alpha, float paintweight, int vgroup_mirror)
994 {
995         Mesh *me= ob->data;
996         MDeformWeight *dw, *uw;
997         int vgroup= ob->actdef-1;
998         
999         if(wp->flag & VP_ONLYVGROUP) {
1000                 dw= ED_vgroup_weight_get(me->dvert+index, vgroup);
1001                 uw= ED_vgroup_weight_get(wp->wpaint_prev+index, vgroup);
1002         }
1003         else {
1004                 dw= ED_vgroup_weight_verify(me->dvert+index, vgroup);
1005                 uw= ED_vgroup_weight_verify(wp->wpaint_prev+index, vgroup);
1006         }
1007         if(dw==NULL || uw==NULL)
1008                 return;
1009         
1010         wpaint_blend(wp, dw, uw, (float)alpha/255.0, paintweight);
1011         
1012         if(me->editflag & ME_EDIT_MIRROR_X) {   /* x mirror painting */
1013                 int j= mesh_get_x_mirror_vert(ob, index);
1014                 if(j>=0) {
1015                         /* copy, not paint again */
1016                         if(vgroup_mirror != -1)
1017                                 uw= ED_vgroup_weight_verify(me->dvert+j, vgroup_mirror);
1018                         else
1019                                 uw= ED_vgroup_weight_verify(me->dvert+j, vgroup);
1020                                 
1021                         uw->weight= dw->weight;
1022                 }
1023         }
1024 }
1025
1026
1027 /* *************** set wpaint operator ****************** */
1028
1029 static int set_wpaint(bContext *C, wmOperator *op)              /* toggle */
1030 {               
1031         Object *ob= CTX_data_active_object(C);
1032         Scene *scene= CTX_data_scene(C);
1033         VPaint *wp= scene->toolsettings->wpaint;
1034         Mesh *me;
1035         
1036         me= get_mesh(ob);
1037         if(ob->id.lib || me==NULL) return OPERATOR_PASS_THROUGH;
1038         
1039         if(me && me->totface>=MAXINDEX) {
1040                 error("Maximum number of faces: %d", MAXINDEX-1);
1041                 ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1042                 return OPERATOR_CANCELLED;
1043         }
1044         
1045         if(ob->mode & OB_MODE_WEIGHT_PAINT) ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1046         else ob->mode |= OB_MODE_WEIGHT_PAINT;
1047         
1048         
1049         /* Weightpaint works by overriding colors in mesh,
1050                 * so need to make sure we recalc on enter and
1051                 * exit (exit needs doing regardless because we
1052                                 * should redeform).
1053                 */
1054         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
1055         
1056         if(ob->mode & OB_MODE_WEIGHT_PAINT) {
1057                 Object *par;
1058                 
1059                 if(wp==NULL)
1060                         wp= scene->toolsettings->wpaint= new_vpaint(1);
1061
1062                 paint_init(&wp->paint, PAINT_CURSOR_WEIGHT_PAINT);
1063                 paint_cursor_start(C, weight_paint_poll);
1064                 
1065                 mesh_octree_table(ob, NULL, NULL, 's');
1066                 
1067                 /* verify if active weight group is also active bone */
1068                 par= modifiers_isDeformedByArmature(ob);
1069                 if(par && (par->mode & OB_MODE_POSE)) {
1070                         bPoseChannel *pchan;
1071                         for(pchan= par->pose->chanbase.first; pchan; pchan= pchan->next)
1072                                 if(pchan->bone->flag & BONE_ACTIVE)
1073                                         break;
1074                                 if(pchan)
1075                                         ED_vgroup_select_by_name(ob, pchan->name);
1076                 }
1077         }
1078         else {
1079                 mesh_octree_table(ob, NULL, NULL, 'e');
1080         }
1081         
1082         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1083         
1084         return OPERATOR_FINISHED;
1085 }
1086
1087 /* for switching to/from mode */
1088 static int paint_poll_test(bContext *C)
1089 {
1090         if(ED_operator_view3d_active(C)==0)
1091                 return 0;
1092         if(CTX_data_edit_object(C))
1093                 return 0;
1094         if(CTX_data_active_object(C)==NULL)
1095                 return 0;
1096         return 1;
1097 }
1098
1099 void PAINT_OT_weight_paint_toggle(wmOperatorType *ot)
1100 {
1101         
1102         /* identifiers */
1103         ot->name= "Weight Paint Mode";
1104         ot->idname= "PAINT_OT_weight_paint_toggle";
1105         
1106         /* api callbacks */
1107         ot->exec= set_wpaint;
1108         ot->poll= paint_poll_test;
1109         
1110         /* flags */
1111         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1112         
1113 }
1114
1115 /* ************ paint radial controls *************/
1116
1117 static int vpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1118 {
1119         Paint *p = paint_get_active(CTX_data_scene(C));
1120         Brush *brush = paint_brush(p);
1121         
1122         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1123         p->paint_cursor = NULL;
1124         brush_radial_control_invoke(op, brush, 1);
1125         return WM_radial_control_invoke(C, op, event);
1126 }
1127
1128 static int vpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1129 {
1130         int ret = WM_radial_control_modal(C, op, event);
1131         if(ret != OPERATOR_RUNNING_MODAL)
1132                 paint_cursor_start(C, vertex_paint_poll);
1133         return ret;
1134 }
1135
1136 static int vpaint_radial_control_exec(bContext *C, wmOperator *op)
1137 {
1138         Brush *brush = paint_brush(&CTX_data_scene(C)->toolsettings->vpaint->paint);
1139         int ret = brush_radial_control_exec(op, brush, 1);
1140         
1141         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1142         
1143         return ret;
1144 }
1145
1146 static int wpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1147 {
1148         Paint *p = paint_get_active(CTX_data_scene(C));
1149         Brush *brush = paint_brush(p);
1150         
1151         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1152         p->paint_cursor = NULL;
1153         brush_radial_control_invoke(op, brush, 1);
1154         return WM_radial_control_invoke(C, op, event);
1155 }
1156
1157 static int wpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1158 {
1159         int ret = WM_radial_control_modal(C, op, event);
1160         if(ret != OPERATOR_RUNNING_MODAL)
1161                 paint_cursor_start(C, weight_paint_poll);
1162         return ret;
1163 }
1164
1165 static int wpaint_radial_control_exec(bContext *C, wmOperator *op)
1166 {
1167         Brush *brush = paint_brush(&CTX_data_scene(C)->toolsettings->wpaint->paint);
1168         int ret = brush_radial_control_exec(op, brush, 1);
1169         
1170         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1171         
1172         return ret;
1173 }
1174
1175 void PAINT_OT_weight_paint_radial_control(wmOperatorType *ot)
1176 {
1177         WM_OT_radial_control_partial(ot);
1178
1179         ot->name= "Weight Paint Radial Control";
1180         ot->idname= "PAINT_OT_weight_paint_radial_control";
1181
1182         ot->invoke= wpaint_radial_control_invoke;
1183         ot->modal= wpaint_radial_control_modal;
1184         ot->exec= wpaint_radial_control_exec;
1185         ot->poll= weight_paint_poll;
1186         
1187         /* flags */
1188         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1189 }
1190
1191 void PAINT_OT_vertex_paint_radial_control(wmOperatorType *ot)
1192 {
1193         WM_OT_radial_control_partial(ot);
1194
1195         ot->name= "Vertex Paint Radial Control";
1196         ot->idname= "PAINT_OT_vertex_paint_radial_control";
1197
1198         ot->invoke= vpaint_radial_control_invoke;
1199         ot->modal= vpaint_radial_control_modal;
1200         ot->exec= vpaint_radial_control_exec;
1201         ot->poll= vertex_paint_poll;
1202         
1203         /* flags */
1204         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1205 }
1206
1207 /* ************ weight paint operator ********** */
1208
1209 struct WPaintData {
1210         ViewContext vc;
1211         int *indexar;
1212         int vgroup_mirror;
1213         float *vertexcosnos;
1214         float wpimat[3][3];
1215 };
1216
1217 static int wpaint_stroke_test_start(bContext *C, wmOperator *op, wmEvent *event)
1218 {
1219         Scene *scene= CTX_data_scene(C);
1220         struct PaintStroke *stroke = op->customdata;
1221         ToolSettings *ts= CTX_data_tool_settings(C);
1222         VPaint *wp= ts->wpaint;
1223         Object *ob= CTX_data_active_object(C);
1224         struct WPaintData *wpd;
1225         Mesh *me;
1226         float mat[4][4], imat[4][4];
1227         
1228         if(scene->obedit) return OPERATOR_CANCELLED;
1229         
1230         me= get_mesh(ob);
1231         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1232         
1233         /* if nothing was added yet, we make dverts and a vertex deform group */
1234         if (!me->dvert)
1235                 ED_vgroup_data_create(&me->id);
1236         
1237         /* make mode data storage */
1238         wpd= MEM_callocN(sizeof(struct WPaintData), "WPaintData");
1239         paint_stroke_set_mode_data(stroke, wpd);
1240         view3d_set_viewcontext(C, &wpd->vc);
1241         wpd->vgroup_mirror= -1;
1242         
1243         //      if(qual & LR_CTRLKEY) {
1244         //              sample_wpaint(scene, ar, v3d, 0);
1245         //              return;
1246         //      }
1247         //      if(qual & LR_SHIFTKEY) {
1248         //              sample_wpaint(scene, ar, v3d, 1);
1249         //              return;
1250         //      }
1251         
1252         /* ALLOCATIONS! no return after this line */
1253         /* painting on subsurfs should give correct points too, this returns me->totvert amount */
1254         wpd->vertexcosnos= mesh_get_mapped_verts_nors(scene, ob);
1255         wpd->indexar= get_indexarray();
1256         copy_wpaint_prev(wp, me->dvert, me->totvert);
1257         
1258         /* this happens on a Bone select, when no vgroup existed yet */
1259         if(ob->actdef<=0) {
1260                 Object *modob;
1261                 if((modob = modifiers_isDeformedByArmature(ob))) {
1262                         bPoseChannel *pchan;
1263                         for(pchan= modob->pose->chanbase.first; pchan; pchan= pchan->next)
1264                                 if(pchan->bone->flag & SELECT)
1265                                         break;
1266                         if(pchan) {
1267                                 bDeformGroup *dg= get_named_vertexgroup(ob, pchan->name);
1268                                 if(dg==NULL)
1269                                         dg= ED_vgroup_add_name(ob, pchan->name);        /* sets actdef */
1270                                 else
1271                                         ob->actdef= get_defgroup_num(ob, dg);
1272                         }
1273                 }
1274         }
1275         if(ob->defbase.first==NULL) {
1276                 ED_vgroup_add(ob);
1277         }       
1278         
1279         //      if(ob->lay & v3d->lay); else error("Active object is not in this layer");
1280         
1281         /* imat for normals */
1282         Mat4MulMat4(mat, ob->obmat, wpd->vc.rv3d->viewmat);
1283         Mat4Invert(imat, mat);
1284         Mat3CpyMat4(wpd->wpimat, imat);
1285         
1286         /* if mirror painting, find the other group */
1287         if(me->editflag & ME_EDIT_MIRROR_X) {
1288                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
1289                 if(defgroup) {
1290                         bDeformGroup *curdef;
1291                         int actdef= 0;
1292                         char name[32];
1293                         
1294                         BLI_strncpy(name, defgroup->name, 32);
1295                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
1296                         
1297                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
1298                                 if (!strcmp(curdef->name, name))
1299                                         break;
1300                         if(curdef==NULL) {
1301                                 int olddef= ob->actdef; /* tsk, ED_vgroup_add sets the active defgroup */
1302                                 curdef= ED_vgroup_add_name (ob, name);
1303                                 ob->actdef= olddef;
1304                         }
1305                         
1306                         if(curdef && curdef!=defgroup)
1307                                 wpd->vgroup_mirror= actdef;
1308                 }
1309         }
1310         
1311         return 1;
1312 }
1313
1314 static void wpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1315 {
1316         ToolSettings *ts= CTX_data_tool_settings(C);
1317         VPaint *wp= ts->wpaint;
1318         Brush *brush = paint_brush(&wp->paint);
1319         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1320         ViewContext *vc= &wpd->vc;
1321         Object *ob= vc->obact;
1322         Mesh *me= ob->data;
1323         float mat[4][4];
1324         float paintweight= ts->vgroup_weight;
1325         int *indexar= wpd->indexar;
1326         int totindex, index, alpha, totw;
1327         float mval[2];
1328
1329         view3d_operator_needs_opengl(C);
1330                         
1331         /* load projection matrix */
1332         wmMultMatrix(ob->obmat);
1333         wmGetSingleMatrix(mat);
1334         wmLoadMatrix(wpd->vc.rv3d->viewmat);
1335
1336         RNA_float_get_array(itemptr, "mouse", mval);
1337         mval[0]-= vc->ar->winrct.xmin;
1338         mval[1]-= vc->ar->winrct.ymin;
1339                         
1340         Mat4SwapMat4(wpd->vc.rv3d->persmat, mat);
1341                         
1342         /* which faces are involved */
1343         if(wp->flag & VP_AREA) {
1344                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush->size);
1345         }
1346         else {
1347                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1348                 if(indexar[0]) totindex= 1;
1349                 else totindex= 0;
1350         }
1351                         
1352         if(wp->flag & VP_COLINDEX) {
1353                 for(index=0; index<totindex; index++) {
1354                         if(indexar[index] && indexar[index]<=me->totface) {
1355                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1356                                                 
1357                                 if(mface->mat_nr!=ob->actcol-1) {
1358                                         indexar[index]= 0;
1359                                 }
1360                         }                                       
1361                 }
1362         }
1363                         
1364         if((me->editflag & ME_EDIT_PAINT_MASK) && me->mface) {
1365                 for(index=0; index<totindex; index++) {
1366                         if(indexar[index] && indexar[index]<=me->totface) {
1367                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1368                                                 
1369                                 if((mface->flag & ME_FACE_SEL)==0) {
1370                                         indexar[index]= 0;
1371                                 }
1372                         }                                       
1373                 }
1374         }
1375                         
1376         /* make sure each vertex gets treated only once */
1377         /* and calculate filter weight */
1378         totw= 0;
1379         if(wp->mode==VP_BLUR) 
1380                 paintweight= 0.0f;
1381         else
1382                 paintweight= ts->vgroup_weight;
1383                         
1384         for(index=0; index<totindex; index++) {
1385                 if(indexar[index] && indexar[index]<=me->totface) {
1386                         MFace *mface= me->mface + (indexar[index]-1);
1387                                         
1388                         (me->dvert+mface->v1)->flag= 1;
1389                         (me->dvert+mface->v2)->flag= 1;
1390                         (me->dvert+mface->v3)->flag= 1;
1391                         if(mface->v4) (me->dvert+mface->v4)->flag= 1;
1392                                         
1393                         if(wp->mode==VP_BLUR) {
1394                                 MDeformWeight *dw, *(*dw_func)(MDeformVert *, int) = ED_vgroup_weight_verify;
1395                                                 
1396                                 if(wp->flag & VP_ONLYVGROUP)
1397                                         dw_func= ED_vgroup_weight_get;
1398                                                 
1399                                 dw= dw_func(me->dvert+mface->v1, ob->actdef-1);
1400                                 if(dw) {paintweight+= dw->weight; totw++;}
1401                                 dw= dw_func(me->dvert+mface->v2, ob->actdef-1);
1402                                 if(dw) {paintweight+= dw->weight; totw++;}
1403                                 dw= dw_func(me->dvert+mface->v3, ob->actdef-1);
1404                                 if(dw) {paintweight+= dw->weight; totw++;}
1405                                 if(mface->v4) {
1406                                         dw= dw_func(me->dvert+mface->v4, ob->actdef-1);
1407                                         if(dw) {paintweight+= dw->weight; totw++;}
1408                                 }
1409                         }
1410                 }
1411         }
1412                         
1413         if(wp->mode==VP_BLUR) 
1414                 paintweight/= (float)totw;
1415                         
1416         for(index=0; index<totindex; index++) {
1417                                 
1418                 if(indexar[index] && indexar[index]<=me->totface) {
1419                         MFace *mface= me->mface + (indexar[index]-1);
1420                                         
1421                         if((me->dvert+mface->v1)->flag) {
1422                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v1, mval);
1423                                 if(alpha) {
1424                                         do_weight_paint_vertex(wp, ob, mface->v1, alpha, paintweight, wpd->vgroup_mirror);
1425                                 }
1426                                 (me->dvert+mface->v1)->flag= 0;
1427                         }
1428                                         
1429                         if((me->dvert+mface->v2)->flag) {
1430                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v2, mval);
1431                                 if(alpha) {
1432                                         do_weight_paint_vertex(wp, ob, mface->v2, alpha, paintweight, wpd->vgroup_mirror);
1433                                 }
1434                                 (me->dvert+mface->v2)->flag= 0;
1435                         }
1436                                         
1437                         if((me->dvert+mface->v3)->flag) {
1438                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v3, mval);
1439                                 if(alpha) {
1440                                         do_weight_paint_vertex(wp, ob, mface->v3, alpha, paintweight, wpd->vgroup_mirror);
1441                                 }
1442                                 (me->dvert+mface->v3)->flag= 0;
1443                         }
1444                                         
1445                         if((me->dvert+mface->v4)->flag) {
1446                                 if(mface->v4) {
1447                                         alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v4, mval);
1448                                         if(alpha) {
1449                                                 do_weight_paint_vertex(wp, ob, mface->v4, alpha, paintweight, wpd->vgroup_mirror);
1450                                         }
1451                                         (me->dvert+mface->v4)->flag= 0;
1452                                 }
1453                         }
1454                 }
1455         }
1456                         
1457         Mat4SwapMat4(vc->rv3d->persmat, mat);
1458                         
1459         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1460         ED_region_tag_redraw(vc->ar);
1461 }
1462
1463 static void wpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
1464 {
1465         ToolSettings *ts= CTX_data_tool_settings(C);
1466         Object *ob= CTX_data_active_object(C);
1467         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1468         
1469         if(wpd->vertexcosnos)
1470                 MEM_freeN(wpd->vertexcosnos);
1471         MEM_freeN(wpd->indexar);
1472         
1473         /* frees prev buffer */
1474         copy_wpaint_prev(ts->wpaint, NULL, 0);
1475         
1476         /* and particles too */
1477         if(ob->particlesystem.first) {
1478                 ParticleSystem *psys;
1479                 int i;
1480                 
1481                 for(psys= ob->particlesystem.first; psys; psys= psys->next) {
1482                         for(i=0; i<PSYS_TOT_VG; i++) {
1483                                 if(psys->vgroup[i]==ob->actdef) {
1484                                         psys->recalc |= PSYS_RECALC_RESET;
1485                                         break;
1486                                 }
1487                         }
1488                 }
1489         }
1490         
1491         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1492         
1493         MEM_freeN(wpd);
1494 }
1495
1496
1497 static int wpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1498 {
1499         
1500         op->customdata = paint_stroke_new(C, wpaint_stroke_test_start,
1501                                           wpaint_stroke_update_step,
1502                                           wpaint_stroke_done);
1503         
1504         /* add modal handler */
1505         WM_event_add_modal_handler(C, op);
1506
1507         op->type->modal(C, op, event);
1508         
1509         return OPERATOR_RUNNING_MODAL;
1510 }
1511
1512 void PAINT_OT_weight_paint(wmOperatorType *ot)
1513 {
1514         
1515         /* identifiers */
1516         ot->name= "Weight Paint";
1517         ot->idname= "PAINT_OT_weight_paint";
1518         
1519         /* api callbacks */
1520         ot->invoke= wpaint_invoke;
1521         ot->modal= paint_stroke_modal;
1522         /* ot->exec= vpaint_exec; <-- needs stroke property */
1523         ot->poll= weight_paint_poll;
1524         
1525         /* flags */
1526         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1527
1528         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1529 }
1530
1531 /* ************ set / clear vertex paint mode ********** */
1532
1533
1534 static int set_vpaint(bContext *C, wmOperator *op)              /* toggle */
1535 {       
1536         Object *ob= CTX_data_active_object(C);
1537         Scene *scene= CTX_data_scene(C);
1538         VPaint *vp= scene->toolsettings->vpaint;
1539         Mesh *me;
1540         
1541         me= get_mesh(ob);
1542         
1543         if(me==NULL || object_data_is_libdata(ob)) {
1544                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1545                 return OPERATOR_PASS_THROUGH;
1546         }
1547         
1548         if(me && me->totface>=MAXINDEX) {
1549                 error("Maximum number of faces: %d", MAXINDEX-1);
1550                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1551                 return OPERATOR_FINISHED;
1552         }
1553         
1554         if(me && me->mcol==NULL) make_vertexcol(scene, 0);
1555         
1556         /* toggle: end vpaint */
1557         if(ob->mode & OB_MODE_VERTEX_PAINT) {
1558                 
1559                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1560         }
1561         else {
1562                 ob->mode |= OB_MODE_VERTEX_PAINT;
1563                 /* Turn off weight painting */
1564                 if (ob->mode & OB_MODE_WEIGHT_PAINT)
1565                         set_wpaint(C, op);
1566                 
1567                 if(vp==NULL)
1568                         vp= scene->toolsettings->vpaint= new_vpaint(0);
1569                 
1570                 paint_cursor_start(C, vertex_paint_poll);
1571
1572                 paint_init(&vp->paint, PAINT_CURSOR_VERTEX_PAINT);
1573         }
1574         
1575         if (me)
1576                 /* update modifier stack for mapping requirements */
1577                 DAG_id_flush_update(&me->id, OB_RECALC_DATA);
1578         
1579         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1580         
1581         return OPERATOR_FINISHED;
1582 }
1583
1584 void PAINT_OT_vertex_paint_toggle(wmOperatorType *ot)
1585 {
1586         
1587         /* identifiers */
1588         ot->name= "Vertex Paint Mode";
1589         ot->idname= "PAINT_OT_vertex_paint_toggle";
1590         
1591         /* api callbacks */
1592         ot->exec= set_vpaint;
1593         ot->poll= paint_poll_test;
1594         
1595         /* flags */
1596         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1597         
1598 }
1599
1600
1601
1602 /* ********************** vertex paint operator ******************* */
1603
1604 /* Implementation notes:
1605
1606 Operator->invoke()
1607   - validate context (add mcol)
1608   - create customdata storage
1609   - call paint once (mouse click)
1610   - add modal handler 
1611
1612 Operator->modal()
1613   - for every mousemove, apply vertex paint
1614   - exit on mouse release, free customdata
1615     (return OPERATOR_FINISHED also removes handler and operator)
1616
1617 For future:
1618   - implement a stroke event (or mousemove with past positons)
1619   - revise whether op->customdata should be added in object, in set_vpaint
1620
1621 */
1622
1623 typedef struct VPaintData {
1624         ViewContext vc;
1625         unsigned int paintcol;
1626         int *indexar;
1627         float *vertexcosnos;
1628         float vpimat[3][3];
1629 } VPaintData;
1630
1631 static int vpaint_stroke_test_start(bContext *C, struct wmOperator *op, wmEvent *event)
1632 {
1633         ToolSettings *ts= CTX_data_tool_settings(C);
1634         struct PaintStroke *stroke = op->customdata;
1635         VPaint *vp= ts->vpaint;
1636         struct VPaintData *vpd;
1637         Object *ob= CTX_data_active_object(C);
1638         Mesh *me;
1639         float mat[4][4], imat[4][4];
1640
1641         /* context checks could be a poll() */
1642         me= get_mesh(ob);
1643         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1644         
1645         if(me->mcol==NULL) make_vertexcol(CTX_data_scene(C), 0);
1646         if(me->mcol==NULL) return OPERATOR_CANCELLED;
1647         
1648         /* make mode data storage */
1649         vpd= MEM_callocN(sizeof(struct VPaintData), "VPaintData");
1650         paint_stroke_set_mode_data(stroke, vpd);
1651         view3d_set_viewcontext(C, &vpd->vc);
1652         
1653         vpd->vertexcosnos= mesh_get_mapped_verts_nors(vpd->vc.scene, ob);
1654         vpd->indexar= get_indexarray();
1655         vpd->paintcol= vpaint_get_current_col(vp);
1656         
1657         /* for filtering */
1658         copy_vpaint_prev(vp, (unsigned int *)me->mcol, me->totface);
1659         
1660         /* some old cruft to sort out later */
1661         Mat4MulMat4(mat, ob->obmat, vpd->vc.rv3d->viewmat);
1662         Mat4Invert(imat, mat);
1663         Mat3CpyMat4(vpd->vpimat, imat);
1664
1665         return 1;
1666 }
1667
1668 static void vpaint_paint_face(VPaint *vp, VPaintData *vpd, Object *ob, int index, float mval[2])
1669 {
1670         ViewContext *vc = &vpd->vc;
1671         Mesh *me = get_mesh(ob);
1672         MFace *mface= ((MFace*)me->mface) + index;
1673         unsigned int *mcol= ((unsigned int*)me->mcol) + 4*index;
1674         unsigned int *mcolorig= ((unsigned int*)vp->vpaint_prev) + 4*index;
1675         int alpha, i;
1676         
1677         if((vp->flag & VP_COLINDEX && mface->mat_nr!=ob->actcol-1) ||
1678            ((me->editflag & ME_EDIT_PAINT_MASK) && !(mface->flag & ME_FACE_SEL)))
1679                 return;
1680
1681         if(vp->mode==VP_BLUR) {
1682                 unsigned int fcol1= mcol_blend( mcol[0], mcol[1], 128);
1683                 if(mface->v4) {
1684                         unsigned int fcol2= mcol_blend( mcol[2], mcol[3], 128);
1685                         vpd->paintcol= mcol_blend( fcol1, fcol2, 128);
1686                 }
1687                 else {
1688                         vpd->paintcol= mcol_blend( mcol[2], fcol1, 170);
1689                 }
1690                 
1691         }
1692
1693         for(i = 0; i < (mface->v4 ? 4 : 3); ++i) {
1694                 alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*(&mface->v1)[i], mval);
1695                 if(alpha)
1696                         vpaint_blend(vp, mcol+i, mcolorig+i, vpd->paintcol, alpha);
1697         }
1698 }
1699
1700 static void vpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1701 {
1702         ToolSettings *ts= CTX_data_tool_settings(C);
1703         struct VPaintData *vpd = paint_stroke_mode_data(stroke);
1704         VPaint *vp= ts->vpaint;
1705         Brush *brush = paint_brush(&vp->paint);
1706         ViewContext *vc= &vpd->vc;
1707         Object *ob= vc->obact;
1708         Mesh *me= ob->data;
1709         float mat[4][4];
1710         int *indexar= vpd->indexar;
1711         int totindex, index;
1712         float mval[2];
1713
1714         RNA_float_get_array(itemptr, "mouse", mval);
1715                         
1716         view3d_operator_needs_opengl(C);
1717                         
1718         /* load projection matrix */
1719         wmMultMatrix(ob->obmat);
1720         wmGetSingleMatrix(mat);
1721         wmLoadMatrix(vc->rv3d->viewmat);
1722
1723         mval[0]-= vc->ar->winrct.xmin;
1724         mval[1]-= vc->ar->winrct.ymin;
1725
1726                         
1727         /* which faces are involved */
1728         if(vp->flag & VP_AREA) {
1729                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush->size);
1730         }
1731         else {
1732                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1733                 if(indexar[0]) totindex= 1;
1734                 else totindex= 0;
1735         }
1736                         
1737         Mat4SwapMat4(vc->rv3d->persmat, mat);
1738                         
1739         for(index=0; index<totindex; index++) {                         
1740                 if(indexar[index] && indexar[index]<=me->totface)
1741                         vpaint_paint_face(vp, vpd, ob, indexar[index]-1, mval);
1742         }
1743                                                 
1744         Mat4SwapMat4(vc->rv3d->persmat, mat);
1745
1746         /* was disabled because it is slow, but necessary for blur */
1747         if(vp->mode == VP_BLUR)
1748                 do_shared_vertexcol(me);
1749                         
1750         ED_region_tag_redraw(vc->ar);
1751                         
1752         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1753 }
1754
1755 static void vpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
1756 {
1757         ToolSettings *ts= CTX_data_tool_settings(C);
1758         struct VPaintData *vpd= paint_stroke_mode_data(stroke);
1759         
1760         if(vpd->vertexcosnos)
1761                 MEM_freeN(vpd->vertexcosnos);
1762         MEM_freeN(vpd->indexar);
1763         
1764         /* frees prev buffer */
1765         copy_vpaint_prev(ts->vpaint, NULL, 0);
1766         
1767         MEM_freeN(vpd);
1768 }
1769
1770 static int vpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1771 {
1772         
1773         op->customdata = paint_stroke_new(C, vpaint_stroke_test_start,
1774                                           vpaint_stroke_update_step,
1775                                           vpaint_stroke_done);
1776         
1777         /* add modal handler */
1778         WM_event_add_modal_handler(C, op);
1779
1780         op->type->modal(C, op, event);
1781         
1782         return OPERATOR_RUNNING_MODAL;
1783 }
1784
1785 void PAINT_OT_vertex_paint(wmOperatorType *ot)
1786 {
1787         /* identifiers */
1788         ot->name= "Vertex Paint";
1789         ot->idname= "PAINT_OT_vertex_paint";
1790         
1791         /* api callbacks */
1792         ot->invoke= vpaint_invoke;
1793         ot->modal= paint_stroke_modal;
1794         /* ot->exec= vpaint_exec; <-- needs stroke property */
1795         ot->poll= vertex_paint_poll;
1796         
1797         /* flags */
1798         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1799
1800         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1801 }
1802