2.5 Paint:
[blender.git] / source / blender / editors / sculpt_paint / paint_vertex.c
1 /**
2  * $Id$
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  */
29
30 #include <math.h>
31 #include <string.h>
32
33 #ifdef WIN32
34 #include <io.h>
35 #else
36 #include <unistd.h>
37 #endif   
38
39 #include "MEM_guardedalloc.h"
40
41 #include "IMB_imbuf.h"
42 #include "IMB_imbuf_types.h"
43
44 #include "BLI_blenlib.h"
45 #include "BLI_arithb.h"
46 #include "MTC_matrixops.h"
47
48 #include "DNA_action_types.h"
49 #include "DNA_armature_types.h"
50 #include "DNA_brush_types.h"
51 #include "DNA_cloth_types.h"
52 #include "DNA_mesh_types.h"
53 #include "DNA_meshdata_types.h"
54 #include "DNA_modifier_types.h"
55 #include "DNA_object_types.h"
56 #include "DNA_object_force.h"
57 #include "DNA_particle_types.h"
58 #include "DNA_screen_types.h"
59 #include "DNA_space_types.h"
60 #include "DNA_scene_types.h"
61 #include "DNA_view3d_types.h"
62 #include "DNA_userdef_types.h"
63
64 #include "RNA_access.h"
65 #include "RNA_define.h"
66
67 #include "BKE_armature.h"
68 #include "BKE_brush.h"
69 #include "BKE_DerivedMesh.h"
70 #include "BKE_cloth.h"
71 #include "BKE_context.h"
72 #include "BKE_customdata.h"
73 #include "BKE_depsgraph.h"
74 #include "BKE_deform.h"
75 #include "BKE_displist.h"
76 #include "BKE_global.h"
77 #include "BKE_mesh.h"
78 #include "BKE_modifier.h"
79 #include "BKE_object.h"
80 #include "BKE_paint.h"
81 #include "BKE_utildefines.h"
82
83 #include "WM_api.h"
84 #include "WM_types.h"
85
86 #include "BIF_gl.h"
87 #include "BIF_glutil.h"
88
89 #include "ED_mesh.h"
90 #include "ED_object.h"
91 #include "ED_screen.h"
92 #include "ED_util.h"
93 #include "ED_view3d.h"
94
95 #include "paint_intern.h"
96
97         /* vp->mode */
98 #define VP_MIX  0
99 #define VP_ADD  1
100 #define VP_SUB  2
101 #define VP_MUL  3
102 #define VP_BLUR 4
103 #define VP_LIGHTEN      5
104 #define VP_DARKEN       6
105
106 #define MAXINDEX        512000
107
108 /* XXX */
109 static void error() {}
110
111 /* polling - retrieve whether cursor should be set or operator should be done */
112
113 static int vp_poll(bContext *C)
114 {
115         Object *ob = CTX_data_active_object(C);
116
117         if(ob && ob->mode & OB_MODE_VERTEX_PAINT &&
118            paint_brush(&CTX_data_tool_settings(C)->vpaint->paint)) {
119                 ScrArea *sa= CTX_wm_area(C);
120                 if(sa->spacetype==SPACE_VIEW3D) {
121                         ARegion *ar= CTX_wm_region(C);
122                         if(ar->regiontype==RGN_TYPE_WINDOW)
123                                 return 1;
124                 }
125         }
126         return 0;
127 }
128
129 static int wp_poll(bContext *C)
130 {
131         Object *ob = CTX_data_active_object(C);
132
133         if(ob && ob->mode & OB_MODE_WEIGHT_PAINT &&
134            paint_brush(&CTX_data_tool_settings(C)->wpaint->paint)) {
135                 ScrArea *sa= CTX_wm_area(C);
136                 if(sa->spacetype==SPACE_VIEW3D) {
137                         ARegion *ar= CTX_wm_region(C);
138                         if(ar->regiontype==RGN_TYPE_WINDOW)
139                                 return 1;
140                 }
141         }
142         return 0;
143 }
144
145 static VPaint *new_vpaint(int wpaint)
146 {
147         VPaint *vp= MEM_callocN(sizeof(VPaint), "VPaint");
148         
149         vp->gamma= vp->mul= 1.0f;
150         
151         vp->flag= VP_AREA+VP_SOFT+VP_SPRAY;
152         
153         if(wpaint)
154                 vp->flag= VP_AREA+VP_SOFT;
155
156         return vp;
157 }
158
159 static int *get_indexarray(void)
160 {
161         return MEM_mallocN(sizeof(int)*MAXINDEX + 2, "vertexpaint");
162 }
163
164
165 /* in contradiction to cpack drawing colors, the MCOL colors (vpaint colors) are per byte! 
166    so not endian sensitive. Mcol = ABGR!!! so be cautious with cpack calls */
167
168 unsigned int rgba_to_mcol(float r, float g, float b, float a)
169 {
170         int ir, ig, ib, ia;
171         unsigned int col;
172         char *cp;
173         
174         ir= floor(255.0*r);
175         if(ir<0) ir= 0; else if(ir>255) ir= 255;
176         ig= floor(255.0*g);
177         if(ig<0) ig= 0; else if(ig>255) ig= 255;
178         ib= floor(255.0*b);
179         if(ib<0) ib= 0; else if(ib>255) ib= 255;
180         ia= floor(255.0*a);
181         if(ia<0) ia= 0; else if(ia>255) ia= 255;
182         
183         cp= (char *)&col;
184         cp[0]= ia;
185         cp[1]= ib;
186         cp[2]= ig;
187         cp[3]= ir;
188         
189         return col;
190         
191 }
192
193 static unsigned int vpaint_get_current_col(VPaint *vp)
194 {
195         Brush *brush = paint_brush(&vp->paint);
196         return rgba_to_mcol(brush->rgb[0], brush->rgb[1], brush->rgb[2], 1.0f);
197 }
198
199 void do_shared_vertexcol(Mesh *me)
200 {
201         /* if no mcol: do not do */
202         /* if tface: only the involved faces, otherwise all */
203         MFace *mface;
204         MTFace *tface;
205         int a;
206         short *scolmain, *scol;
207         char *mcol;
208         
209         if(me->mcol==0 || me->totvert==0 || me->totface==0) return;
210         
211         scolmain= MEM_callocN(4*sizeof(short)*me->totvert, "colmain");
212         
213         tface= me->mtface;
214         mface= me->mface;
215         mcol= (char *)me->mcol;
216         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
217                 if((tface && tface->mode & TF_SHAREDCOL) || (G.f & G_FACESELECT)==0) {
218                         scol= scolmain+4*mface->v1;
219                         scol[0]++; scol[1]+= mcol[1]; scol[2]+= mcol[2]; scol[3]+= mcol[3];
220                         scol= scolmain+4*mface->v2;
221                         scol[0]++; scol[1]+= mcol[5]; scol[2]+= mcol[6]; scol[3]+= mcol[7];
222                         scol= scolmain+4*mface->v3;
223                         scol[0]++; scol[1]+= mcol[9]; scol[2]+= mcol[10]; scol[3]+= mcol[11];
224                         if(mface->v4) {
225                                 scol= scolmain+4*mface->v4;
226                                 scol[0]++; scol[1]+= mcol[13]; scol[2]+= mcol[14]; scol[3]+= mcol[15];
227                         }
228                 }
229                 if(tface) tface++;
230         }
231         
232         a= me->totvert;
233         scol= scolmain;
234         while(a--) {
235                 if(scol[0]>1) {
236                         scol[1]/= scol[0];
237                         scol[2]/= scol[0];
238                         scol[3]/= scol[0];
239                 }
240                 scol+= 4;
241         }
242         
243         tface= me->mtface;
244         mface= me->mface;
245         mcol= (char *)me->mcol;
246         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
247                 if((tface && tface->mode & TF_SHAREDCOL) || (G.f & G_FACESELECT)==0) {
248                         scol= scolmain+4*mface->v1;
249                         mcol[1]= scol[1]; mcol[2]= scol[2]; mcol[3]= scol[3];
250                         scol= scolmain+4*mface->v2;
251                         mcol[5]= scol[1]; mcol[6]= scol[2]; mcol[7]= scol[3];
252                         scol= scolmain+4*mface->v3;
253                         mcol[9]= scol[1]; mcol[10]= scol[2]; mcol[11]= scol[3];
254                         if(mface->v4) {
255                                 scol= scolmain+4*mface->v4;
256                                 mcol[13]= scol[1]; mcol[14]= scol[2]; mcol[15]= scol[3];
257                         }
258                 }
259                 if(tface) tface++;
260         }
261
262         MEM_freeN(scolmain);
263 }
264
265 void make_vertexcol(Scene *scene, int shade)    /* single ob */
266 {
267         Object *ob;
268         Mesh *me;
269
270         if(scene->obedit) {
271                 error("Unable to perform function in Edit Mode");
272                 return;
273         }
274         
275         ob= OBACT;
276         if(!ob || ob->id.lib) return;
277         me= get_mesh(ob);
278         if(me==0) return;
279
280         /* copies from shadedisplist to mcol */
281         if(!me->mcol) {
282                 CustomData_add_layer(&me->fdata, CD_MCOL, CD_CALLOC, NULL, me->totface);
283                 mesh_update_customdata_pointers(me);
284         }
285
286         if(shade)
287                 shadeMeshMCol(scene, ob, me);
288         else
289                 memset(me->mcol, 255, 4*sizeof(MCol)*me->totface);
290         
291         DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
292         
293 }
294
295 static void copy_vpaint_prev(VPaint *vp, unsigned int *mcol, int tot)
296 {
297         if(vp->vpaint_prev) {
298                 MEM_freeN(vp->vpaint_prev);
299                 vp->vpaint_prev= NULL;
300         }
301         vp->tot= tot;   
302         
303         if(mcol==NULL || tot==0) return;
304         
305         vp->vpaint_prev= MEM_mallocN(4*sizeof(int)*tot, "vpaint_prev");
306         memcpy(vp->vpaint_prev, mcol, 4*sizeof(int)*tot);
307         
308 }
309
310 static void copy_wpaint_prev (VPaint *wp, MDeformVert *dverts, int dcount)
311 {
312         if (wp->wpaint_prev) {
313                 free_dverts(wp->wpaint_prev, wp->tot);
314                 wp->wpaint_prev= NULL;
315         }
316         
317         if(dverts && dcount) {
318                 
319                 wp->wpaint_prev = MEM_mallocN (sizeof(MDeformVert)*dcount, "wpaint prev");
320                 wp->tot = dcount;
321                 copy_dverts (wp->wpaint_prev, dverts, dcount);
322         }
323 }
324
325
326 void clear_vpaint(Scene *scene)
327 {
328         Mesh *me;
329         Object *ob;
330         unsigned int *to, paintcol;
331         int a;
332         
333         ob= OBACT;
334         me= get_mesh(ob);
335         if(!ob || ob->id.lib) return;
336
337         if(!(ob->mode & OB_MODE_VERTEX_PAINT)) return;
338
339         if(me==0 || me->mcol==0 || me->totface==0) return;
340
341         paintcol= vpaint_get_current_col(scene->toolsettings->vpaint);
342
343         to= (unsigned int *)me->mcol;
344         a= 4*me->totface;
345         while(a--) {
346                 *to= paintcol;
347                 to++; 
348         }
349         DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
350         
351 }
352
353 void clear_vpaint_selectedfaces(Scene *scene)
354 {
355         Mesh *me;
356         MFace *mf;
357         Object *ob;
358         unsigned int paintcol, *mcol;
359         int i;
360
361         ob= OBACT;
362         me= get_mesh(ob);
363         if(me==0 || me->totface==0) return;
364
365         if(!me->mcol)
366                 make_vertexcol(scene, 0);
367
368         paintcol= vpaint_get_current_col(scene->toolsettings->vpaint);
369
370         mf = me->mface;
371         mcol = (unsigned int*)me->mcol;
372         for (i = 0; i < me->totface; i++, mf++, mcol+=4) {
373                 if (mf->flag & ME_FACE_SEL) {
374                         mcol[0] = paintcol;
375                         mcol[1] = paintcol;
376                         mcol[2] = paintcol;
377                         mcol[3] = paintcol;
378                 }
379         }
380         
381         DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
382 }
383
384
385 /* fills in the selected faces with the current weight and vertex group */
386 void clear_wpaint_selectedfaces(Scene *scene)
387 {
388         ToolSettings *ts= scene->toolsettings;
389         VPaint *wp= ts->wpaint;
390         float paintweight= ts->vgroup_weight;
391         Mesh *me;
392         MFace *mface;
393         Object *ob;
394         MDeformWeight *dw, *uw;
395         int *indexar;
396         int index, vgroup;
397         unsigned int faceverts[5]={0,0,0,0,0};
398         unsigned char i;
399         int vgroup_mirror= -1;
400         
401         ob= OBACT;
402         me= ob->data;
403         if(me==0 || me->totface==0 || me->dvert==0 || !me->mface) return;
404         
405         indexar= get_indexarray();
406         for(index=0, mface=me->mface; index<me->totface; index++, mface++) {
407                 if((mface->flag & ME_FACE_SEL)==0)
408                         indexar[index]= 0;
409                 else
410                         indexar[index]= index+1;
411         }
412         
413         vgroup= ob->actdef-1;
414         
415         /* directly copied from weight_paint, should probaby split into a seperate function */
416         /* if mirror painting, find the other group */          
417         if(wp->flag & VP_MIRROR_X) {
418                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
419                 if(defgroup) {
420                         bDeformGroup *curdef;
421                         int actdef= 0;
422                         char name[32];
423
424                         BLI_strncpy(name, defgroup->name, 32);
425                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
426                         
427                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
428                                 if (!strcmp(curdef->name, name))
429                                         break;
430                         if(curdef==NULL) {
431                                 int olddef= ob->actdef; /* tsk, add_defgroup sets the active defgroup */
432                                 curdef= add_defgroup_name (ob, name);
433                                 ob->actdef= olddef;
434                         }
435                         
436                         if(curdef && curdef!=defgroup)
437                                 vgroup_mirror= actdef;
438                 }
439         }
440         /* end copy from weight_paint*/
441         
442         copy_wpaint_prev(wp, me->dvert, me->totvert);
443         
444         for(index=0; index<me->totface; index++) {
445                 if(indexar[index] && indexar[index]<=me->totface) {
446                         mface= me->mface + (indexar[index]-1);
447                         /* just so we can loop through the verts */
448                         faceverts[0]= mface->v1;
449                         faceverts[1]= mface->v2;
450                         faceverts[2]= mface->v3;
451                         faceverts[3]= mface->v4;
452                         for (i=0; i<3 || faceverts[i]; i++) {
453                                 if(!((me->dvert+faceverts[i])->flag)) {
454                                         dw= verify_defweight(me->dvert+faceverts[i], vgroup);
455                                         if(dw) {
456                                                 uw= verify_defweight(wp->wpaint_prev+faceverts[i], vgroup);
457                                                 uw->weight= dw->weight; /* set the undo weight */
458                                                 dw->weight= paintweight;
459                                                 
460                                                 if(wp->flag & VP_MIRROR_X) {    /* x mirror painting */
461                                                         int j= mesh_get_x_mirror_vert(ob, faceverts[i]);
462                                                         if(j>=0) {
463                                                                 /* copy, not paint again */
464                                                                 if(vgroup_mirror != -1) {
465                                                                         dw= verify_defweight(me->dvert+j, vgroup_mirror);
466                                                                         uw= verify_defweight(wp->wpaint_prev+j, vgroup_mirror);
467                                                                 } else {
468                                                                         dw= verify_defweight(me->dvert+j, vgroup);
469                                                                         uw= verify_defweight(wp->wpaint_prev+j, vgroup);
470                                                                 }
471                                                                 uw->weight= dw->weight; /* set the undo weight */
472                                                                 dw->weight= paintweight;
473                                                         }
474                                                 }
475                                         }
476                                         (me->dvert+faceverts[i])->flag= 1;
477                                 }
478                         }
479                 }
480         }
481         
482         index=0;
483         while (index<me->totvert) {
484                 (me->dvert+index)->flag= 0;
485                 index++;
486         }
487         
488         MEM_freeN(indexar);
489         copy_wpaint_prev(wp, NULL, 0);
490
491         DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
492 }
493
494
495 void vpaint_dogamma(Scene *scene)
496 {
497         VPaint *vp= scene->toolsettings->vpaint;
498         Mesh *me;
499         Object *ob;
500         float igam, fac;
501         int a, temp;
502         unsigned char *cp, gamtab[256];
503
504         ob= OBACT;
505         me= get_mesh(ob);
506
507         if(!(ob->mode & OB_MODE_VERTEX_PAINT)) return;
508         if(me==0 || me->mcol==0 || me->totface==0) return;
509
510         igam= 1.0/vp->gamma;
511         for(a=0; a<256; a++) {
512                 
513                 fac= ((float)a)/255.0;
514                 fac= vp->mul*pow( fac, igam);
515                 
516                 temp= 255.9*fac;
517                 
518                 if(temp<=0) gamtab[a]= 0;
519                 else if(temp>=255) gamtab[a]= 255;
520                 else gamtab[a]= temp;
521         }
522
523         a= 4*me->totface;
524         cp= (unsigned char *)me->mcol;
525         while(a--) {
526                 
527                 cp[1]= gamtab[ cp[1] ];
528                 cp[2]= gamtab[ cp[2] ];
529                 cp[3]= gamtab[ cp[3] ];
530                 
531                 cp+= 4;
532         }
533 }
534
535 static unsigned int mcol_blend(unsigned int col1, unsigned int col2, int fac)
536 {
537         char *cp1, *cp2, *cp;
538         int mfac;
539         unsigned int col=0;
540         
541         if(fac==0) return col1;
542         if(fac>=255) return col2;
543
544         mfac= 255-fac;
545         
546         cp1= (char *)&col1;
547         cp2= (char *)&col2;
548         cp=  (char *)&col;
549         
550         cp[0]= 255;
551         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
552         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
553         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
554         
555         return col;
556 }
557
558 static unsigned int mcol_add(unsigned int col1, unsigned int col2, int fac)
559 {
560         char *cp1, *cp2, *cp;
561         int temp;
562         unsigned int col=0;
563         
564         if(fac==0) return col1;
565         
566         cp1= (char *)&col1;
567         cp2= (char *)&col2;
568         cp=  (char *)&col;
569         
570         cp[0]= 255;
571         temp= cp1[1] + ((fac*cp2[1])/255);
572         if(temp>254) cp[1]= 255; else cp[1]= temp;
573         temp= cp1[2] + ((fac*cp2[2])/255);
574         if(temp>254) cp[2]= 255; else cp[2]= temp;
575         temp= cp1[3] + ((fac*cp2[3])/255);
576         if(temp>254) cp[3]= 255; else cp[3]= temp;
577         
578         return col;
579 }
580
581 static unsigned int mcol_sub(unsigned int col1, unsigned int col2, int fac)
582 {
583         char *cp1, *cp2, *cp;
584         int temp;
585         unsigned int col=0;
586         
587         if(fac==0) return col1;
588         
589         cp1= (char *)&col1;
590         cp2= (char *)&col2;
591         cp=  (char *)&col;
592         
593         cp[0]= 255;
594         temp= cp1[1] - ((fac*cp2[1])/255);
595         if(temp<0) cp[1]= 0; else cp[1]= temp;
596         temp= cp1[2] - ((fac*cp2[2])/255);
597         if(temp<0) cp[2]= 0; else cp[2]= temp;
598         temp= cp1[3] - ((fac*cp2[3])/255);
599         if(temp<0) cp[3]= 0; else cp[3]= temp;
600         
601         return col;
602 }
603
604 static unsigned int mcol_mul(unsigned int col1, unsigned int col2, int fac)
605 {
606         char *cp1, *cp2, *cp;
607         int mfac;
608         unsigned int col=0;
609         
610         if(fac==0) return col1;
611
612         mfac= 255-fac;
613         
614         cp1= (char *)&col1;
615         cp2= (char *)&col2;
616         cp=  (char *)&col;
617         
618         /* first mul, then blend the fac */
619         cp[0]= 255;
620         cp[1]= (mfac*cp1[1] + fac*((cp2[1]*cp1[1])/255)  )/255;
621         cp[2]= (mfac*cp1[2] + fac*((cp2[2]*cp1[2])/255)  )/255;
622         cp[3]= (mfac*cp1[3] + fac*((cp2[3]*cp1[3])/255)  )/255;
623
624         
625         return col;
626 }
627
628 static unsigned int mcol_lighten(unsigned int col1, unsigned int col2, int fac)
629 {
630         char *cp1, *cp2, *cp;
631         int mfac;
632         unsigned int col=0;
633         
634         if(fac==0) return col1;
635         if(fac>=255) return col2;
636
637         mfac= 255-fac;
638         
639         cp1= (char *)&col1;
640         cp2= (char *)&col2;
641         cp=  (char *)&col;
642         
643         /* See if are lighter, if so mix, else dont do anything.
644         if the paint col is darker then the original, then ignore */
645         if (cp1[1]+cp1[2]+cp1[3] > cp2[1]+cp2[2]+cp2[3])
646                 return col1;
647         
648         cp[0]= 255;
649         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
650         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
651         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
652         
653         return col;
654 }
655
656 static unsigned int mcol_darken(unsigned int col1, unsigned int col2, int fac)
657 {
658         char *cp1, *cp2, *cp;
659         int mfac;
660         unsigned int col=0;
661         
662         if(fac==0) return col1;
663         if(fac>=255) return col2;
664
665         mfac= 255-fac;
666         
667         cp1= (char *)&col1;
668         cp2= (char *)&col2;
669         cp=  (char *)&col;
670         
671         /* See if were darker, if so mix, else dont do anything.
672         if the paint col is brighter then the original, then ignore */
673         if (cp1[1]+cp1[2]+cp1[3] < cp2[1]+cp2[2]+cp2[3])
674                 return col1;
675         
676         cp[0]= 255;
677         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
678         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
679         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
680         return col;
681 }
682
683 static void vpaint_blend(VPaint *vp, unsigned int *col, unsigned int *colorig, unsigned int paintcol, int alpha)
684 {
685         Brush *brush = paint_brush(&vp->paint);
686
687         if(vp->mode==VP_MIX || vp->mode==VP_BLUR) *col= mcol_blend( *col, paintcol, alpha);
688         else if(vp->mode==VP_ADD) *col= mcol_add( *col, paintcol, alpha);
689         else if(vp->mode==VP_SUB) *col= mcol_sub( *col, paintcol, alpha);
690         else if(vp->mode==VP_MUL) *col= mcol_mul( *col, paintcol, alpha);
691         else if(vp->mode==VP_LIGHTEN) *col= mcol_lighten( *col, paintcol, alpha);
692         else if(vp->mode==VP_DARKEN) *col= mcol_darken( *col, paintcol, alpha);
693         
694         /* if no spray, clip color adding with colorig & orig alpha */
695         if((vp->flag & VP_SPRAY)==0) {
696                 unsigned int testcol=0, a;
697                 char *cp, *ct, *co;
698                 
699                 alpha= (int)(255.0*brush->alpha);
700                 
701                 if(vp->mode==VP_MIX || vp->mode==VP_BLUR) testcol= mcol_blend( *colorig, paintcol, alpha);
702                 else if(vp->mode==VP_ADD) testcol= mcol_add( *colorig, paintcol, alpha);
703                 else if(vp->mode==VP_SUB) testcol= mcol_sub( *colorig, paintcol, alpha);
704                 else if(vp->mode==VP_MUL) testcol= mcol_mul( *colorig, paintcol, alpha);
705                 else if(vp->mode==VP_LIGHTEN)  testcol= mcol_lighten( *colorig, paintcol, alpha);
706                 else if(vp->mode==VP_DARKEN)   testcol= mcol_darken( *colorig, paintcol, alpha);
707                 
708                 cp= (char *)col;
709                 ct= (char *)&testcol;
710                 co= (char *)colorig;
711                 
712                 for(a=0; a<4; a++) {
713                         if( ct[a]<co[a] ) {
714                                 if( cp[a]<ct[a] ) cp[a]= ct[a];
715                                 else if( cp[a]>co[a] ) cp[a]= co[a];
716                         }
717                         else {
718                                 if( cp[a]<co[a] ) cp[a]= co[a];
719                                 else if( cp[a]>ct[a] ) cp[a]= ct[a];
720                         }
721                 }
722         }
723 }
724
725
726 static int sample_backbuf_area(ViewContext *vc, int *indexar, int totface, int x, int y, float size)
727 {
728         struct ImBuf *ibuf;
729         int a, tot=0, index;
730         
731         if(totface+4>=MAXINDEX) return 0;
732         
733         if(size>64.0) size= 64.0;
734         
735         ibuf= view3d_read_backbuf(vc, x-size, y-size, x+size, y+size);
736         if(ibuf) {
737                 unsigned int *rt= ibuf->rect;
738
739                 memset(indexar, 0, sizeof(int)*totface+4);      /* plus 2! first element is total, +2 was giving valgrind errors, +4 seems ok */
740                 
741                 size= ibuf->x*ibuf->y;
742                 while(size--) {
743                                 
744                         if(*rt) {
745                                 index= WM_framebuffer_to_index(*rt);
746                                 if(index>0 && index<=totface)
747                                         indexar[index] = 1;
748                         }
749                 
750                         rt++;
751                 }
752                 
753                 for(a=1; a<=totface; a++) {
754                         if(indexar[a]) indexar[tot++]= a;
755                 }
756
757                 IMB_freeImBuf(ibuf);
758         }
759         
760         return tot;
761 }
762
763 static int calc_vp_alpha_dl(VPaint *vp, ViewContext *vc, float vpimat[][3], float *vert_nor, float *mval)
764 {
765         Brush *brush = paint_brush(&vp->paint);
766         float fac, dx, dy;
767         int alpha;
768         short vertco[2];
769         
770         if(vp->flag & VP_SOFT) {
771                 project_short_noclip(vc->ar, vert_nor, vertco);
772                 dx= mval[0]-vertco[0];
773                 dy= mval[1]-vertco[1];
774                 
775                 fac= sqrt(dx*dx + dy*dy);
776                 if(fac > brush->size) return 0;
777                 if(vp->flag & VP_HARD)
778                         alpha= 255;
779                 else
780                         alpha= 255.0*brush->alpha*(1.0-fac/brush->size);
781         }
782         else {
783                 alpha= 255.0*brush->alpha;
784         }
785
786         if(vp->flag & VP_NORMALS) {
787                 float *no= vert_nor+3;
788                 
789                         /* transpose ! */
790                 fac= vpimat[2][0]*no[0]+vpimat[2][1]*no[1]+vpimat[2][2]*no[2];
791                 if(fac>0.0) {
792                         dx= vpimat[0][0]*no[0]+vpimat[0][1]*no[1]+vpimat[0][2]*no[2];
793                         dy= vpimat[1][0]*no[0]+vpimat[1][1]*no[1]+vpimat[1][2]*no[2];
794                         
795                         alpha*= fac/sqrt(dx*dx + dy*dy + fac*fac);
796                 }
797                 else return 0;
798         }
799         
800         return alpha;
801 }
802
803 static void wpaint_blend(VPaint *wp, MDeformWeight *dw, MDeformWeight *uw, float alpha, float paintval)
804 {
805         Brush *brush = paint_brush(&wp->paint);
806         
807         if(dw==NULL || uw==NULL) return;
808         
809         if(wp->mode==VP_MIX || wp->mode==VP_BLUR)
810                 dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
811         else if(wp->mode==VP_ADD)
812                 dw->weight += paintval*alpha;
813         else if(wp->mode==VP_SUB) 
814                 dw->weight -= paintval*alpha;
815         else if(wp->mode==VP_MUL) 
816                 /* first mul, then blend the fac */
817                 dw->weight = ((1.0-alpha) + alpha*paintval)*dw->weight;
818         else if(wp->mode==VP_LIGHTEN) {
819                 if (dw->weight < paintval)
820                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
821         } else if(wp->mode==VP_DARKEN) {
822                 if (dw->weight > paintval)
823                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
824         }
825         CLAMP(dw->weight, 0.0f, 1.0f);
826         
827         /* if no spray, clip result with orig weight & orig alpha */
828         if((wp->flag & VP_SPRAY)==0) {
829                 float testw=0.0f;
830                 
831                 alpha= brush->alpha;
832                 if(wp->mode==VP_MIX || wp->mode==VP_BLUR)
833                         testw = paintval*alpha + uw->weight*(1.0-alpha);
834                 else if(wp->mode==VP_ADD)
835                         testw = uw->weight + paintval*alpha;
836                 else if(wp->mode==VP_SUB) 
837                         testw = uw->weight - paintval*alpha;
838                 else if(wp->mode==VP_MUL) 
839                         /* first mul, then blend the fac */
840                         testw = ((1.0-alpha) + alpha*paintval)*uw->weight;              
841                 else if(wp->mode==VP_LIGHTEN) {
842                         if (uw->weight < paintval)
843                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
844                         else
845                                 testw = uw->weight;
846                 } else if(wp->mode==VP_DARKEN) {
847                         if (uw->weight > paintval)
848                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
849                         else
850                                 testw = uw->weight;
851                 }
852                 CLAMP(testw, 0.0f, 1.0f);
853                 
854                 if( testw<uw->weight ) {
855                         if(dw->weight < testw) dw->weight= testw;
856                         else if(dw->weight > uw->weight) dw->weight= uw->weight;
857                 }
858                 else {
859                         if(dw->weight > testw) dw->weight= testw;
860                         else if(dw->weight < uw->weight) dw->weight= uw->weight;
861                 }
862         }
863         
864 }
865
866 /* ----------------------------------------------------- */
867
868 /* used for 3d view, on active object, assumes me->dvert exists */
869 /* if mode==1: */
870 /*     samples cursor location, and gives menu with vertex groups to activate */
871 /* else */
872 /*     sets wp->weight to the closest weight value to vertex */
873 /*     note: we cant sample frontbuf, weight colors are interpolated too unpredictable */
874 void sample_wpaint(Scene *scene, ARegion *ar, View3D *v3d, int mode)
875 {
876         ViewContext vc;
877         ToolSettings *ts= scene->toolsettings;
878         Object *ob= OBACT;
879         Mesh *me= get_mesh(ob);
880         int index;
881         short mval[2], sco[2];
882
883         if (!me) return;
884         
885 //      getmouseco_areawin(mval);
886         index= view3d_sample_backbuf(&vc, mval[0], mval[1]);
887         
888         if(index && index<=me->totface) {
889                 MFace *mface;
890                 
891                 mface= ((MFace *)me->mface) + index-1;
892                 
893                 if(mode==1) {   /* sampe which groups are in here */
894                         MDeformVert *dv;
895                         int a, totgroup;
896                         
897                         totgroup= BLI_countlist(&ob->defbase);
898                         if(totgroup) {
899                                 int totmenu=0;
900                                 int *groups=MEM_callocN(totgroup*sizeof(int), "groups");
901                                 
902                                 dv= me->dvert+mface->v1;
903                                 for(a=0; a<dv->totweight; a++) {
904                                         if (dv->dw[a].def_nr<totgroup)
905                                                 groups[dv->dw[a].def_nr]= 1;
906                                 }
907                                 dv= me->dvert+mface->v2;
908                                 for(a=0; a<dv->totweight; a++) {
909                                         if (dv->dw[a].def_nr<totgroup)
910                                                 groups[dv->dw[a].def_nr]= 1;
911                                 }
912                                 dv= me->dvert+mface->v3;
913                                 for(a=0; a<dv->totweight; a++) {
914                                         if (dv->dw[a].def_nr<totgroup)
915                                                 groups[dv->dw[a].def_nr]= 1;
916                                 }
917                                 if(mface->v4) {
918                                         dv= me->dvert+mface->v4;
919                                         for(a=0; a<dv->totweight; a++) {
920                                                 if (dv->dw[a].def_nr<totgroup)
921                                                         groups[dv->dw[a].def_nr]= 1;
922                                         }
923                                 }
924                                 for(a=0; a<totgroup; a++)
925                                         if(groups[a]) totmenu++;
926                                 
927                                 if(totmenu==0) {
928                                         //notice("No Vertex Group Selected");
929                                 }
930                                 else {
931                                         bDeformGroup *dg;
932                                         short val;
933                                         char item[40], *str= MEM_mallocN(40*totmenu+40, "menu");
934                                         
935                                         strcpy(str, "Vertex Groups %t");
936                                         for(a=0, dg=ob->defbase.first; dg && a<totgroup; a++, dg= dg->next) {
937                                                 if(groups[a]) {
938                                                         sprintf(item, "|%s %%x%d", dg->name, a);
939                                                         strcat(str, item);
940                                                 }
941                                         }
942                                         
943                                         val= 0; // XXX pupmenu(str);
944                                         if(val>=0) {
945                                                 ob->actdef= val+1;
946                                                 DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
947                                         }
948                                         MEM_freeN(str);
949                                 }
950                                 MEM_freeN(groups);
951                         }
952 //                      else notice("No Vertex Groups in Object");
953                 }
954                 else {
955                         DerivedMesh *dm;
956                         MDeformWeight *dw;
957                         float w1, w2, w3, w4, co[3], fac;
958                         
959                         dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
960                         if(dm->getVertCo==NULL) {
961                                 //notice("Not supported yet");
962                         }
963                         else {
964                                 /* calc 3 or 4 corner weights */
965                                 dm->getVertCo(dm, mface->v1, co);
966                                 project_short_noclip(ar, co, sco);
967                                 w1= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
968                                 
969                                 dm->getVertCo(dm, mface->v2, co);
970                                 project_short_noclip(ar, co, sco);
971                                 w2= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
972                                 
973                                 dm->getVertCo(dm, mface->v3, co);
974                                 project_short_noclip(ar, co, sco);
975                                 w3= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
976                                 
977                                 if(mface->v4) {
978                                         dm->getVertCo(dm, mface->v4, co);
979                                         project_short_noclip(ar, co, sco);
980                                         w4= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
981                                 }
982                                 else w4= 1.0e10;
983                                 
984                                 fac= MIN4(w1, w2, w3, w4);
985                                 if(w1==fac) {
986                                         dw= get_defweight(me->dvert+mface->v1, ob->actdef-1);
987                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
988                                 }
989                                 else if(w2==fac) {
990                                         dw= get_defweight(me->dvert+mface->v2, ob->actdef-1);
991                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
992                                 }
993                                 else if(w3==fac) {
994                                         dw= get_defweight(me->dvert+mface->v3, ob->actdef-1);
995                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
996                                 }
997                                 else if(w4==fac) {
998                                         if(mface->v4) {
999                                                 dw= get_defweight(me->dvert+mface->v4, ob->actdef-1);
1000                                                 if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
1001                                         }
1002                                 }
1003                         }
1004                         dm->release(dm);
1005                 }               
1006                 
1007         }
1008         
1009 }
1010
1011 static void do_weight_paint_vertex(VPaint *wp, Object *ob, int index, int alpha, float paintweight, int vgroup_mirror)
1012 {
1013         Mesh *me= ob->data;
1014         MDeformWeight *dw, *uw;
1015         int vgroup= ob->actdef-1;
1016         
1017         if(wp->flag & VP_ONLYVGROUP) {
1018                 dw= get_defweight(me->dvert+index, vgroup);
1019                 uw= get_defweight(wp->wpaint_prev+index, vgroup);
1020         }
1021         else {
1022                 dw= verify_defweight(me->dvert+index, vgroup);
1023                 uw= verify_defweight(wp->wpaint_prev+index, vgroup);
1024         }
1025         if(dw==NULL || uw==NULL)
1026                 return;
1027         
1028         wpaint_blend(wp, dw, uw, (float)alpha/255.0, paintweight);
1029         
1030         if(wp->flag & VP_MIRROR_X) {    /* x mirror painting */
1031                 int j= mesh_get_x_mirror_vert(ob, index);
1032                 if(j>=0) {
1033                         /* copy, not paint again */
1034                         if(vgroup_mirror != -1)
1035                                 uw= verify_defweight(me->dvert+j, vgroup_mirror);
1036                         else
1037                                 uw= verify_defweight(me->dvert+j, vgroup);
1038                                 
1039                         uw->weight= dw->weight;
1040                 }
1041         }
1042 }
1043
1044
1045 /* *************** set wpaint operator ****************** */
1046
1047 static int set_wpaint(bContext *C, wmOperator *op)              /* toggle */
1048 {               
1049         Object *ob= CTX_data_active_object(C);
1050         Scene *scene= CTX_data_scene(C);
1051         VPaint *wp= scene->toolsettings->wpaint;
1052         Mesh *me;
1053         
1054         me= get_mesh(ob);
1055         if(ob->id.lib || me==NULL) return OPERATOR_PASS_THROUGH;
1056         
1057         if(me && me->totface>=MAXINDEX) {
1058                 error("Maximum number of faces: %d", MAXINDEX-1);
1059                 ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1060                 return OPERATOR_CANCELLED;
1061         }
1062         
1063         if(ob->mode & OB_MODE_WEIGHT_PAINT) ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1064         else ob->mode |= OB_MODE_WEIGHT_PAINT;
1065         
1066         
1067         /* Weightpaint works by overriding colors in mesh,
1068                 * so need to make sure we recalc on enter and
1069                 * exit (exit needs doing regardless because we
1070                                 * should redeform).
1071                 */
1072         DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
1073         
1074         if(ob->mode & OB_MODE_WEIGHT_PAINT) {
1075                 Object *par;
1076                 
1077                 if(wp==NULL)
1078                         wp= scene->toolsettings->wpaint= new_vpaint(1);
1079
1080                 paint_init(&wp->paint, PAINT_CURSOR_WEIGHT_PAINT);
1081                 paint_cursor_start(C, wp_poll);
1082                 
1083                 mesh_octree_table(ob, NULL, NULL, 's');
1084                 
1085                 /* verify if active weight group is also active bone */
1086                 par= modifiers_isDeformedByArmature(ob);
1087                 if(par && (par->mode & OB_MODE_POSE)) {
1088                         bPoseChannel *pchan;
1089                         for(pchan= par->pose->chanbase.first; pchan; pchan= pchan->next)
1090                                 if(pchan->bone->flag & BONE_ACTIVE)
1091                                         break;
1092                                 if(pchan)
1093                                         vertexgroup_select_by_name(ob, pchan->name);
1094                 }
1095         }
1096         else {
1097                 mesh_octree_table(ob, NULL, NULL, 'e');
1098         }
1099         
1100         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1101         
1102         return OPERATOR_FINISHED;
1103 }
1104
1105 /* for switching to/from mode */
1106 static int paint_poll_test(bContext *C)
1107 {
1108         if(ED_operator_view3d_active(C)==0)
1109                 return 0;
1110         if(CTX_data_edit_object(C))
1111                 return 0;
1112         if(CTX_data_active_object(C)==NULL)
1113                 return 0;
1114         return 1;
1115 }
1116
1117 void PAINT_OT_weight_paint_toggle(wmOperatorType *ot)
1118 {
1119         
1120         /* identifiers */
1121         ot->name= "Weight Paint Mode";
1122         ot->idname= "PAINT_OT_weight_paint_toggle";
1123         
1124         /* api callbacks */
1125         ot->exec= set_wpaint;
1126         ot->poll= paint_poll_test;
1127         
1128         /* flags */
1129         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1130         
1131 }
1132
1133 /* ************ paint radial controls *************/
1134
1135 static int vpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1136 {
1137         Paint *p = paint_get_active(CTX_data_scene(C));
1138         Brush *brush = paint_brush(p);
1139         
1140         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1141         p->paint_cursor = NULL;
1142         brush_radial_control_invoke(op, brush, 1);
1143         return WM_radial_control_invoke(C, op, event);
1144 }
1145
1146 static int vpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1147 {
1148         int ret = WM_radial_control_modal(C, op, event);
1149         if(ret != OPERATOR_RUNNING_MODAL)
1150                 paint_cursor_start(C, vp_poll);
1151         return ret;
1152 }
1153
1154 static int vpaint_radial_control_exec(bContext *C, wmOperator *op)
1155 {
1156         Brush *brush = paint_brush(&CTX_data_scene(C)->toolsettings->vpaint->paint);
1157         return brush_radial_control_exec(op, brush, 1);
1158 }
1159
1160 static int wpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1161 {
1162         Paint *p = paint_get_active(CTX_data_scene(C));
1163         Brush *brush = paint_brush(p);
1164         
1165         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1166         p->paint_cursor = NULL;
1167         brush_radial_control_invoke(op, brush, 1);
1168         return WM_radial_control_invoke(C, op, event);
1169 }
1170
1171 static int wpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1172 {
1173         int ret = WM_radial_control_modal(C, op, event);
1174         if(ret != OPERATOR_RUNNING_MODAL)
1175                 paint_cursor_start(C, wp_poll);
1176         return ret;
1177 }
1178
1179 static int wpaint_radial_control_exec(bContext *C, wmOperator *op)
1180 {
1181         Brush *brush = paint_brush(&CTX_data_scene(C)->toolsettings->wpaint->paint);
1182         return brush_radial_control_exec(op, brush, 1);
1183 }
1184
1185 void PAINT_OT_weight_paint_radial_control(wmOperatorType *ot)
1186 {
1187         WM_OT_radial_control_partial(ot);
1188
1189         ot->name= "Weight Paint Radial Control";
1190         ot->idname= "PAINT_OT_weight_paint_radial_control";
1191
1192         ot->invoke= wpaint_radial_control_invoke;
1193         ot->modal= wpaint_radial_control_modal;
1194         ot->exec= wpaint_radial_control_exec;
1195         ot->poll= wp_poll;
1196         
1197         /* flags */
1198         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1199 }
1200
1201 void PAINT_OT_vertex_paint_radial_control(wmOperatorType *ot)
1202 {
1203         WM_OT_radial_control_partial(ot);
1204
1205         ot->name= "Vertex Paint Radial Control";
1206         ot->idname= "PAINT_OT_vertex_paint_radial_control";
1207
1208         ot->invoke= vpaint_radial_control_invoke;
1209         ot->modal= vpaint_radial_control_modal;
1210         ot->exec= vpaint_radial_control_exec;
1211         ot->poll= vp_poll;
1212         
1213         /* flags */
1214         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1215 }
1216
1217 /* ************ weight paint operator ********** */
1218
1219 struct WPaintData {
1220         ViewContext vc;
1221         int *indexar;
1222         int vgroup_mirror;
1223         float *vertexcosnos;
1224         float wpimat[3][3];
1225 };
1226
1227 static int wpaint_stroke_test_start(bContext *C, wmOperator *op, wmEvent *event)
1228 {
1229         Scene *scene= CTX_data_scene(C);
1230         struct PaintStroke *stroke = op->customdata;
1231         ToolSettings *ts= CTX_data_tool_settings(C);
1232         VPaint *wp= ts->wpaint;
1233         Object *ob= CTX_data_active_object(C);
1234         struct WPaintData *wpd;
1235         Mesh *me;
1236         float mat[4][4], imat[4][4];
1237         
1238         if(scene->obedit) return OPERATOR_CANCELLED;
1239         
1240         me= get_mesh(ob);
1241         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1242         
1243         /* if nothing was added yet, we make dverts and a vertex deform group */
1244         if (!me->dvert)
1245                 create_dverts(&me->id);
1246         
1247         /* make mode data storage */
1248         wpd= MEM_callocN(sizeof(struct WPaintData), "WPaintData");
1249         paint_stroke_set_mode_data(stroke, wpd);
1250         view3d_set_viewcontext(C, &wpd->vc);
1251         wpd->vgroup_mirror= -1;
1252         
1253         //      if(qual & LR_CTRLKEY) {
1254         //              sample_wpaint(scene, ar, v3d, 0);
1255         //              return;
1256         //      }
1257         //      if(qual & LR_SHIFTKEY) {
1258         //              sample_wpaint(scene, ar, v3d, 1);
1259         //              return;
1260         //      }
1261         
1262         /* ALLOCATIONS! no return after this line */
1263         /* painting on subsurfs should give correct points too, this returns me->totvert amount */
1264         wpd->vertexcosnos= mesh_get_mapped_verts_nors(scene, ob);
1265         wpd->indexar= get_indexarray();
1266         copy_wpaint_prev(wp, me->dvert, me->totvert);
1267         
1268         /* this happens on a Bone select, when no vgroup existed yet */
1269         if(ob->actdef<=0) {
1270                 Object *modob;
1271                 if((modob = modifiers_isDeformedByArmature(ob))) {
1272                         bPoseChannel *pchan;
1273                         for(pchan= modob->pose->chanbase.first; pchan; pchan= pchan->next)
1274                                 if(pchan->bone->flag & SELECT)
1275                                         break;
1276                         if(pchan) {
1277                                 bDeformGroup *dg= get_named_vertexgroup(ob, pchan->name);
1278                                 if(dg==NULL)
1279                                         dg= add_defgroup_name(ob, pchan->name); /* sets actdef */
1280                                 else
1281                                         ob->actdef= get_defgroup_num(ob, dg);
1282                         }
1283                 }
1284         }
1285         if(ob->defbase.first==NULL) {
1286                 add_defgroup(ob);
1287         }       
1288         
1289         //      if(ob->lay & v3d->lay); else error("Active object is not in this layer");
1290         
1291         /* imat for normals */
1292         Mat4MulMat4(mat, ob->obmat, wpd->vc.rv3d->viewmat);
1293         Mat4Invert(imat, mat);
1294         Mat3CpyMat4(wpd->wpimat, imat);
1295         
1296         /* if mirror painting, find the other group */
1297         if(wp->flag & VP_MIRROR_X) {
1298                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
1299                 if(defgroup) {
1300                         bDeformGroup *curdef;
1301                         int actdef= 0;
1302                         char name[32];
1303                         
1304                         BLI_strncpy(name, defgroup->name, 32);
1305                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
1306                         
1307                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
1308                                 if (!strcmp(curdef->name, name))
1309                                         break;
1310                         if(curdef==NULL) {
1311                                 int olddef= ob->actdef; /* tsk, add_defgroup sets the active defgroup */
1312                                 curdef= add_defgroup_name (ob, name);
1313                                 ob->actdef= olddef;
1314                         }
1315                         
1316                         if(curdef && curdef!=defgroup)
1317                                 wpd->vgroup_mirror= actdef;
1318                 }
1319         }
1320         
1321         return 1;
1322 }
1323
1324 static void wpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1325 {
1326         ToolSettings *ts= CTX_data_tool_settings(C);
1327         VPaint *wp= ts->wpaint;
1328         Brush *brush = paint_brush(&wp->paint);
1329         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1330         ViewContext *vc= &wpd->vc;
1331         Object *ob= vc->obact;
1332         Mesh *me= ob->data;
1333         float mat[4][4];
1334         float paintweight= ts->vgroup_weight;
1335         int *indexar= wpd->indexar;
1336         int totindex, index, alpha, totw;
1337         float mval[2];
1338
1339         view3d_operator_needs_opengl(C);
1340                         
1341         /* load projection matrix */
1342         wmMultMatrix(ob->obmat);
1343         wmGetSingleMatrix(mat);
1344         wmLoadMatrix(wpd->vc.rv3d->viewmat);
1345
1346         RNA_float_get_array(itemptr, "mouse", mval);
1347         mval[0]-= vc->ar->winrct.xmin;
1348         mval[1]-= vc->ar->winrct.ymin;
1349                         
1350         MTC_Mat4SwapMat4(wpd->vc.rv3d->persmat, mat);
1351                         
1352         /* which faces are involved */
1353         if(wp->flag & VP_AREA) {
1354                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush->size);
1355         }
1356         else {
1357                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1358                 if(indexar[0]) totindex= 1;
1359                 else totindex= 0;
1360         }
1361                         
1362         if(wp->flag & VP_COLINDEX) {
1363                 for(index=0; index<totindex; index++) {
1364                         if(indexar[index] && indexar[index]<=me->totface) {
1365                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1366                                                 
1367                                 if(mface->mat_nr!=ob->actcol-1) {
1368                                         indexar[index]= 0;
1369                                 }
1370                         }                                       
1371                 }
1372         }
1373                         
1374         if((G.f & G_FACESELECT) && me->mface) {
1375                 for(index=0; index<totindex; index++) {
1376                         if(indexar[index] && indexar[index]<=me->totface) {
1377                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1378                                                 
1379                                 if((mface->flag & ME_FACE_SEL)==0) {
1380                                         indexar[index]= 0;
1381                                 }
1382                         }                                       
1383                 }
1384         }
1385                         
1386         /* make sure each vertex gets treated only once */
1387         /* and calculate filter weight */
1388         totw= 0;
1389         if(wp->mode==VP_BLUR) 
1390                 paintweight= 0.0f;
1391         else
1392                 paintweight= ts->vgroup_weight;
1393                         
1394         for(index=0; index<totindex; index++) {
1395                 if(indexar[index] && indexar[index]<=me->totface) {
1396                         MFace *mface= me->mface + (indexar[index]-1);
1397                                         
1398                         (me->dvert+mface->v1)->flag= 1;
1399                         (me->dvert+mface->v2)->flag= 1;
1400                         (me->dvert+mface->v3)->flag= 1;
1401                         if(mface->v4) (me->dvert+mface->v4)->flag= 1;
1402                                         
1403                         if(wp->mode==VP_BLUR) {
1404                                 MDeformWeight *dw, *(*dw_func)(MDeformVert *, int) = verify_defweight;
1405                                                 
1406                                 if(wp->flag & VP_ONLYVGROUP)
1407                                         dw_func= get_defweight;
1408                                                 
1409                                 dw= dw_func(me->dvert+mface->v1, ob->actdef-1);
1410                                 if(dw) {paintweight+= dw->weight; totw++;}
1411                                 dw= dw_func(me->dvert+mface->v2, ob->actdef-1);
1412                                 if(dw) {paintweight+= dw->weight; totw++;}
1413                                 dw= dw_func(me->dvert+mface->v3, ob->actdef-1);
1414                                 if(dw) {paintweight+= dw->weight; totw++;}
1415                                 if(mface->v4) {
1416                                         dw= dw_func(me->dvert+mface->v4, ob->actdef-1);
1417                                         if(dw) {paintweight+= dw->weight; totw++;}
1418                                 }
1419                         }
1420                 }
1421         }
1422                         
1423         if(wp->mode==VP_BLUR) 
1424                 paintweight/= (float)totw;
1425                         
1426         for(index=0; index<totindex; index++) {
1427                                 
1428                 if(indexar[index] && indexar[index]<=me->totface) {
1429                         MFace *mface= me->mface + (indexar[index]-1);
1430                                         
1431                         if((me->dvert+mface->v1)->flag) {
1432                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v1, mval);
1433                                 if(alpha) {
1434                                         do_weight_paint_vertex(wp, ob, mface->v1, alpha, paintweight, wpd->vgroup_mirror);
1435                                 }
1436                                 (me->dvert+mface->v1)->flag= 0;
1437                         }
1438                                         
1439                         if((me->dvert+mface->v2)->flag) {
1440                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v2, mval);
1441                                 if(alpha) {
1442                                         do_weight_paint_vertex(wp, ob, mface->v2, alpha, paintweight, wpd->vgroup_mirror);
1443                                 }
1444                                 (me->dvert+mface->v2)->flag= 0;
1445                         }
1446                                         
1447                         if((me->dvert+mface->v3)->flag) {
1448                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v3, mval);
1449                                 if(alpha) {
1450                                         do_weight_paint_vertex(wp, ob, mface->v3, alpha, paintweight, wpd->vgroup_mirror);
1451                                 }
1452                                 (me->dvert+mface->v3)->flag= 0;
1453                         }
1454                                         
1455                         if((me->dvert+mface->v4)->flag) {
1456                                 if(mface->v4) {
1457                                         alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v4, mval);
1458                                         if(alpha) {
1459                                                 do_weight_paint_vertex(wp, ob, mface->v4, alpha, paintweight, wpd->vgroup_mirror);
1460                                         }
1461                                         (me->dvert+mface->v4)->flag= 0;
1462                                 }
1463                         }
1464                 }
1465         }
1466                         
1467         MTC_Mat4SwapMat4(vc->rv3d->persmat, mat);
1468                         
1469         DAG_object_flush_update(vc->scene, ob, OB_RECALC_DATA);
1470         ED_region_tag_redraw(vc->ar);
1471 }
1472
1473 static void wpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
1474 {
1475         ToolSettings *ts= CTX_data_tool_settings(C);
1476         Object *ob= CTX_data_active_object(C);
1477         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1478         
1479         if(wpd->vertexcosnos)
1480                 MEM_freeN(wpd->vertexcosnos);
1481         MEM_freeN(wpd->indexar);
1482         
1483         /* frees prev buffer */
1484         copy_wpaint_prev(ts->wpaint, NULL, 0);
1485         
1486         /* and particles too */
1487         if(ob->particlesystem.first) {
1488                 ParticleSystem *psys;
1489                 int i;
1490                 
1491                 for(psys= ob->particlesystem.first; psys; psys= psys->next) {
1492                         for(i=0; i<PSYS_TOT_VG; i++) {
1493                                 if(psys->vgroup[i]==ob->actdef) {
1494                                         psys->recalc |= PSYS_RECALC_RESET;
1495                                         break;
1496                                 }
1497                         }
1498                 }
1499         }
1500         
1501         DAG_object_flush_update(CTX_data_scene(C), ob, OB_RECALC_DATA);
1502         
1503         MEM_freeN(wpd);
1504 }
1505
1506
1507 static int wpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1508 {
1509         
1510         op->customdata = paint_stroke_new(C, wpaint_stroke_test_start,
1511                                           wpaint_stroke_update_step,
1512                                           wpaint_stroke_done);
1513         
1514         /* add modal handler */
1515         WM_event_add_modal_handler(C, &CTX_wm_window(C)->handlers, op);
1516         
1517         return OPERATOR_RUNNING_MODAL;
1518 }
1519
1520 void PAINT_OT_weight_paint(wmOperatorType *ot)
1521 {
1522         
1523         /* identifiers */
1524         ot->name= "Weight Paint";
1525         ot->idname= "PAINT_OT_weight_paint";
1526         
1527         /* api callbacks */
1528         ot->invoke= wpaint_invoke;
1529         ot->modal= paint_stroke_modal;
1530         /* ot->exec= vpaint_exec; <-- needs stroke property */
1531         ot->poll= wp_poll;
1532         
1533         /* flags */
1534         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1535
1536         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1537 }
1538
1539 /* ************ set / clear vertex paint mode ********** */
1540
1541
1542 static int set_vpaint(bContext *C, wmOperator *op)              /* toggle */
1543 {       
1544         Object *ob= CTX_data_active_object(C);
1545         Scene *scene= CTX_data_scene(C);
1546         VPaint *vp= scene->toolsettings->vpaint;
1547         Mesh *me;
1548         
1549         me= get_mesh(ob);
1550         
1551         if(me==NULL || object_data_is_libdata(ob)) {
1552                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1553                 return OPERATOR_PASS_THROUGH;
1554         }
1555         
1556         if(me && me->totface>=MAXINDEX) {
1557                 error("Maximum number of faces: %d", MAXINDEX-1);
1558                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1559                 return OPERATOR_FINISHED;
1560         }
1561         
1562         if(me && me->mcol==NULL) make_vertexcol(scene, 0);
1563         
1564         /* toggle: end vpaint */
1565         if(ob->mode & OB_MODE_VERTEX_PAINT) {
1566                 
1567                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1568         }
1569         else {
1570                 ob->mode |= OB_MODE_VERTEX_PAINT;
1571                 /* Turn off weight painting */
1572                 if (ob->mode & OB_MODE_WEIGHT_PAINT)
1573                         set_wpaint(C, op);
1574                 
1575                 if(vp==NULL)
1576                         vp= scene->toolsettings->vpaint= new_vpaint(0);
1577                 
1578                 paint_cursor_start(C, vp_poll);
1579
1580                 paint_init(&vp->paint, PAINT_CURSOR_VERTEX_PAINT);
1581         }
1582         
1583         if (me)
1584                 /* update modifier stack for mapping requirements */
1585                 DAG_object_flush_update(scene, ob, OB_RECALC_DATA);
1586         
1587         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1588         
1589         return OPERATOR_FINISHED;
1590 }
1591
1592 void PAINT_OT_vertex_paint_toggle(wmOperatorType *ot)
1593 {
1594         
1595         /* identifiers */
1596         ot->name= "Vertex Paint Mode";
1597         ot->idname= "PAINT_OT_vertex_paint_toggle";
1598         
1599         /* api callbacks */
1600         ot->exec= set_vpaint;
1601         ot->poll= paint_poll_test;
1602         
1603         /* flags */
1604         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1605         
1606 }
1607
1608
1609
1610 /* ********************** vertex paint operator ******************* */
1611
1612 /* Implementation notes:
1613
1614 Operator->invoke()
1615   - validate context (add mcol)
1616   - create customdata storage
1617   - call paint once (mouse click)
1618   - add modal handler 
1619
1620 Operator->modal()
1621   - for every mousemove, apply vertex paint
1622   - exit on mouse release, free customdata
1623     (return OPERATOR_FINISHED also removes handler and operator)
1624
1625 For future:
1626   - implement a stroke event (or mousemove with past positons)
1627   - revise whether op->customdata should be added in object, in set_vpaint
1628
1629 */
1630
1631 struct VPaintData {
1632         ViewContext vc;
1633         unsigned int paintcol;
1634         int *indexar;
1635         float *vertexcosnos;
1636         float vpimat[3][3];
1637 };
1638
1639 static int vpaint_stroke_test_start(bContext *C, struct wmOperator *op, wmEvent *event)
1640 {
1641         ToolSettings *ts= CTX_data_tool_settings(C);
1642         struct PaintStroke *stroke = op->customdata;
1643         VPaint *vp= ts->vpaint;
1644         struct VPaintData *vpd;
1645         Object *ob= CTX_data_active_object(C);
1646         Mesh *me;
1647         float mat[4][4], imat[4][4];
1648
1649         /* context checks could be a poll() */
1650         me= get_mesh(ob);
1651         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1652         
1653         if(me->mcol==NULL) make_vertexcol(CTX_data_scene(C), 0);
1654         if(me->mcol==NULL) return OPERATOR_CANCELLED;
1655         
1656         /* make mode data storage */
1657         vpd= MEM_callocN(sizeof(struct VPaintData), "VPaintData");
1658         paint_stroke_set_mode_data(stroke, vpd);
1659         view3d_set_viewcontext(C, &vpd->vc);
1660         
1661         vpd->vertexcosnos= mesh_get_mapped_verts_nors(vpd->vc.scene, ob);
1662         vpd->indexar= get_indexarray();
1663         vpd->paintcol= vpaint_get_current_col(vp);
1664         
1665         /* for filtering */
1666         copy_vpaint_prev(vp, (unsigned int *)me->mcol, me->totface);
1667         
1668         /* some old cruft to sort out later */
1669         Mat4MulMat4(mat, ob->obmat, vpd->vc.rv3d->viewmat);
1670         Mat4Invert(imat, mat);
1671         Mat3CpyMat4(vpd->vpimat, imat);
1672
1673         return 1;
1674 }
1675
1676 static void vpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1677 {
1678         ToolSettings *ts= CTX_data_tool_settings(C);
1679         struct VPaintData *vpd = paint_stroke_mode_data(stroke);
1680         VPaint *vp= ts->vpaint;
1681         Brush *brush = paint_brush(&vp->paint);
1682         ViewContext *vc= &vpd->vc;
1683         Object *ob= vc->obact;
1684         Mesh *me= ob->data;
1685         float mat[4][4];
1686         int *indexar= vpd->indexar;
1687         int totindex, index;
1688         float mval[2];
1689
1690         RNA_float_get_array(itemptr, "mouse", mval);
1691                         
1692         view3d_operator_needs_opengl(C);
1693                         
1694         /* load projection matrix */
1695         wmMultMatrix(ob->obmat);
1696         wmGetSingleMatrix(mat);
1697         wmLoadMatrix(vc->rv3d->viewmat);
1698
1699         mval[0]-= vc->ar->winrct.xmin;
1700         mval[1]-= vc->ar->winrct.ymin;
1701
1702                         
1703         /* which faces are involved */
1704         if(vp->flag & VP_AREA) {
1705                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush->size);
1706         }
1707         else {
1708                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1709                 if(indexar[0]) totindex= 1;
1710                 else totindex= 0;
1711         }
1712                         
1713         MTC_Mat4SwapMat4(vc->rv3d->persmat, mat);
1714                         
1715         if(vp->flag & VP_COLINDEX) {
1716                 for(index=0; index<totindex; index++) {
1717                         if(indexar[index] && indexar[index]<=me->totface) {
1718                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1719                                                 
1720                                 if(mface->mat_nr!=ob->actcol-1) {
1721                                         indexar[index]= 0;
1722                                 }
1723                         }                                       
1724                 }
1725         }
1726         if((G.f & G_FACESELECT) && me->mface) {
1727                 for(index=0; index<totindex; index++) {
1728                         if(indexar[index] && indexar[index]<=me->totface) {
1729                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1730                                                 
1731                                 if((mface->flag & ME_FACE_SEL)==0)
1732                                         indexar[index]= 0;
1733                         }                                       
1734                 }
1735         }
1736                         
1737         for(index=0; index<totindex; index++) {
1738                                 
1739                 if(indexar[index] && indexar[index]<=me->totface) {
1740                         MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1741                         unsigned int *mcol=       ( (unsigned int *)me->mcol) + 4*(indexar[index]-1);
1742                         unsigned int *mcolorig= ( (unsigned int *)vp->vpaint_prev) + 4*(indexar[index]-1);
1743                         int alpha;
1744                                         
1745                         if(vp->mode==VP_BLUR) {
1746                                 unsigned int fcol1= mcol_blend( mcol[0], mcol[1], 128);
1747                                 if(mface->v4) {
1748                                         unsigned int fcol2= mcol_blend( mcol[2], mcol[3], 128);
1749                                         vpd->paintcol= mcol_blend( fcol1, fcol2, 128);
1750                                 }
1751                                 else {
1752                                         vpd->paintcol= mcol_blend( mcol[2], fcol1, 170);
1753                                 }
1754                                                 
1755                         }
1756                                         
1757                         alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*mface->v1, mval);
1758                         if(alpha) vpaint_blend(vp, mcol, mcolorig, vpd->paintcol, alpha);
1759                                         
1760                         alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*mface->v2, mval);
1761                         if(alpha) vpaint_blend(vp, mcol+1, mcolorig+1, vpd->paintcol, alpha);
1762                                         
1763                         alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*mface->v3, mval);
1764                         if(alpha) vpaint_blend(vp, mcol+2, mcolorig+2, vpd->paintcol, alpha);
1765                                         
1766                         if(mface->v4) {
1767                                 alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*mface->v4, mval);
1768                                 if(alpha) vpaint_blend(vp, mcol+3, mcolorig+3, vpd->paintcol, alpha);
1769                         }
1770                 }
1771         }
1772                                                 
1773         MTC_Mat4SwapMat4(vc->rv3d->persmat, mat);
1774                         
1775         do_shared_vertexcol(me);
1776                         
1777         ED_region_tag_redraw(vc->ar);
1778                         
1779         DAG_object_flush_update(vc->scene, ob, OB_RECALC_DATA);
1780 }
1781
1782 static void vpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
1783 {
1784         ToolSettings *ts= CTX_data_tool_settings(C);
1785         struct VPaintData *vpd= paint_stroke_mode_data(stroke);
1786         
1787         if(vpd->vertexcosnos)
1788                 MEM_freeN(vpd->vertexcosnos);
1789         MEM_freeN(vpd->indexar);
1790         
1791         /* frees prev buffer */
1792         copy_vpaint_prev(ts->vpaint, NULL, 0);
1793         
1794         MEM_freeN(vpd);
1795 }
1796
1797 static int vpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1798 {
1799         
1800         op->customdata = paint_stroke_new(C, vpaint_stroke_test_start,
1801                                           vpaint_stroke_update_step,
1802                                           vpaint_stroke_done);
1803         
1804         /* add modal handler */
1805         WM_event_add_modal_handler(C, &CTX_wm_window(C)->handlers, op);
1806         
1807         return OPERATOR_RUNNING_MODAL;
1808 }
1809
1810 void PAINT_OT_vertex_paint(wmOperatorType *ot)
1811 {
1812         /* identifiers */
1813         ot->name= "Vertex Paint";
1814         ot->idname= "PAINT_OT_vertex_paint";
1815         
1816         /* api callbacks */
1817         ot->invoke= vpaint_invoke;
1818         ot->modal= paint_stroke_modal;
1819         /* ot->exec= vpaint_exec; <-- needs stroke property */
1820         ot->poll= vp_poll;
1821         
1822         /* flags */
1823         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1824
1825         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1826 }
1827