1dac98780da8adf3e00616f667ac3af736e47e6f
[blender.git] / source / blender / editors / sculpt_paint / paint_vertex.c
1 /**
2  * $Id$
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  */
29
30 #include <math.h>
31 #include <string.h>
32
33 #ifdef WIN32
34 #include <io.h>
35 #else
36 #include <unistd.h>
37 #endif   
38
39 #include "MEM_guardedalloc.h"
40
41 #include "IMB_imbuf.h"
42 #include "IMB_imbuf_types.h"
43
44 #include "BLI_blenlib.h"
45 #include "BLI_arithb.h"
46
47
48 #include "DNA_action_types.h"
49 #include "DNA_armature_types.h"
50 #include "DNA_brush_types.h"
51 #include "DNA_cloth_types.h"
52 #include "DNA_mesh_types.h"
53 #include "DNA_meshdata_types.h"
54 #include "DNA_modifier_types.h"
55 #include "DNA_object_types.h"
56 #include "DNA_object_force.h"
57 #include "DNA_particle_types.h"
58 #include "DNA_screen_types.h"
59 #include "DNA_space_types.h"
60 #include "DNA_scene_types.h"
61 #include "DNA_view3d_types.h"
62 #include "DNA_userdef_types.h"
63
64 #include "RNA_access.h"
65 #include "RNA_define.h"
66
67 #include "BKE_armature.h"
68 #include "BKE_brush.h"
69 #include "BKE_DerivedMesh.h"
70 #include "BKE_cloth.h"
71 #include "BKE_context.h"
72 #include "BKE_customdata.h"
73 #include "BKE_depsgraph.h"
74 #include "BKE_deform.h"
75 #include "BKE_displist.h"
76 #include "BKE_global.h"
77 #include "BKE_mesh.h"
78 #include "BKE_modifier.h"
79 #include "BKE_object.h"
80 #include "BKE_paint.h"
81 #include "BKE_utildefines.h"
82
83 #include "WM_api.h"
84 #include "WM_types.h"
85
86 #include "BIF_gl.h"
87 #include "BIF_glutil.h"
88
89 #include "ED_mesh.h"
90 #include "ED_object.h"
91 #include "ED_screen.h"
92 #include "ED_util.h"
93 #include "ED_view3d.h"
94
95 #include "paint_intern.h"
96
97         /* vp->mode */
98 #define VP_MIX  0
99 #define VP_ADD  1
100 #define VP_SUB  2
101 #define VP_MUL  3
102 #define VP_BLUR 4
103 #define VP_LIGHTEN      5
104 #define VP_DARKEN       6
105
106 #define MAXINDEX        512000
107
108 /* XXX */
109 static void error() {}
110
111 /* polling - retrieve whether cursor should be set or operator should be done */
112
113
114 /* Returns true if vertex paint mode is active */
115 int vertex_paint_mode_poll(bContext *C)
116 {
117         Object *ob = CTX_data_active_object(C);
118
119         return ob && ob->mode == OB_MODE_VERTEX_PAINT;
120 }
121
122 int vertex_paint_poll(bContext *C)
123 {
124         if(vertex_paint_mode_poll(C) && 
125            paint_brush(&CTX_data_tool_settings(C)->vpaint->paint)) {
126                 ScrArea *sa= CTX_wm_area(C);
127                 if(sa->spacetype==SPACE_VIEW3D) {
128                         ARegion *ar= CTX_wm_region(C);
129                         if(ar->regiontype==RGN_TYPE_WINDOW)
130                                 return 1;
131                 }
132         }
133         return 0;
134 }
135
136 int weight_paint_poll(bContext *C)
137 {
138         Object *ob = CTX_data_active_object(C);
139
140         if(ob && ob->mode & OB_MODE_WEIGHT_PAINT &&
141            paint_brush(&CTX_data_tool_settings(C)->wpaint->paint)) {
142                 ScrArea *sa= CTX_wm_area(C);
143                 if(sa->spacetype==SPACE_VIEW3D) {
144                         ARegion *ar= CTX_wm_region(C);
145                         if(ar->regiontype==RGN_TYPE_WINDOW)
146                                 return 1;
147                 }
148         }
149         return 0;
150 }
151
152 static VPaint *new_vpaint(int wpaint)
153 {
154         VPaint *vp= MEM_callocN(sizeof(VPaint), "VPaint");
155         
156         vp->gamma= vp->mul= 1.0f;
157         
158         vp->flag= VP_AREA+VP_SOFT+VP_SPRAY;
159         
160         if(wpaint)
161                 vp->flag= VP_AREA+VP_SOFT;
162
163         return vp;
164 }
165
166 static int *get_indexarray(void)
167 {
168         return MEM_mallocN(sizeof(int)*MAXINDEX + 2, "vertexpaint");
169 }
170
171
172 /* in contradiction to cpack drawing colors, the MCOL colors (vpaint colors) are per byte! 
173    so not endian sensitive. Mcol = ABGR!!! so be cautious with cpack calls */
174
175 unsigned int rgba_to_mcol(float r, float g, float b, float a)
176 {
177         int ir, ig, ib, ia;
178         unsigned int col;
179         char *cp;
180         
181         ir= floor(255.0*r);
182         if(ir<0) ir= 0; else if(ir>255) ir= 255;
183         ig= floor(255.0*g);
184         if(ig<0) ig= 0; else if(ig>255) ig= 255;
185         ib= floor(255.0*b);
186         if(ib<0) ib= 0; else if(ib>255) ib= 255;
187         ia= floor(255.0*a);
188         if(ia<0) ia= 0; else if(ia>255) ia= 255;
189         
190         cp= (char *)&col;
191         cp[0]= ia;
192         cp[1]= ib;
193         cp[2]= ig;
194         cp[3]= ir;
195         
196         return col;
197         
198 }
199
200 unsigned int vpaint_get_current_col(VPaint *vp)
201 {
202         Brush *brush = paint_brush(&vp->paint);
203         return rgba_to_mcol(brush->rgb[0], brush->rgb[1], brush->rgb[2], 1.0f);
204 }
205
206 static void do_shared_vertexcol(Mesh *me)
207 {
208         /* if no mcol: do not do */
209         /* if tface: only the involved faces, otherwise all */
210         MFace *mface;
211         MTFace *tface;
212         int a;
213         short *scolmain, *scol;
214         char *mcol;
215         
216         if(me->mcol==0 || me->totvert==0 || me->totface==0) return;
217         
218         scolmain= MEM_callocN(4*sizeof(short)*me->totvert, "colmain");
219         
220         tface= me->mtface;
221         mface= me->mface;
222         mcol= (char *)me->mcol;
223         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
224                 if((tface && tface->mode & TF_SHAREDCOL) || (me->editflag & ME_EDIT_PAINT_MASK)==0) {
225                         scol= scolmain+4*mface->v1;
226                         scol[0]++; scol[1]+= mcol[1]; scol[2]+= mcol[2]; scol[3]+= mcol[3];
227                         scol= scolmain+4*mface->v2;
228                         scol[0]++; scol[1]+= mcol[5]; scol[2]+= mcol[6]; scol[3]+= mcol[7];
229                         scol= scolmain+4*mface->v3;
230                         scol[0]++; scol[1]+= mcol[9]; scol[2]+= mcol[10]; scol[3]+= mcol[11];
231                         if(mface->v4) {
232                                 scol= scolmain+4*mface->v4;
233                                 scol[0]++; scol[1]+= mcol[13]; scol[2]+= mcol[14]; scol[3]+= mcol[15];
234                         }
235                 }
236                 if(tface) tface++;
237         }
238         
239         a= me->totvert;
240         scol= scolmain;
241         while(a--) {
242                 if(scol[0]>1) {
243                         scol[1]/= scol[0];
244                         scol[2]/= scol[0];
245                         scol[3]/= scol[0];
246                 }
247                 scol+= 4;
248         }
249         
250         tface= me->mtface;
251         mface= me->mface;
252         mcol= (char *)me->mcol;
253         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
254                 if((tface && tface->mode & TF_SHAREDCOL) || (me->editflag & ME_EDIT_PAINT_MASK)==0) {
255                         scol= scolmain+4*mface->v1;
256                         mcol[1]= scol[1]; mcol[2]= scol[2]; mcol[3]= scol[3];
257                         scol= scolmain+4*mface->v2;
258                         mcol[5]= scol[1]; mcol[6]= scol[2]; mcol[7]= scol[3];
259                         scol= scolmain+4*mface->v3;
260                         mcol[9]= scol[1]; mcol[10]= scol[2]; mcol[11]= scol[3];
261                         if(mface->v4) {
262                                 scol= scolmain+4*mface->v4;
263                                 mcol[13]= scol[1]; mcol[14]= scol[2]; mcol[15]= scol[3];
264                         }
265                 }
266                 if(tface) tface++;
267         }
268
269         MEM_freeN(scolmain);
270 }
271
272 static void make_vertexcol(Object *ob)  /* single ob */
273 {
274         Mesh *me;
275         if(!ob || ob->id.lib) return;
276         me= get_mesh(ob);
277         if(me==0) return;
278         if(me->edit_mesh) return;
279
280         /* copies from shadedisplist to mcol */
281         if(!me->mcol) {
282                 CustomData_add_layer(&me->fdata, CD_MCOL, CD_CALLOC, NULL, me->totface);
283                 mesh_update_customdata_pointers(me);
284         }
285
286         //if(shade)
287         //      shadeMeshMCol(scene, ob, me);
288         //else
289
290         memset(me->mcol, 255, 4*sizeof(MCol)*me->totface);
291         
292         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
293         
294 }
295
296 static void copy_vpaint_prev(VPaint *vp, unsigned int *mcol, int tot)
297 {
298         if(vp->vpaint_prev) {
299                 MEM_freeN(vp->vpaint_prev);
300                 vp->vpaint_prev= NULL;
301         }
302         vp->tot= tot;   
303         
304         if(mcol==NULL || tot==0) return;
305         
306         vp->vpaint_prev= MEM_mallocN(4*sizeof(int)*tot, "vpaint_prev");
307         memcpy(vp->vpaint_prev, mcol, 4*sizeof(int)*tot);
308         
309 }
310
311 static void copy_wpaint_prev (VPaint *wp, MDeformVert *dverts, int dcount)
312 {
313         if (wp->wpaint_prev) {
314                 free_dverts(wp->wpaint_prev, wp->tot);
315                 wp->wpaint_prev= NULL;
316         }
317         
318         if(dverts && dcount) {
319                 
320                 wp->wpaint_prev = MEM_mallocN (sizeof(MDeformVert)*dcount, "wpaint prev");
321                 wp->tot = dcount;
322                 copy_dverts (wp->wpaint_prev, dverts, dcount);
323         }
324 }
325
326
327 void vpaint_fill(Object *ob, unsigned int paintcol)
328 {
329         Mesh *me;
330         MFace *mf;
331         unsigned int *mcol;
332         int i, selected;
333
334         me= get_mesh(ob);
335         if(me==0 || me->totface==0) return;
336
337         if(!me->mcol)
338                 make_vertexcol(ob);
339
340         selected= (me->editflag & ME_EDIT_PAINT_MASK);
341
342         mf = me->mface;
343         mcol = (unsigned int*)me->mcol;
344         for (i = 0; i < me->totface; i++, mf++, mcol+=4) {
345                 if (!selected || mf->flag & ME_FACE_SEL) {
346                         mcol[0] = paintcol;
347                         mcol[1] = paintcol;
348                         mcol[2] = paintcol;
349                         mcol[3] = paintcol;
350                 }
351         }
352         
353         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
354 }
355
356
357 /* fills in the selected faces with the current weight and vertex group */
358 void wpaint_fill(VPaint *wp, Object *ob, float paintweight)
359 {
360         Mesh *me;
361         MFace *mface;
362         MDeformWeight *dw, *uw;
363         int *indexar;
364         int index, vgroup;
365         unsigned int faceverts[5]={0,0,0,0,0};
366         unsigned char i;
367         int vgroup_mirror= -1;
368         int selected;
369         
370         me= ob->data;
371         if(me==0 || me->totface==0 || me->dvert==0 || !me->mface) return;
372         
373         selected= (me->editflag & ME_EDIT_PAINT_MASK);
374
375         indexar= get_indexarray();
376         if(selected) {
377                 for(index=0, mface=me->mface; index<me->totface; index++, mface++) {
378                         if((mface->flag & ME_FACE_SEL)==0)
379                                 indexar[index]= 0;
380                         else
381                                 indexar[index]= index+1;
382                 }
383         }
384         else {
385                 for(index=0; index<me->totface; index++)
386                         indexar[index]= index+1;
387         }
388         
389         vgroup= ob->actdef-1;
390         
391         /* directly copied from weight_paint, should probaby split into a seperate function */
392         /* if mirror painting, find the other group */          
393         if(me->editflag & ME_EDIT_MIRROR_X) {
394                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
395                 if(defgroup) {
396                         bDeformGroup *curdef;
397                         int actdef= 0;
398                         char name[32];
399
400                         BLI_strncpy(name, defgroup->name, 32);
401                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
402                         
403                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
404                                 if (!strcmp(curdef->name, name))
405                                         break;
406                         if(curdef==NULL) {
407                                 int olddef= ob->actdef; /* tsk, ED_vgroup_add sets the active defgroup */
408                                 curdef= ED_vgroup_add_name (ob, name);
409                                 ob->actdef= olddef;
410                         }
411                         
412                         if(curdef && curdef!=defgroup)
413                                 vgroup_mirror= actdef;
414                 }
415         }
416         /* end copy from weight_paint*/
417         
418         copy_wpaint_prev(wp, me->dvert, me->totvert);
419         
420         for(index=0; index<me->totface; index++) {
421                 if(indexar[index] && indexar[index]<=me->totface) {
422                         mface= me->mface + (indexar[index]-1);
423                         /* just so we can loop through the verts */
424                         faceverts[0]= mface->v1;
425                         faceverts[1]= mface->v2;
426                         faceverts[2]= mface->v3;
427                         faceverts[3]= mface->v4;
428                         for (i=0; i<3 || faceverts[i]; i++) {
429                                 if(!((me->dvert+faceverts[i])->flag)) {
430                                         dw= ED_vgroup_weight_verify(me->dvert+faceverts[i], vgroup);
431                                         if(dw) {
432                                                 uw= ED_vgroup_weight_verify(wp->wpaint_prev+faceverts[i], vgroup);
433                                                 uw->weight= dw->weight; /* set the undo weight */
434                                                 dw->weight= paintweight;
435                                                 
436                                                 if(me->editflag & ME_EDIT_MIRROR_X) {   /* x mirror painting */
437                                                         int j= mesh_get_x_mirror_vert(ob, faceverts[i]);
438                                                         if(j>=0) {
439                                                                 /* copy, not paint again */
440                                                                 if(vgroup_mirror != -1) {
441                                                                         dw= ED_vgroup_weight_verify(me->dvert+j, vgroup_mirror);
442                                                                         uw= ED_vgroup_weight_verify(wp->wpaint_prev+j, vgroup_mirror);
443                                                                 } else {
444                                                                         dw= ED_vgroup_weight_verify(me->dvert+j, vgroup);
445                                                                         uw= ED_vgroup_weight_verify(wp->wpaint_prev+j, vgroup);
446                                                                 }
447                                                                 uw->weight= dw->weight; /* set the undo weight */
448                                                                 dw->weight= paintweight;
449                                                         }
450                                                 }
451                                         }
452                                         (me->dvert+faceverts[i])->flag= 1;
453                                 }
454                         }
455                 }
456         }
457         
458         index=0;
459         while (index<me->totvert) {
460                 (me->dvert+index)->flag= 0;
461                 index++;
462         }
463         
464         MEM_freeN(indexar);
465         copy_wpaint_prev(wp, NULL, 0);
466
467         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
468 }
469
470
471 void vpaint_dogamma(Scene *scene)
472 {
473         VPaint *vp= scene->toolsettings->vpaint;
474         Mesh *me;
475         Object *ob;
476         float igam, fac;
477         int a, temp;
478         unsigned char *cp, gamtab[256];
479
480         ob= OBACT;
481         me= get_mesh(ob);
482
483         if(!(ob->mode & OB_MODE_VERTEX_PAINT)) return;
484         if(me==0 || me->mcol==0 || me->totface==0) return;
485
486         igam= 1.0/vp->gamma;
487         for(a=0; a<256; a++) {
488                 
489                 fac= ((float)a)/255.0;
490                 fac= vp->mul*pow( fac, igam);
491                 
492                 temp= 255.9*fac;
493                 
494                 if(temp<=0) gamtab[a]= 0;
495                 else if(temp>=255) gamtab[a]= 255;
496                 else gamtab[a]= temp;
497         }
498
499         a= 4*me->totface;
500         cp= (unsigned char *)me->mcol;
501         while(a--) {
502                 
503                 cp[1]= gamtab[ cp[1] ];
504                 cp[2]= gamtab[ cp[2] ];
505                 cp[3]= gamtab[ cp[3] ];
506                 
507                 cp+= 4;
508         }
509 }
510
511 static unsigned int mcol_blend(unsigned int col1, unsigned int col2, int fac)
512 {
513         char *cp1, *cp2, *cp;
514         int mfac;
515         unsigned int col=0;
516         
517         if(fac==0) return col1;
518         if(fac>=255) return col2;
519
520         mfac= 255-fac;
521         
522         cp1= (char *)&col1;
523         cp2= (char *)&col2;
524         cp=  (char *)&col;
525         
526         cp[0]= 255;
527         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
528         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
529         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
530         
531         return col;
532 }
533
534 static unsigned int mcol_add(unsigned int col1, unsigned int col2, int fac)
535 {
536         char *cp1, *cp2, *cp;
537         int temp;
538         unsigned int col=0;
539         
540         if(fac==0) return col1;
541         
542         cp1= (char *)&col1;
543         cp2= (char *)&col2;
544         cp=  (char *)&col;
545         
546         cp[0]= 255;
547         temp= cp1[1] + ((fac*cp2[1])/255);
548         if(temp>254) cp[1]= 255; else cp[1]= temp;
549         temp= cp1[2] + ((fac*cp2[2])/255);
550         if(temp>254) cp[2]= 255; else cp[2]= temp;
551         temp= cp1[3] + ((fac*cp2[3])/255);
552         if(temp>254) cp[3]= 255; else cp[3]= temp;
553         
554         return col;
555 }
556
557 static unsigned int mcol_sub(unsigned int col1, unsigned int col2, int fac)
558 {
559         char *cp1, *cp2, *cp;
560         int temp;
561         unsigned int col=0;
562         
563         if(fac==0) return col1;
564         
565         cp1= (char *)&col1;
566         cp2= (char *)&col2;
567         cp=  (char *)&col;
568         
569         cp[0]= 255;
570         temp= cp1[1] - ((fac*cp2[1])/255);
571         if(temp<0) cp[1]= 0; else cp[1]= temp;
572         temp= cp1[2] - ((fac*cp2[2])/255);
573         if(temp<0) cp[2]= 0; else cp[2]= temp;
574         temp= cp1[3] - ((fac*cp2[3])/255);
575         if(temp<0) cp[3]= 0; else cp[3]= temp;
576         
577         return col;
578 }
579
580 static unsigned int mcol_mul(unsigned int col1, unsigned int col2, int fac)
581 {
582         char *cp1, *cp2, *cp;
583         int mfac;
584         unsigned int col=0;
585         
586         if(fac==0) return col1;
587
588         mfac= 255-fac;
589         
590         cp1= (char *)&col1;
591         cp2= (char *)&col2;
592         cp=  (char *)&col;
593         
594         /* first mul, then blend the fac */
595         cp[0]= 255;
596         cp[1]= (mfac*cp1[1] + fac*((cp2[1]*cp1[1])/255)  )/255;
597         cp[2]= (mfac*cp1[2] + fac*((cp2[2]*cp1[2])/255)  )/255;
598         cp[3]= (mfac*cp1[3] + fac*((cp2[3]*cp1[3])/255)  )/255;
599
600         
601         return col;
602 }
603
604 static unsigned int mcol_lighten(unsigned int col1, unsigned int col2, int fac)
605 {
606         char *cp1, *cp2, *cp;
607         int mfac;
608         unsigned int col=0;
609         
610         if(fac==0) return col1;
611         if(fac>=255) return col2;
612
613         mfac= 255-fac;
614         
615         cp1= (char *)&col1;
616         cp2= (char *)&col2;
617         cp=  (char *)&col;
618         
619         /* See if are lighter, if so mix, else dont do anything.
620         if the paint col is darker then the original, then ignore */
621         if (cp1[1]+cp1[2]+cp1[3] > cp2[1]+cp2[2]+cp2[3])
622                 return col1;
623         
624         cp[0]= 255;
625         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
626         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
627         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
628         
629         return col;
630 }
631
632 static unsigned int mcol_darken(unsigned int col1, unsigned int col2, int fac)
633 {
634         char *cp1, *cp2, *cp;
635         int mfac;
636         unsigned int col=0;
637         
638         if(fac==0) return col1;
639         if(fac>=255) return col2;
640
641         mfac= 255-fac;
642         
643         cp1= (char *)&col1;
644         cp2= (char *)&col2;
645         cp=  (char *)&col;
646         
647         /* See if were darker, if so mix, else dont do anything.
648         if the paint col is brighter then the original, then ignore */
649         if (cp1[1]+cp1[2]+cp1[3] < cp2[1]+cp2[2]+cp2[3])
650                 return col1;
651         
652         cp[0]= 255;
653         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
654         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
655         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
656         return col;
657 }
658
659 static void vpaint_blend(VPaint *vp, unsigned int *col, unsigned int *colorig, unsigned int paintcol, int alpha)
660 {
661         Brush *brush = paint_brush(&vp->paint);
662
663         if(vp->mode==VP_MIX || vp->mode==VP_BLUR) *col= mcol_blend( *col, paintcol, alpha);
664         else if(vp->mode==VP_ADD) *col= mcol_add( *col, paintcol, alpha);
665         else if(vp->mode==VP_SUB) *col= mcol_sub( *col, paintcol, alpha);
666         else if(vp->mode==VP_MUL) *col= mcol_mul( *col, paintcol, alpha);
667         else if(vp->mode==VP_LIGHTEN) *col= mcol_lighten( *col, paintcol, alpha);
668         else if(vp->mode==VP_DARKEN) *col= mcol_darken( *col, paintcol, alpha);
669         
670         /* if no spray, clip color adding with colorig & orig alpha */
671         if((vp->flag & VP_SPRAY)==0) {
672                 unsigned int testcol=0, a;
673                 char *cp, *ct, *co;
674                 
675                 alpha= (int)(255.0*brush->alpha);
676                 
677                 if(vp->mode==VP_MIX || vp->mode==VP_BLUR) testcol= mcol_blend( *colorig, paintcol, alpha);
678                 else if(vp->mode==VP_ADD) testcol= mcol_add( *colorig, paintcol, alpha);
679                 else if(vp->mode==VP_SUB) testcol= mcol_sub( *colorig, paintcol, alpha);
680                 else if(vp->mode==VP_MUL) testcol= mcol_mul( *colorig, paintcol, alpha);
681                 else if(vp->mode==VP_LIGHTEN)  testcol= mcol_lighten( *colorig, paintcol, alpha);
682                 else if(vp->mode==VP_DARKEN)   testcol= mcol_darken( *colorig, paintcol, alpha);
683                 
684                 cp= (char *)col;
685                 ct= (char *)&testcol;
686                 co= (char *)colorig;
687                 
688                 for(a=0; a<4; a++) {
689                         if( ct[a]<co[a] ) {
690                                 if( cp[a]<ct[a] ) cp[a]= ct[a];
691                                 else if( cp[a]>co[a] ) cp[a]= co[a];
692                         }
693                         else {
694                                 if( cp[a]<co[a] ) cp[a]= co[a];
695                                 else if( cp[a]>ct[a] ) cp[a]= ct[a];
696                         }
697                 }
698         }
699 }
700
701
702 static int sample_backbuf_area(ViewContext *vc, int *indexar, int totface, int x, int y, float size)
703 {
704         struct ImBuf *ibuf;
705         int a, tot=0, index;
706         
707         if(totface+4>=MAXINDEX) return 0;
708         
709         /* brecht: disabled this because it obviously failes for
710            brushes with size > 64, why is this here? */
711         /*if(size>64.0) size= 64.0;*/
712         
713         ibuf= view3d_read_backbuf(vc, x-size, y-size, x+size, y+size);
714         if(ibuf) {
715                 unsigned int *rt= ibuf->rect;
716
717                 memset(indexar, 0, sizeof(int)*totface+4);      /* plus 2! first element is total, +2 was giving valgrind errors, +4 seems ok */
718                 
719                 size= ibuf->x*ibuf->y;
720                 while(size--) {
721                                 
722                         if(*rt) {
723                                 index= WM_framebuffer_to_index(*rt);
724                                 if(index>0 && index<=totface)
725                                         indexar[index] = 1;
726                         }
727                 
728                         rt++;
729                 }
730                 
731                 for(a=1; a<=totface; a++) {
732                         if(indexar[a]) indexar[tot++]= a;
733                 }
734
735                 IMB_freeImBuf(ibuf);
736         }
737         
738         return tot;
739 }
740
741 static int calc_vp_alpha_dl(VPaint *vp, ViewContext *vc, float vpimat[][3], float *vert_nor, float *mval)
742 {
743         Brush *brush = paint_brush(&vp->paint);
744         float fac, dx, dy;
745         int alpha;
746         short vertco[2];
747         
748         if(vp->flag & VP_SOFT) {
749                 project_short_noclip(vc->ar, vert_nor, vertco);
750                 dx= mval[0]-vertco[0];
751                 dy= mval[1]-vertco[1];
752                 
753                 fac= sqrt(dx*dx + dy*dy);
754                 if(fac > brush->size) return 0;
755                 if(vp->flag & VP_HARD)
756                         alpha= 255;
757                 else
758                         alpha= 255.0*brush->alpha*(1.0-fac/brush->size);
759         }
760         else {
761                 alpha= 255.0*brush->alpha;
762         }
763
764         if(vp->flag & VP_NORMALS) {
765                 float *no= vert_nor+3;
766                 
767                         /* transpose ! */
768                 fac= vpimat[2][0]*no[0]+vpimat[2][1]*no[1]+vpimat[2][2]*no[2];
769                 if(fac>0.0) {
770                         dx= vpimat[0][0]*no[0]+vpimat[0][1]*no[1]+vpimat[0][2]*no[2];
771                         dy= vpimat[1][0]*no[0]+vpimat[1][1]*no[1]+vpimat[1][2]*no[2];
772                         
773                         alpha*= fac/sqrt(dx*dx + dy*dy + fac*fac);
774                 }
775                 else return 0;
776         }
777         
778         return alpha;
779 }
780
781 static void wpaint_blend(VPaint *wp, MDeformWeight *dw, MDeformWeight *uw, float alpha, float paintval)
782 {
783         Brush *brush = paint_brush(&wp->paint);
784         
785         if(dw==NULL || uw==NULL) return;
786         
787         if(wp->mode==VP_MIX || wp->mode==VP_BLUR)
788                 dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
789         else if(wp->mode==VP_ADD)
790                 dw->weight += paintval*alpha;
791         else if(wp->mode==VP_SUB) 
792                 dw->weight -= paintval*alpha;
793         else if(wp->mode==VP_MUL) 
794                 /* first mul, then blend the fac */
795                 dw->weight = ((1.0-alpha) + alpha*paintval)*dw->weight;
796         else if(wp->mode==VP_LIGHTEN) {
797                 if (dw->weight < paintval)
798                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
799         } else if(wp->mode==VP_DARKEN) {
800                 if (dw->weight > paintval)
801                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
802         }
803         CLAMP(dw->weight, 0.0f, 1.0f);
804         
805         /* if no spray, clip result with orig weight & orig alpha */
806         if((wp->flag & VP_SPRAY)==0) {
807                 float testw=0.0f;
808                 
809                 alpha= brush->alpha;
810                 if(wp->mode==VP_MIX || wp->mode==VP_BLUR)
811                         testw = paintval*alpha + uw->weight*(1.0-alpha);
812                 else if(wp->mode==VP_ADD)
813                         testw = uw->weight + paintval*alpha;
814                 else if(wp->mode==VP_SUB) 
815                         testw = uw->weight - paintval*alpha;
816                 else if(wp->mode==VP_MUL) 
817                         /* first mul, then blend the fac */
818                         testw = ((1.0-alpha) + alpha*paintval)*uw->weight;              
819                 else if(wp->mode==VP_LIGHTEN) {
820                         if (uw->weight < paintval)
821                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
822                         else
823                                 testw = uw->weight;
824                 } else if(wp->mode==VP_DARKEN) {
825                         if (uw->weight > paintval)
826                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
827                         else
828                                 testw = uw->weight;
829                 }
830                 CLAMP(testw, 0.0f, 1.0f);
831                 
832                 if( testw<uw->weight ) {
833                         if(dw->weight < testw) dw->weight= testw;
834                         else if(dw->weight > uw->weight) dw->weight= uw->weight;
835                 }
836                 else {
837                         if(dw->weight > testw) dw->weight= testw;
838                         else if(dw->weight < uw->weight) dw->weight= uw->weight;
839                 }
840         }
841         
842 }
843
844 /* ----------------------------------------------------- */
845
846 /* used for 3d view, on active object, assumes me->dvert exists */
847 /* if mode==1: */
848 /*     samples cursor location, and gives menu with vertex groups to activate */
849 /* else */
850 /*     sets wp->weight to the closest weight value to vertex */
851 /*     note: we cant sample frontbuf, weight colors are interpolated too unpredictable */
852 void sample_wpaint(Scene *scene, ARegion *ar, View3D *v3d, int mode)
853 {
854         ViewContext vc;
855         ToolSettings *ts= scene->toolsettings;
856         Object *ob= OBACT;
857         Mesh *me= get_mesh(ob);
858         int index;
859         short mval[2] = {0, 0}, sco[2];
860
861         if (!me) return;
862         
863 //      getmouseco_areawin(mval);
864         index= view3d_sample_backbuf(&vc, mval[0], mval[1]);
865         
866         if(index && index<=me->totface) {
867                 MFace *mface;
868                 
869                 mface= ((MFace *)me->mface) + index-1;
870                 
871                 if(mode==1) {   /* sampe which groups are in here */
872                         MDeformVert *dv;
873                         int a, totgroup;
874                         
875                         totgroup= BLI_countlist(&ob->defbase);
876                         if(totgroup) {
877                                 int totmenu=0;
878                                 int *groups=MEM_callocN(totgroup*sizeof(int), "groups");
879                                 
880                                 dv= me->dvert+mface->v1;
881                                 for(a=0; a<dv->totweight; a++) {
882                                         if (dv->dw[a].def_nr<totgroup)
883                                                 groups[dv->dw[a].def_nr]= 1;
884                                 }
885                                 dv= me->dvert+mface->v2;
886                                 for(a=0; a<dv->totweight; a++) {
887                                         if (dv->dw[a].def_nr<totgroup)
888                                                 groups[dv->dw[a].def_nr]= 1;
889                                 }
890                                 dv= me->dvert+mface->v3;
891                                 for(a=0; a<dv->totweight; a++) {
892                                         if (dv->dw[a].def_nr<totgroup)
893                                                 groups[dv->dw[a].def_nr]= 1;
894                                 }
895                                 if(mface->v4) {
896                                         dv= me->dvert+mface->v4;
897                                         for(a=0; a<dv->totweight; a++) {
898                                                 if (dv->dw[a].def_nr<totgroup)
899                                                         groups[dv->dw[a].def_nr]= 1;
900                                         }
901                                 }
902                                 for(a=0; a<totgroup; a++)
903                                         if(groups[a]) totmenu++;
904                                 
905                                 if(totmenu==0) {
906                                         //notice("No Vertex Group Selected");
907                                 }
908                                 else {
909                                         bDeformGroup *dg;
910                                         short val;
911                                         char item[40], *str= MEM_mallocN(40*totmenu+40, "menu");
912                                         
913                                         strcpy(str, "Vertex Groups %t");
914                                         for(a=0, dg=ob->defbase.first; dg && a<totgroup; a++, dg= dg->next) {
915                                                 if(groups[a]) {
916                                                         sprintf(item, "|%s %%x%d", dg->name, a);
917                                                         strcat(str, item);
918                                                 }
919                                         }
920                                         
921                                         val= 0; // XXX pupmenu(str);
922                                         if(val>=0) {
923                                                 ob->actdef= val+1;
924                                                 DAG_id_flush_update(&me->id, OB_RECALC_DATA);
925                                         }
926                                         MEM_freeN(str);
927                                 }
928                                 MEM_freeN(groups);
929                         }
930 //                      else notice("No Vertex Groups in Object");
931                 }
932                 else {
933                         DerivedMesh *dm;
934                         MDeformWeight *dw;
935                         float w1, w2, w3, w4, co[3], fac;
936                         
937                         dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
938                         if(dm->getVertCo==NULL) {
939                                 //notice("Not supported yet");
940                         }
941                         else {
942                                 /* calc 3 or 4 corner weights */
943                                 dm->getVertCo(dm, mface->v1, co);
944                                 project_short_noclip(ar, co, sco);
945                                 w1= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
946                                 
947                                 dm->getVertCo(dm, mface->v2, co);
948                                 project_short_noclip(ar, co, sco);
949                                 w2= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
950                                 
951                                 dm->getVertCo(dm, mface->v3, co);
952                                 project_short_noclip(ar, co, sco);
953                                 w3= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
954                                 
955                                 if(mface->v4) {
956                                         dm->getVertCo(dm, mface->v4, co);
957                                         project_short_noclip(ar, co, sco);
958                                         w4= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
959                                 }
960                                 else w4= 1.0e10;
961                                 
962                                 fac= MIN4(w1, w2, w3, w4);
963                                 if(w1==fac) {
964                                         dw= ED_vgroup_weight_get(me->dvert+mface->v1, ob->actdef-1);
965                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
966                                 }
967                                 else if(w2==fac) {
968                                         dw= ED_vgroup_weight_get(me->dvert+mface->v2, ob->actdef-1);
969                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
970                                 }
971                                 else if(w3==fac) {
972                                         dw= ED_vgroup_weight_get(me->dvert+mface->v3, ob->actdef-1);
973                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
974                                 }
975                                 else if(w4==fac) {
976                                         if(mface->v4) {
977                                                 dw= ED_vgroup_weight_get(me->dvert+mface->v4, ob->actdef-1);
978                                                 if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
979                                         }
980                                 }
981                         }
982                         dm->release(dm);
983                 }               
984                 
985         }
986         
987 }
988
989 static void do_weight_paint_vertex(VPaint *wp, Object *ob, int index, int alpha, float paintweight, int vgroup_mirror)
990 {
991         Mesh *me= ob->data;
992         MDeformWeight *dw, *uw;
993         int vgroup= ob->actdef-1;
994         
995         if(wp->flag & VP_ONLYVGROUP) {
996                 dw= ED_vgroup_weight_get(me->dvert+index, vgroup);
997                 uw= ED_vgroup_weight_get(wp->wpaint_prev+index, vgroup);
998         }
999         else {
1000                 dw= ED_vgroup_weight_verify(me->dvert+index, vgroup);
1001                 uw= ED_vgroup_weight_verify(wp->wpaint_prev+index, vgroup);
1002         }
1003         if(dw==NULL || uw==NULL)
1004                 return;
1005         
1006         wpaint_blend(wp, dw, uw, (float)alpha/255.0, paintweight);
1007         
1008         if(me->editflag & ME_EDIT_MIRROR_X) {   /* x mirror painting */
1009                 int j= mesh_get_x_mirror_vert(ob, index);
1010                 if(j>=0) {
1011                         /* copy, not paint again */
1012                         if(vgroup_mirror != -1)
1013                                 uw= ED_vgroup_weight_verify(me->dvert+j, vgroup_mirror);
1014                         else
1015                                 uw= ED_vgroup_weight_verify(me->dvert+j, vgroup);
1016                                 
1017                         uw->weight= dw->weight;
1018                 }
1019         }
1020 }
1021
1022
1023 /* *************** set wpaint operator ****************** */
1024
1025 static int set_wpaint(bContext *C, wmOperator *op)              /* toggle */
1026 {               
1027         Object *ob= CTX_data_active_object(C);
1028         Scene *scene= CTX_data_scene(C);
1029         VPaint *wp= scene->toolsettings->wpaint;
1030         Mesh *me;
1031         
1032         me= get_mesh(ob);
1033         if(ob->id.lib || me==NULL) return OPERATOR_PASS_THROUGH;
1034         
1035         if(me && me->totface>=MAXINDEX) {
1036                 error("Maximum number of faces: %d", MAXINDEX-1);
1037                 ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1038                 return OPERATOR_CANCELLED;
1039         }
1040         
1041         if(ob->mode & OB_MODE_WEIGHT_PAINT) ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1042         else ob->mode |= OB_MODE_WEIGHT_PAINT;
1043         
1044         
1045         /* Weightpaint works by overriding colors in mesh,
1046                 * so need to make sure we recalc on enter and
1047                 * exit (exit needs doing regardless because we
1048                                 * should redeform).
1049                 */
1050         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
1051         
1052         if(ob->mode & OB_MODE_WEIGHT_PAINT) {
1053                 Object *par;
1054                 
1055                 if(wp==NULL)
1056                         wp= scene->toolsettings->wpaint= new_vpaint(1);
1057
1058                 paint_init(&wp->paint, PAINT_CURSOR_WEIGHT_PAINT);
1059                 paint_cursor_start(C, weight_paint_poll);
1060                 
1061                 mesh_octree_table(ob, NULL, NULL, 's');
1062                 
1063                 /* verify if active weight group is also active bone */
1064                 par= modifiers_isDeformedByArmature(ob);
1065                 if(par && (par->mode & OB_MODE_POSE)) {
1066                         bPoseChannel *pchan;
1067                         for(pchan= par->pose->chanbase.first; pchan; pchan= pchan->next)
1068                                 if(pchan->bone->flag & BONE_ACTIVE)
1069                                         break;
1070                                 if(pchan)
1071                                         ED_vgroup_select_by_name(ob, pchan->name);
1072                 }
1073         }
1074         else {
1075                 mesh_octree_table(ob, NULL, NULL, 'e');
1076         }
1077         
1078         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1079         
1080         return OPERATOR_FINISHED;
1081 }
1082
1083 /* for switching to/from mode */
1084 static int paint_poll_test(bContext *C)
1085 {
1086         if(ED_operator_view3d_active(C)==0)
1087                 return 0;
1088         if(CTX_data_edit_object(C))
1089                 return 0;
1090         if(CTX_data_active_object(C)==NULL)
1091                 return 0;
1092         return 1;
1093 }
1094
1095 void PAINT_OT_weight_paint_toggle(wmOperatorType *ot)
1096 {
1097         
1098         /* identifiers */
1099         ot->name= "Weight Paint Mode";
1100         ot->idname= "PAINT_OT_weight_paint_toggle";
1101         
1102         /* api callbacks */
1103         ot->exec= set_wpaint;
1104         ot->poll= paint_poll_test;
1105         
1106         /* flags */
1107         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1108         
1109 }
1110
1111 /* ************ paint radial controls *************/
1112
1113 static int vpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1114 {
1115         Paint *p = paint_get_active(CTX_data_scene(C));
1116         Brush *brush = paint_brush(p);
1117         
1118         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1119         p->paint_cursor = NULL;
1120         brush_radial_control_invoke(op, brush, 1);
1121         return WM_radial_control_invoke(C, op, event);
1122 }
1123
1124 static int vpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1125 {
1126         int ret = WM_radial_control_modal(C, op, event);
1127         if(ret != OPERATOR_RUNNING_MODAL)
1128                 paint_cursor_start(C, vertex_paint_poll);
1129         return ret;
1130 }
1131
1132 static int vpaint_radial_control_exec(bContext *C, wmOperator *op)
1133 {
1134         Brush *brush = paint_brush(&CTX_data_scene(C)->toolsettings->vpaint->paint);
1135         int ret = brush_radial_control_exec(op, brush, 1);
1136         
1137         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1138         
1139         return ret;
1140 }
1141
1142 static int wpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1143 {
1144         Paint *p = paint_get_active(CTX_data_scene(C));
1145         Brush *brush = paint_brush(p);
1146         
1147         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1148         p->paint_cursor = NULL;
1149         brush_radial_control_invoke(op, brush, 1);
1150         return WM_radial_control_invoke(C, op, event);
1151 }
1152
1153 static int wpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1154 {
1155         int ret = WM_radial_control_modal(C, op, event);
1156         if(ret != OPERATOR_RUNNING_MODAL)
1157                 paint_cursor_start(C, weight_paint_poll);
1158         return ret;
1159 }
1160
1161 static int wpaint_radial_control_exec(bContext *C, wmOperator *op)
1162 {
1163         Brush *brush = paint_brush(&CTX_data_scene(C)->toolsettings->wpaint->paint);
1164         int ret = brush_radial_control_exec(op, brush, 1);
1165         
1166         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1167         
1168         return ret;
1169 }
1170
1171 void PAINT_OT_weight_paint_radial_control(wmOperatorType *ot)
1172 {
1173         WM_OT_radial_control_partial(ot);
1174
1175         ot->name= "Weight Paint Radial Control";
1176         ot->idname= "PAINT_OT_weight_paint_radial_control";
1177
1178         ot->invoke= wpaint_radial_control_invoke;
1179         ot->modal= wpaint_radial_control_modal;
1180         ot->exec= wpaint_radial_control_exec;
1181         ot->poll= weight_paint_poll;
1182         
1183         /* flags */
1184         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1185 }
1186
1187 void PAINT_OT_vertex_paint_radial_control(wmOperatorType *ot)
1188 {
1189         WM_OT_radial_control_partial(ot);
1190
1191         ot->name= "Vertex Paint Radial Control";
1192         ot->idname= "PAINT_OT_vertex_paint_radial_control";
1193
1194         ot->invoke= vpaint_radial_control_invoke;
1195         ot->modal= vpaint_radial_control_modal;
1196         ot->exec= vpaint_radial_control_exec;
1197         ot->poll= vertex_paint_poll;
1198         
1199         /* flags */
1200         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1201 }
1202
1203 /* ************ weight paint operator ********** */
1204
1205 struct WPaintData {
1206         ViewContext vc;
1207         int *indexar;
1208         int vgroup_mirror;
1209         float *vertexcosnos;
1210         float wpimat[3][3];
1211 };
1212
1213 static int wpaint_stroke_test_start(bContext *C, wmOperator *op, wmEvent *event)
1214 {
1215         Scene *scene= CTX_data_scene(C);
1216         struct PaintStroke *stroke = op->customdata;
1217         ToolSettings *ts= CTX_data_tool_settings(C);
1218         VPaint *wp= ts->wpaint;
1219         Object *ob= CTX_data_active_object(C);
1220         struct WPaintData *wpd;
1221         Mesh *me;
1222         float mat[4][4], imat[4][4];
1223         
1224         if(scene->obedit) return OPERATOR_CANCELLED;
1225         
1226         me= get_mesh(ob);
1227         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1228         
1229         /* if nothing was added yet, we make dverts and a vertex deform group */
1230         if (!me->dvert)
1231                 ED_vgroup_data_create(&me->id);
1232         
1233         /* make mode data storage */
1234         wpd= MEM_callocN(sizeof(struct WPaintData), "WPaintData");
1235         paint_stroke_set_mode_data(stroke, wpd);
1236         view3d_set_viewcontext(C, &wpd->vc);
1237         wpd->vgroup_mirror= -1;
1238         
1239         //      if(qual & LR_CTRLKEY) {
1240         //              sample_wpaint(scene, ar, v3d, 0);
1241         //              return;
1242         //      }
1243         //      if(qual & LR_SHIFTKEY) {
1244         //              sample_wpaint(scene, ar, v3d, 1);
1245         //              return;
1246         //      }
1247         
1248         /* ALLOCATIONS! no return after this line */
1249         /* painting on subsurfs should give correct points too, this returns me->totvert amount */
1250         wpd->vertexcosnos= mesh_get_mapped_verts_nors(scene, ob);
1251         wpd->indexar= get_indexarray();
1252         copy_wpaint_prev(wp, me->dvert, me->totvert);
1253         
1254         /* this happens on a Bone select, when no vgroup existed yet */
1255         if(ob->actdef<=0) {
1256                 Object *modob;
1257                 if((modob = modifiers_isDeformedByArmature(ob))) {
1258                         bPoseChannel *pchan;
1259                         for(pchan= modob->pose->chanbase.first; pchan; pchan= pchan->next)
1260                                 if(pchan->bone->flag & SELECT)
1261                                         break;
1262                         if(pchan) {
1263                                 bDeformGroup *dg= get_named_vertexgroup(ob, pchan->name);
1264                                 if(dg==NULL)
1265                                         dg= ED_vgroup_add_name(ob, pchan->name);        /* sets actdef */
1266                                 else
1267                                         ob->actdef= get_defgroup_num(ob, dg);
1268                         }
1269                 }
1270         }
1271         if(ob->defbase.first==NULL) {
1272                 ED_vgroup_add(ob);
1273         }       
1274         
1275         //      if(ob->lay & v3d->lay); else error("Active object is not in this layer");
1276         
1277         /* imat for normals */
1278         Mat4MulMat4(mat, ob->obmat, wpd->vc.rv3d->viewmat);
1279         Mat4Invert(imat, mat);
1280         Mat3CpyMat4(wpd->wpimat, imat);
1281         
1282         /* if mirror painting, find the other group */
1283         if(me->editflag & ME_EDIT_MIRROR_X) {
1284                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
1285                 if(defgroup) {
1286                         bDeformGroup *curdef;
1287                         int actdef= 0;
1288                         char name[32];
1289                         
1290                         BLI_strncpy(name, defgroup->name, 32);
1291                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
1292                         
1293                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
1294                                 if (!strcmp(curdef->name, name))
1295                                         break;
1296                         if(curdef==NULL) {
1297                                 int olddef= ob->actdef; /* tsk, ED_vgroup_add sets the active defgroup */
1298                                 curdef= ED_vgroup_add_name (ob, name);
1299                                 ob->actdef= olddef;
1300                         }
1301                         
1302                         if(curdef && curdef!=defgroup)
1303                                 wpd->vgroup_mirror= actdef;
1304                 }
1305         }
1306         
1307         return 1;
1308 }
1309
1310 static void wpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1311 {
1312         ToolSettings *ts= CTX_data_tool_settings(C);
1313         VPaint *wp= ts->wpaint;
1314         Brush *brush = paint_brush(&wp->paint);
1315         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1316         ViewContext *vc= &wpd->vc;
1317         Object *ob= vc->obact;
1318         Mesh *me= ob->data;
1319         float mat[4][4];
1320         float paintweight= ts->vgroup_weight;
1321         int *indexar= wpd->indexar;
1322         int totindex, index, alpha, totw;
1323         float mval[2];
1324
1325         view3d_operator_needs_opengl(C);
1326                         
1327         /* load projection matrix */
1328         wmMultMatrix(ob->obmat);
1329         wmGetSingleMatrix(mat);
1330         wmLoadMatrix(wpd->vc.rv3d->viewmat);
1331
1332         RNA_float_get_array(itemptr, "mouse", mval);
1333         mval[0]-= vc->ar->winrct.xmin;
1334         mval[1]-= vc->ar->winrct.ymin;
1335                         
1336         Mat4SwapMat4(wpd->vc.rv3d->persmat, mat);
1337                         
1338         /* which faces are involved */
1339         if(wp->flag & VP_AREA) {
1340                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush->size);
1341         }
1342         else {
1343                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1344                 if(indexar[0]) totindex= 1;
1345                 else totindex= 0;
1346         }
1347                         
1348         if(wp->flag & VP_COLINDEX) {
1349                 for(index=0; index<totindex; index++) {
1350                         if(indexar[index] && indexar[index]<=me->totface) {
1351                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1352                                                 
1353                                 if(mface->mat_nr!=ob->actcol-1) {
1354                                         indexar[index]= 0;
1355                                 }
1356                         }                                       
1357                 }
1358         }
1359                         
1360         if((me->editflag & ME_EDIT_PAINT_MASK) && me->mface) {
1361                 for(index=0; index<totindex; index++) {
1362                         if(indexar[index] && indexar[index]<=me->totface) {
1363                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1364                                                 
1365                                 if((mface->flag & ME_FACE_SEL)==0) {
1366                                         indexar[index]= 0;
1367                                 }
1368                         }                                       
1369                 }
1370         }
1371                         
1372         /* make sure each vertex gets treated only once */
1373         /* and calculate filter weight */
1374         totw= 0;
1375         if(wp->mode==VP_BLUR) 
1376                 paintweight= 0.0f;
1377         else
1378                 paintweight= ts->vgroup_weight;
1379                         
1380         for(index=0; index<totindex; index++) {
1381                 if(indexar[index] && indexar[index]<=me->totface) {
1382                         MFace *mface= me->mface + (indexar[index]-1);
1383                                         
1384                         (me->dvert+mface->v1)->flag= 1;
1385                         (me->dvert+mface->v2)->flag= 1;
1386                         (me->dvert+mface->v3)->flag= 1;
1387                         if(mface->v4) (me->dvert+mface->v4)->flag= 1;
1388                                         
1389                         if(wp->mode==VP_BLUR) {
1390                                 MDeformWeight *dw, *(*dw_func)(MDeformVert *, int) = ED_vgroup_weight_verify;
1391                                                 
1392                                 if(wp->flag & VP_ONLYVGROUP)
1393                                         dw_func= ED_vgroup_weight_get;
1394                                                 
1395                                 dw= dw_func(me->dvert+mface->v1, ob->actdef-1);
1396                                 if(dw) {paintweight+= dw->weight; totw++;}
1397                                 dw= dw_func(me->dvert+mface->v2, ob->actdef-1);
1398                                 if(dw) {paintweight+= dw->weight; totw++;}
1399                                 dw= dw_func(me->dvert+mface->v3, ob->actdef-1);
1400                                 if(dw) {paintweight+= dw->weight; totw++;}
1401                                 if(mface->v4) {
1402                                         dw= dw_func(me->dvert+mface->v4, ob->actdef-1);
1403                                         if(dw) {paintweight+= dw->weight; totw++;}
1404                                 }
1405                         }
1406                 }
1407         }
1408                         
1409         if(wp->mode==VP_BLUR) 
1410                 paintweight/= (float)totw;
1411                         
1412         for(index=0; index<totindex; index++) {
1413                                 
1414                 if(indexar[index] && indexar[index]<=me->totface) {
1415                         MFace *mface= me->mface + (indexar[index]-1);
1416                                         
1417                         if((me->dvert+mface->v1)->flag) {
1418                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v1, mval);
1419                                 if(alpha) {
1420                                         do_weight_paint_vertex(wp, ob, mface->v1, alpha, paintweight, wpd->vgroup_mirror);
1421                                 }
1422                                 (me->dvert+mface->v1)->flag= 0;
1423                         }
1424                                         
1425                         if((me->dvert+mface->v2)->flag) {
1426                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v2, mval);
1427                                 if(alpha) {
1428                                         do_weight_paint_vertex(wp, ob, mface->v2, alpha, paintweight, wpd->vgroup_mirror);
1429                                 }
1430                                 (me->dvert+mface->v2)->flag= 0;
1431                         }
1432                                         
1433                         if((me->dvert+mface->v3)->flag) {
1434                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v3, mval);
1435                                 if(alpha) {
1436                                         do_weight_paint_vertex(wp, ob, mface->v3, alpha, paintweight, wpd->vgroup_mirror);
1437                                 }
1438                                 (me->dvert+mface->v3)->flag= 0;
1439                         }
1440                                         
1441                         if((me->dvert+mface->v4)->flag) {
1442                                 if(mface->v4) {
1443                                         alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v4, mval);
1444                                         if(alpha) {
1445                                                 do_weight_paint_vertex(wp, ob, mface->v4, alpha, paintweight, wpd->vgroup_mirror);
1446                                         }
1447                                         (me->dvert+mface->v4)->flag= 0;
1448                                 }
1449                         }
1450                 }
1451         }
1452                         
1453         Mat4SwapMat4(vc->rv3d->persmat, mat);
1454                         
1455         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1456         ED_region_tag_redraw(vc->ar);
1457 }
1458
1459 static void wpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
1460 {
1461         ToolSettings *ts= CTX_data_tool_settings(C);
1462         Object *ob= CTX_data_active_object(C);
1463         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1464         
1465         if(wpd->vertexcosnos)
1466                 MEM_freeN(wpd->vertexcosnos);
1467         MEM_freeN(wpd->indexar);
1468         
1469         /* frees prev buffer */
1470         copy_wpaint_prev(ts->wpaint, NULL, 0);
1471         
1472         /* and particles too */
1473         if(ob->particlesystem.first) {
1474                 ParticleSystem *psys;
1475                 int i;
1476                 
1477                 for(psys= ob->particlesystem.first; psys; psys= psys->next) {
1478                         for(i=0; i<PSYS_TOT_VG; i++) {
1479                                 if(psys->vgroup[i]==ob->actdef) {
1480                                         psys->recalc |= PSYS_RECALC_RESET;
1481                                         break;
1482                                 }
1483                         }
1484                 }
1485         }
1486         
1487         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1488         
1489         MEM_freeN(wpd);
1490 }
1491
1492
1493 static int wpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1494 {
1495         
1496         op->customdata = paint_stroke_new(C, wpaint_stroke_test_start,
1497                                           wpaint_stroke_update_step,
1498                                           wpaint_stroke_done);
1499         
1500         /* add modal handler */
1501         WM_event_add_modal_handler(C, op);
1502
1503         op->type->modal(C, op, event);
1504         
1505         return OPERATOR_RUNNING_MODAL;
1506 }
1507
1508 void PAINT_OT_weight_paint(wmOperatorType *ot)
1509 {
1510         
1511         /* identifiers */
1512         ot->name= "Weight Paint";
1513         ot->idname= "PAINT_OT_weight_paint";
1514         
1515         /* api callbacks */
1516         ot->invoke= wpaint_invoke;
1517         ot->modal= paint_stroke_modal;
1518         /* ot->exec= vpaint_exec; <-- needs stroke property */
1519         ot->poll= weight_paint_poll;
1520         
1521         /* flags */
1522         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1523
1524         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1525 }
1526
1527 static int weight_paint_set_exec(bContext *C, wmOperator *op)
1528 {
1529         struct Scene *scene= CTX_data_scene(C);
1530         Object *obact = CTX_data_active_object(C);
1531
1532         wpaint_fill(scene->toolsettings->wpaint, obact, scene->toolsettings->vgroup_weight);
1533         ED_region_tag_redraw(CTX_wm_region(C)); // XXX - should redraw all 3D views
1534         return OPERATOR_FINISHED;
1535 }
1536
1537 void PAINT_OT_weight_set(wmOperatorType *ot)
1538 {
1539         /* identifiers */
1540         ot->name= "Weight Set";
1541         ot->idname= "PAINT_OT_weight_set";
1542
1543         /* api callbacks */
1544         ot->exec= weight_paint_set_exec;
1545         ot->poll= facemask_paint_poll;
1546
1547         /* flags */
1548         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1549
1550         RNA_def_boolean(ot->srna, "extend", 0, "Extend", "Extend selection instead of deselecting everything first.");
1551 }
1552
1553 /* ************ set / clear vertex paint mode ********** */
1554
1555
1556 static int set_vpaint(bContext *C, wmOperator *op)              /* toggle */
1557 {       
1558         Object *ob= CTX_data_active_object(C);
1559         Scene *scene= CTX_data_scene(C);
1560         VPaint *vp= scene->toolsettings->vpaint;
1561         Mesh *me;
1562         
1563         me= get_mesh(ob);
1564         
1565         if(me==NULL || object_data_is_libdata(ob)) {
1566                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1567                 return OPERATOR_PASS_THROUGH;
1568         }
1569         
1570         if(me && me->totface>=MAXINDEX) {
1571                 error("Maximum number of faces: %d", MAXINDEX-1);
1572                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1573                 return OPERATOR_FINISHED;
1574         }
1575         
1576         if(me && me->mcol==NULL) make_vertexcol(ob);
1577         
1578         /* toggle: end vpaint */
1579         if(ob->mode & OB_MODE_VERTEX_PAINT) {
1580                 
1581                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1582         }
1583         else {
1584                 ob->mode |= OB_MODE_VERTEX_PAINT;
1585                 /* Turn off weight painting */
1586                 if (ob->mode & OB_MODE_WEIGHT_PAINT)
1587                         set_wpaint(C, op);
1588                 
1589                 if(vp==NULL)
1590                         vp= scene->toolsettings->vpaint= new_vpaint(0);
1591                 
1592                 paint_cursor_start(C, vertex_paint_poll);
1593
1594                 paint_init(&vp->paint, PAINT_CURSOR_VERTEX_PAINT);
1595         }
1596         
1597         if (me)
1598                 /* update modifier stack for mapping requirements */
1599                 DAG_id_flush_update(&me->id, OB_RECALC_DATA);
1600         
1601         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1602         
1603         return OPERATOR_FINISHED;
1604 }
1605
1606 void PAINT_OT_vertex_paint_toggle(wmOperatorType *ot)
1607 {
1608         
1609         /* identifiers */
1610         ot->name= "Vertex Paint Mode";
1611         ot->idname= "PAINT_OT_vertex_paint_toggle";
1612         
1613         /* api callbacks */
1614         ot->exec= set_vpaint;
1615         ot->poll= paint_poll_test;
1616         
1617         /* flags */
1618         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1619         
1620 }
1621
1622
1623
1624 /* ********************** vertex paint operator ******************* */
1625
1626 /* Implementation notes:
1627
1628 Operator->invoke()
1629   - validate context (add mcol)
1630   - create customdata storage
1631   - call paint once (mouse click)
1632   - add modal handler 
1633
1634 Operator->modal()
1635   - for every mousemove, apply vertex paint
1636   - exit on mouse release, free customdata
1637     (return OPERATOR_FINISHED also removes handler and operator)
1638
1639 For future:
1640   - implement a stroke event (or mousemove with past positons)
1641   - revise whether op->customdata should be added in object, in set_vpaint
1642
1643 */
1644
1645 typedef struct VPaintData {
1646         ViewContext vc;
1647         unsigned int paintcol;
1648         int *indexar;
1649         float *vertexcosnos;
1650         float vpimat[3][3];
1651 } VPaintData;
1652
1653 static int vpaint_stroke_test_start(bContext *C, struct wmOperator *op, wmEvent *event)
1654 {
1655         ToolSettings *ts= CTX_data_tool_settings(C);
1656         struct PaintStroke *stroke = op->customdata;
1657         VPaint *vp= ts->vpaint;
1658         struct VPaintData *vpd;
1659         Object *ob= CTX_data_active_object(C);
1660         Mesh *me;
1661         float mat[4][4], imat[4][4];
1662
1663         /* context checks could be a poll() */
1664         me= get_mesh(ob);
1665         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1666         
1667         if(me->mcol==NULL) make_vertexcol(ob);
1668         if(me->mcol==NULL) return OPERATOR_CANCELLED;
1669         
1670         /* make mode data storage */
1671         vpd= MEM_callocN(sizeof(struct VPaintData), "VPaintData");
1672         paint_stroke_set_mode_data(stroke, vpd);
1673         view3d_set_viewcontext(C, &vpd->vc);
1674         
1675         vpd->vertexcosnos= mesh_get_mapped_verts_nors(vpd->vc.scene, ob);
1676         vpd->indexar= get_indexarray();
1677         vpd->paintcol= vpaint_get_current_col(vp);
1678         
1679         /* for filtering */
1680         copy_vpaint_prev(vp, (unsigned int *)me->mcol, me->totface);
1681         
1682         /* some old cruft to sort out later */
1683         Mat4MulMat4(mat, ob->obmat, vpd->vc.rv3d->viewmat);
1684         Mat4Invert(imat, mat);
1685         Mat3CpyMat4(vpd->vpimat, imat);
1686
1687         return 1;
1688 }
1689
1690 static void vpaint_paint_face(VPaint *vp, VPaintData *vpd, Object *ob, int index, float mval[2])
1691 {
1692         ViewContext *vc = &vpd->vc;
1693         Mesh *me = get_mesh(ob);
1694         MFace *mface= ((MFace*)me->mface) + index;
1695         unsigned int *mcol= ((unsigned int*)me->mcol) + 4*index;
1696         unsigned int *mcolorig= ((unsigned int*)vp->vpaint_prev) + 4*index;
1697         int alpha, i;
1698         
1699         if((vp->flag & VP_COLINDEX && mface->mat_nr!=ob->actcol-1) ||
1700            ((me->editflag & ME_EDIT_PAINT_MASK) && !(mface->flag & ME_FACE_SEL)))
1701                 return;
1702
1703         if(vp->mode==VP_BLUR) {
1704                 unsigned int fcol1= mcol_blend( mcol[0], mcol[1], 128);
1705                 if(mface->v4) {
1706                         unsigned int fcol2= mcol_blend( mcol[2], mcol[3], 128);
1707                         vpd->paintcol= mcol_blend( fcol1, fcol2, 128);
1708                 }
1709                 else {
1710                         vpd->paintcol= mcol_blend( mcol[2], fcol1, 170);
1711                 }
1712                 
1713         }
1714
1715         for(i = 0; i < (mface->v4 ? 4 : 3); ++i) {
1716                 alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*(&mface->v1)[i], mval);
1717                 if(alpha)
1718                         vpaint_blend(vp, mcol+i, mcolorig+i, vpd->paintcol, alpha);
1719         }
1720 }
1721
1722 static void vpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1723 {
1724         ToolSettings *ts= CTX_data_tool_settings(C);
1725         struct VPaintData *vpd = paint_stroke_mode_data(stroke);
1726         VPaint *vp= ts->vpaint;
1727         Brush *brush = paint_brush(&vp->paint);
1728         ViewContext *vc= &vpd->vc;
1729         Object *ob= vc->obact;
1730         Mesh *me= ob->data;
1731         float mat[4][4];
1732         int *indexar= vpd->indexar;
1733         int totindex, index;
1734         float mval[2];
1735
1736         RNA_float_get_array(itemptr, "mouse", mval);
1737                         
1738         view3d_operator_needs_opengl(C);
1739                         
1740         /* load projection matrix */
1741         wmMultMatrix(ob->obmat);
1742         wmGetSingleMatrix(mat);
1743         wmLoadMatrix(vc->rv3d->viewmat);
1744
1745         mval[0]-= vc->ar->winrct.xmin;
1746         mval[1]-= vc->ar->winrct.ymin;
1747
1748                         
1749         /* which faces are involved */
1750         if(vp->flag & VP_AREA) {
1751                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush->size);
1752         }
1753         else {
1754                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1755                 if(indexar[0]) totindex= 1;
1756                 else totindex= 0;
1757         }
1758                         
1759         Mat4SwapMat4(vc->rv3d->persmat, mat);
1760                         
1761         for(index=0; index<totindex; index++) {                         
1762                 if(indexar[index] && indexar[index]<=me->totface)
1763                         vpaint_paint_face(vp, vpd, ob, indexar[index]-1, mval);
1764         }
1765                                                 
1766         Mat4SwapMat4(vc->rv3d->persmat, mat);
1767
1768         /* was disabled because it is slow, but necessary for blur */
1769         if(vp->mode == VP_BLUR)
1770                 do_shared_vertexcol(me);
1771                         
1772         ED_region_tag_redraw(vc->ar);
1773                         
1774         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1775 }
1776
1777 static void vpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
1778 {
1779         ToolSettings *ts= CTX_data_tool_settings(C);
1780         struct VPaintData *vpd= paint_stroke_mode_data(stroke);
1781         
1782         if(vpd->vertexcosnos)
1783                 MEM_freeN(vpd->vertexcosnos);
1784         MEM_freeN(vpd->indexar);
1785         
1786         /* frees prev buffer */
1787         copy_vpaint_prev(ts->vpaint, NULL, 0);
1788         
1789         MEM_freeN(vpd);
1790 }
1791
1792 static int vpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1793 {
1794         
1795         op->customdata = paint_stroke_new(C, vpaint_stroke_test_start,
1796                                           vpaint_stroke_update_step,
1797                                           vpaint_stroke_done);
1798         
1799         /* add modal handler */
1800         WM_event_add_modal_handler(C, op);
1801
1802         op->type->modal(C, op, event);
1803         
1804         return OPERATOR_RUNNING_MODAL;
1805 }
1806
1807 void PAINT_OT_vertex_paint(wmOperatorType *ot)
1808 {
1809         /* identifiers */
1810         ot->name= "Vertex Paint";
1811         ot->idname= "PAINT_OT_vertex_paint";
1812         
1813         /* api callbacks */
1814         ot->invoke= vpaint_invoke;
1815         ot->modal= paint_stroke_modal;
1816         /* ot->exec= vpaint_exec; <-- needs stroke property */
1817         ot->poll= vertex_paint_poll;
1818         
1819         /* flags */
1820         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1821
1822         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1823 }
1824