I've begun treating unlocked misc vertex groups differently than unlocked bone groups-
[blender.git] / source / blender / editors / sculpt_paint / paint_vertex.c
1 /*
2  * $Id$
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19  *
20  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  */
29
30 /** \file blender/editors/sculpt_paint/paint_vertex.c
31  *  \ingroup edsculpt
32  */
33
34
35 #include <math.h>
36 #include <string.h>
37
38 #ifdef WIN32
39 #include <io.h>
40 #else
41 #include <unistd.h>
42 #endif   
43
44 #include "MEM_guardedalloc.h"
45
46 #include "BLI_blenlib.h"
47 #include "BLI_math.h"
48 #include "BLI_utildefines.h"
49 #include "BLI_ghash.h"
50
51 #include "IMB_imbuf.h"
52 #include "IMB_imbuf_types.h"
53
54 #include "DNA_armature_types.h"
55 #include "DNA_mesh_types.h"
56 #include "DNA_particle_types.h"
57 #include "DNA_scene_types.h"
58 #include "DNA_brush_types.h"
59 #include "DNA_object_types.h"
60 #include "DNA_meshdata_types.h"
61
62 #include "RNA_access.h"
63 #include "RNA_define.h"
64
65 #include "BKE_DerivedMesh.h"
66 #include "BKE_action.h"
67 #include "BKE_brush.h"
68 #include "BKE_context.h"
69 #include "BKE_depsgraph.h"
70 #include "BKE_deform.h"
71 #include "BKE_mesh.h"
72 #include "BKE_modifier.h"
73 #include "BKE_object.h"
74 #include "BKE_paint.h"
75
76
77 #include "WM_api.h"
78 #include "WM_types.h"
79
80
81 #include "ED_armature.h"
82 #include "ED_mesh.h"
83 #include "ED_screen.h"
84 #include "ED_view3d.h"
85
86 #include "paint_intern.h"
87
88 /* brush->vertexpaint_tool */
89 #define VP_MIX  0
90 #define VP_ADD  1
91 #define VP_SUB  2
92 #define VP_MUL  3
93 #define VP_BLUR 4
94 #define VP_LIGHTEN      5
95 #define VP_DARKEN       6
96
97 /* polling - retrieve whether cursor should be set or operator should be done */
98
99
100 /* Returns true if vertex paint mode is active */
101 int vertex_paint_mode_poll(bContext *C)
102 {
103         Object *ob = CTX_data_active_object(C);
104
105         return ob && ob->mode == OB_MODE_VERTEX_PAINT && ((Mesh *)ob->data)->totface;
106 }
107
108 int vertex_paint_poll(bContext *C)
109 {
110         if(vertex_paint_mode_poll(C) && 
111            paint_brush(&CTX_data_tool_settings(C)->vpaint->paint)) {
112                 ScrArea *sa= CTX_wm_area(C);
113                 if(sa->spacetype==SPACE_VIEW3D) {
114                         ARegion *ar= CTX_wm_region(C);
115                         if(ar->regiontype==RGN_TYPE_WINDOW)
116                                 return 1;
117                         }
118                 }
119         return 0;
120 }
121
122 int weight_paint_mode_poll(bContext *C)
123 {
124         Object *ob = CTX_data_active_object(C);
125
126         return ob && ob->mode == OB_MODE_WEIGHT_PAINT && ((Mesh *)ob->data)->totface;
127 }
128
129 int weight_paint_poll(bContext *C)
130 {
131         Object *ob= CTX_data_active_object(C);
132         ScrArea *sa;
133
134         if(     (ob != NULL) &&
135                 (ob->mode & OB_MODE_WEIGHT_PAINT) &&
136                 (paint_brush(&CTX_data_tool_settings(C)->wpaint->paint) != NULL) &&
137                 (sa= CTX_wm_area(C)) &&
138                 (sa->spacetype == SPACE_VIEW3D)
139         ) {
140                 ARegion *ar= CTX_wm_region(C);
141                 if(ar->regiontype==RGN_TYPE_WINDOW) {
142                         return 1;
143                 }
144         }
145         return 0;
146 }
147
148 static VPaint *new_vpaint(int wpaint)
149 {
150         VPaint *vp= MEM_callocN(sizeof(VPaint), "VPaint");
151         
152         vp->flag= VP_AREA+VP_SPRAY;
153         
154         if(wpaint)
155                 vp->flag= VP_AREA;
156
157         return vp;
158 }
159
160 static int *get_indexarray(Mesh *me)
161 {
162         return MEM_mallocN(sizeof(int)*(me->totface+1), "vertexpaint");
163 }
164
165
166 /* in contradiction to cpack drawing colors, the MCOL colors (vpaint colors) are per byte! 
167    so not endian sensitive. Mcol = ABGR!!! so be cautious with cpack calls */
168
169 static unsigned int rgba_to_mcol(float r, float g, float b, float a)
170 {
171         int ir, ig, ib, ia;
172         unsigned int col;
173         char *cp;
174         
175         ir= floor(255.0f * r);
176         if(ir<0) ir= 0; else if(ir>255) ir= 255;
177         ig= floor(255.0f * g);
178         if(ig<0) ig= 0; else if(ig>255) ig= 255;
179         ib= floor(255.0f * b);
180         if(ib<0) ib= 0; else if(ib>255) ib= 255;
181         ia= floor(255.0f * a);
182         if(ia<0) ia= 0; else if(ia>255) ia= 255;
183         
184         cp= (char *)&col;
185         cp[0]= ia;
186         cp[1]= ib;
187         cp[2]= ig;
188         cp[3]= ir;
189         
190         return col;
191         
192 }
193
194 unsigned int vpaint_get_current_col(VPaint *vp)
195 {
196         Brush *brush = paint_brush(&vp->paint);
197         return rgba_to_mcol(brush->rgb[0], brush->rgb[1], brush->rgb[2], 1.0f);
198 }
199
200 static void do_shared_vertexcol(Mesh *me)
201 {
202         /* if no mcol: do not do */
203         /* if tface: only the involved faces, otherwise all */
204         MFace *mface;
205         MTFace *tface;
206         int a;
207         short *scolmain, *scol;
208         char *mcol;
209         
210         if(me->mcol==NULL || me->totvert==0 || me->totface==0) return;
211         
212         scolmain= MEM_callocN(4*sizeof(short)*me->totvert, "colmain");
213         
214         tface= me->mtface;
215         mface= me->mface;
216         mcol= (char *)me->mcol;
217         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
218                 if((tface && tface->mode & TF_SHAREDCOL) || (me->editflag & ME_EDIT_PAINT_MASK)==0) {
219                         scol= scolmain+4*mface->v1;
220                         scol[0]++; scol[1]+= mcol[1]; scol[2]+= mcol[2]; scol[3]+= mcol[3];
221                         scol= scolmain+4*mface->v2;
222                         scol[0]++; scol[1]+= mcol[5]; scol[2]+= mcol[6]; scol[3]+= mcol[7];
223                         scol= scolmain+4*mface->v3;
224                         scol[0]++; scol[1]+= mcol[9]; scol[2]+= mcol[10]; scol[3]+= mcol[11];
225                         if(mface->v4) {
226                                 scol= scolmain+4*mface->v4;
227                                 scol[0]++; scol[1]+= mcol[13]; scol[2]+= mcol[14]; scol[3]+= mcol[15];
228                         }
229                 }
230                 if(tface) tface++;
231         }
232         
233         a= me->totvert;
234         scol= scolmain;
235         while(a--) {
236                 if(scol[0]>1) {
237                         scol[1]/= scol[0];
238                         scol[2]/= scol[0];
239                         scol[3]/= scol[0];
240                 }
241                 scol+= 4;
242         }
243         
244         tface= me->mtface;
245         mface= me->mface;
246         mcol= (char *)me->mcol;
247         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
248                 if((tface && tface->mode & TF_SHAREDCOL) || (me->editflag & ME_EDIT_PAINT_MASK)==0) {
249                         scol= scolmain+4*mface->v1;
250                         mcol[1]= scol[1]; mcol[2]= scol[2]; mcol[3]= scol[3];
251                         scol= scolmain+4*mface->v2;
252                         mcol[5]= scol[1]; mcol[6]= scol[2]; mcol[7]= scol[3];
253                         scol= scolmain+4*mface->v3;
254                         mcol[9]= scol[1]; mcol[10]= scol[2]; mcol[11]= scol[3];
255                         if(mface->v4) {
256                                 scol= scolmain+4*mface->v4;
257                                 mcol[13]= scol[1]; mcol[14]= scol[2]; mcol[15]= scol[3];
258                         }
259                 }
260                 if(tface) tface++;
261         }
262
263         MEM_freeN(scolmain);
264 }
265
266 static void make_vertexcol(Object *ob)  /* single ob */
267 {
268         Mesh *me;
269         if(!ob || ob->id.lib) return;
270         me= get_mesh(ob);
271         if(me==NULL) return;
272         if(me->edit_mesh) return;
273
274         /* copies from shadedisplist to mcol */
275         if(!me->mcol) {
276                 CustomData_add_layer(&me->fdata, CD_MCOL, CD_CALLOC, NULL, me->totface);
277                 mesh_update_customdata_pointers(me);
278         }
279
280         //if(shade)
281         //      shadeMeshMCol(scene, ob, me);
282         //else
283
284         memset(me->mcol, 255, 4*sizeof(MCol)*me->totface);
285         
286         DAG_id_tag_update(&me->id, 0);
287         
288 }
289
290 static void copy_vpaint_prev(VPaint *vp, unsigned int *mcol, int tot)
291 {
292         if(vp->vpaint_prev) {
293                 MEM_freeN(vp->vpaint_prev);
294                 vp->vpaint_prev= NULL;
295         }
296         vp->tot= tot;   
297         
298         if(mcol==NULL || tot==0) return;
299         
300         vp->vpaint_prev= MEM_mallocN(4*sizeof(int)*tot, "vpaint_prev");
301         memcpy(vp->vpaint_prev, mcol, 4*sizeof(int)*tot);
302         
303 }
304
305 static void copy_wpaint_prev (VPaint *wp, MDeformVert *dverts, int dcount)
306 {
307         if (wp->wpaint_prev) {
308                 free_dverts(wp->wpaint_prev, wp->tot);
309                 wp->wpaint_prev= NULL;
310         }
311         
312         if(dverts && dcount) {
313                 
314                 wp->wpaint_prev = MEM_mallocN (sizeof(MDeformVert)*dcount, "wpaint prev");
315                 wp->tot = dcount;
316                 copy_dverts (wp->wpaint_prev, dverts, dcount);
317         }
318 }
319
320
321 void vpaint_fill(Object *ob, unsigned int paintcol)
322 {
323         Mesh *me;
324         MFace *mf;
325         unsigned int *mcol;
326         int i, selected;
327
328         me= get_mesh(ob);
329         if(me==NULL || me->totface==0) return;
330
331         if(!me->mcol)
332                 make_vertexcol(ob);
333
334         selected= (me->editflag & ME_EDIT_PAINT_MASK);
335
336         mf = me->mface;
337         mcol = (unsigned int*)me->mcol;
338         for (i = 0; i < me->totface; i++, mf++, mcol+=4) {
339                 if (!selected || mf->flag & ME_FACE_SEL) {
340                         mcol[0] = paintcol;
341                         mcol[1] = paintcol;
342                         mcol[2] = paintcol;
343                         mcol[3] = paintcol;
344                 }
345         }
346         
347         DAG_id_tag_update(&me->id, 0);
348 }
349
350 /* fills in the selected faces with the current weight and vertex group */
351 void wpaint_fill(VPaint *wp, Object *ob, float paintweight)
352 {
353         Mesh *me;
354         MFace *mface;
355         MDeformWeight *dw, *uw;
356         int *indexar;
357         int index, vgroup;
358         unsigned int faceverts[5]={0,0,0,0,0};
359         unsigned char i;
360         int vgroup_mirror= -1;
361         int selected;
362         
363         me= ob->data;
364         if(me==NULL || me->totface==0 || me->dvert==NULL || !me->mface) return;
365
366         selected= (me->editflag & ME_EDIT_PAINT_MASK);
367
368         indexar= get_indexarray(me);
369
370         if(selected) {
371                 for(index=0, mface=me->mface; index<me->totface; index++, mface++) {
372                         if((mface->flag & ME_FACE_SEL)==0)
373                                 indexar[index]= 0;
374                         else
375                                 indexar[index]= index+1;
376                 }
377         }
378         else {
379                 for(index=0; index<me->totface; index++)
380                         indexar[index]= index+1;
381         }
382         
383         vgroup= ob->actdef-1;
384         
385         /* directly copied from weight_paint, should probaby split into a separate function */
386         /* if mirror painting, find the other group */          
387         if(me->editflag & ME_EDIT_MIRROR_X) {
388                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
389                 if(defgroup) {
390                         bDeformGroup *curdef;
391                         int actdef= 0;
392                         char name[32];
393
394                         flip_side_name(name, defgroup->name, FALSE);
395
396                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
397                                 if (!strcmp(curdef->name, name))
398                                         break;
399                         if(curdef==NULL) {
400                                 int olddef= ob->actdef; /* tsk, ED_vgroup_add sets the active defgroup */
401                                 curdef= ED_vgroup_add_name (ob, name);
402                                 ob->actdef= olddef;
403                         }
404                         
405                         if(curdef && curdef!=defgroup)
406                                 vgroup_mirror= actdef;
407                 }
408         }
409         /* end copy from weight_paint*/
410         
411         copy_wpaint_prev(wp, me->dvert, me->totvert);
412         
413         for(index=0; index<me->totface; index++) {
414                 if(indexar[index] && indexar[index]<=me->totface) {
415                         mface= me->mface + (indexar[index]-1);
416                         /* just so we can loop through the verts */
417                         faceverts[0]= mface->v1;
418                         faceverts[1]= mface->v2;
419                         faceverts[2]= mface->v3;
420                         faceverts[3]= mface->v4;
421                         for (i=0; i<3 || faceverts[i]; i++) {
422                                 if(!((me->dvert+faceverts[i])->flag)) {
423                                         dw= defvert_verify_index(me->dvert+faceverts[i], vgroup);
424                                         if(dw) {
425                                                 uw= defvert_verify_index(wp->wpaint_prev+faceverts[i], vgroup);
426                                                 uw->weight= dw->weight; /* set the undo weight */
427                                                 dw->weight= paintweight;
428                                                 
429                                                 if(me->editflag & ME_EDIT_MIRROR_X) {   /* x mirror painting */
430                                                         int j= mesh_get_x_mirror_vert(ob, faceverts[i]);
431                                                         if(j>=0) {
432                                                                 /* copy, not paint again */
433                                                                 if(vgroup_mirror != -1) {
434                                                                         dw= defvert_verify_index(me->dvert+j, vgroup_mirror);
435                                                                         uw= defvert_verify_index(wp->wpaint_prev+j, vgroup_mirror);
436                                                                 } else {
437                                                                         dw= defvert_verify_index(me->dvert+j, vgroup);
438                                                                         uw= defvert_verify_index(wp->wpaint_prev+j, vgroup);
439                                                                 }
440                                                                 uw->weight= dw->weight; /* set the undo weight */
441                                                                 dw->weight= paintweight;
442                                                         }
443                                                 }
444                                         }
445                                         (me->dvert+faceverts[i])->flag= 1;
446                                 }
447                         }
448                 }
449         }
450         
451         index=0;
452         while (index<me->totvert) {
453                 (me->dvert+index)->flag= 0;
454                 index++;
455         }
456         
457         MEM_freeN(indexar);
458         copy_wpaint_prev(wp, NULL, 0);
459
460         DAG_id_tag_update(&me->id, 0);
461 }
462
463 /* XXX: should be re-implemented as a vertex/weight paint 'color correct' operator
464  
465 void vpaint_dogamma(Scene *scene)
466 {
467         VPaint *vp= scene->toolsettings->vpaint;
468         Mesh *me;
469         Object *ob;
470         float igam, fac;
471         int a, temp;
472         unsigned char *cp, gamtab[256];
473
474         ob= OBACT;
475         me= get_mesh(ob);
476
477         if(!(ob->mode & OB_MODE_VERTEX_PAINT)) return;
478         if(me==0 || me->mcol==0 || me->totface==0) return;
479
480         igam= 1.0/vp->gamma;
481         for(a=0; a<256; a++) {
482                 
483                 fac= ((float)a)/255.0;
484                 fac= vp->mul*pow( fac, igam);
485                 
486                 temp= 255.9*fac;
487                 
488                 if(temp<=0) gamtab[a]= 0;
489                 else if(temp>=255) gamtab[a]= 255;
490                 else gamtab[a]= temp;
491         }
492
493         a= 4*me->totface;
494         cp= (unsigned char *)me->mcol;
495         while(a--) {
496                 
497                 cp[1]= gamtab[ cp[1] ];
498                 cp[2]= gamtab[ cp[2] ];
499                 cp[3]= gamtab[ cp[3] ];
500                 
501                 cp+= 4;
502         }
503 }
504  */
505
506 static unsigned int mcol_blend(unsigned int col1, unsigned int col2, int fac)
507 {
508         char *cp1, *cp2, *cp;
509         int mfac;
510         unsigned int col=0;
511         
512         if(fac==0) return col1;
513         if(fac>=255) return col2;
514
515         mfac= 255-fac;
516         
517         cp1= (char *)&col1;
518         cp2= (char *)&col2;
519         cp=  (char *)&col;
520         
521         cp[0]= 255;
522         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
523         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
524         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
525         
526         return col;
527 }
528
529 static unsigned int mcol_add(unsigned int col1, unsigned int col2, int fac)
530 {
531         char *cp1, *cp2, *cp;
532         int temp;
533         unsigned int col=0;
534         
535         if(fac==0) return col1;
536         
537         cp1= (char *)&col1;
538         cp2= (char *)&col2;
539         cp=  (char *)&col;
540         
541         cp[0]= 255;
542         temp= cp1[1] + ((fac*cp2[1])/255);
543         if(temp>254) cp[1]= 255; else cp[1]= temp;
544         temp= cp1[2] + ((fac*cp2[2])/255);
545         if(temp>254) cp[2]= 255; else cp[2]= temp;
546         temp= cp1[3] + ((fac*cp2[3])/255);
547         if(temp>254) cp[3]= 255; else cp[3]= temp;
548         
549         return col;
550 }
551
552 static unsigned int mcol_sub(unsigned int col1, unsigned int col2, int fac)
553 {
554         char *cp1, *cp2, *cp;
555         int temp;
556         unsigned int col=0;
557         
558         if(fac==0) return col1;
559         
560         cp1= (char *)&col1;
561         cp2= (char *)&col2;
562         cp=  (char *)&col;
563         
564         cp[0]= 255;
565         temp= cp1[1] - ((fac*cp2[1])/255);
566         if(temp<0) cp[1]= 0; else cp[1]= temp;
567         temp= cp1[2] - ((fac*cp2[2])/255);
568         if(temp<0) cp[2]= 0; else cp[2]= temp;
569         temp= cp1[3] - ((fac*cp2[3])/255);
570         if(temp<0) cp[3]= 0; else cp[3]= temp;
571         
572         return col;
573 }
574
575 static unsigned int mcol_mul(unsigned int col1, unsigned int col2, int fac)
576 {
577         char *cp1, *cp2, *cp;
578         int mfac;
579         unsigned int col=0;
580         
581         if(fac==0) return col1;
582
583         mfac= 255-fac;
584         
585         cp1= (char *)&col1;
586         cp2= (char *)&col2;
587         cp=  (char *)&col;
588         
589         /* first mul, then blend the fac */
590         cp[0]= 255;
591         cp[1]= (mfac*cp1[1] + fac*((cp2[1]*cp1[1])/255)  )/255;
592         cp[2]= (mfac*cp1[2] + fac*((cp2[2]*cp1[2])/255)  )/255;
593         cp[3]= (mfac*cp1[3] + fac*((cp2[3]*cp1[3])/255)  )/255;
594
595         
596         return col;
597 }
598
599 static unsigned int mcol_lighten(unsigned int col1, unsigned int col2, int fac)
600 {
601         char *cp1, *cp2, *cp;
602         int mfac;
603         unsigned int col=0;
604         
605         if(fac==0) return col1;
606         if(fac>=255) return col2;
607
608         mfac= 255-fac;
609         
610         cp1= (char *)&col1;
611         cp2= (char *)&col2;
612         cp=  (char *)&col;
613         
614         /* See if are lighter, if so mix, else dont do anything.
615         if the paint col is darker then the original, then ignore */
616         if (cp1[1]+cp1[2]+cp1[3] > cp2[1]+cp2[2]+cp2[3])
617                 return col1;
618         
619         cp[0]= 255;
620         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
621         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
622         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
623         
624         return col;
625 }
626
627 static unsigned int mcol_darken(unsigned int col1, unsigned int col2, int fac)
628 {
629         char *cp1, *cp2, *cp;
630         int mfac;
631         unsigned int col=0;
632         
633         if(fac==0) return col1;
634         if(fac>=255) return col2;
635
636         mfac= 255-fac;
637         
638         cp1= (char *)&col1;
639         cp2= (char *)&col2;
640         cp=  (char *)&col;
641         
642         /* See if were darker, if so mix, else dont do anything.
643         if the paint col is brighter then the original, then ignore */
644         if (cp1[1]+cp1[2]+cp1[3] < cp2[1]+cp2[2]+cp2[3])
645                 return col1;
646         
647         cp[0]= 255;
648         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
649         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
650         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
651         return col;
652 }
653
654 static void vpaint_blend(VPaint *vp, unsigned int *col, unsigned int *colorig, unsigned int paintcol, int alpha)
655 {
656         Brush *brush = paint_brush(&vp->paint);
657
658         if(brush->vertexpaint_tool==VP_MIX || brush->vertexpaint_tool==VP_BLUR) *col= mcol_blend( *col, paintcol, alpha);
659         else if(brush->vertexpaint_tool==VP_ADD) *col= mcol_add( *col, paintcol, alpha);
660         else if(brush->vertexpaint_tool==VP_SUB) *col= mcol_sub( *col, paintcol, alpha);
661         else if(brush->vertexpaint_tool==VP_MUL) *col= mcol_mul( *col, paintcol, alpha);
662         else if(brush->vertexpaint_tool==VP_LIGHTEN) *col= mcol_lighten( *col, paintcol, alpha);
663         else if(brush->vertexpaint_tool==VP_DARKEN) *col= mcol_darken( *col, paintcol, alpha);
664         
665         /* if no spray, clip color adding with colorig & orig alpha */
666         if((vp->flag & VP_SPRAY)==0) {
667                 unsigned int testcol=0, a;
668                 char *cp, *ct, *co;
669                 
670                 alpha= (int)(255.0f*brush_alpha(brush));
671                 
672                 if(brush->vertexpaint_tool==VP_MIX || brush->vertexpaint_tool==VP_BLUR) testcol= mcol_blend( *colorig, paintcol, alpha);
673                 else if(brush->vertexpaint_tool==VP_ADD) testcol= mcol_add( *colorig, paintcol, alpha);
674                 else if(brush->vertexpaint_tool==VP_SUB) testcol= mcol_sub( *colorig, paintcol, alpha);
675                 else if(brush->vertexpaint_tool==VP_MUL) testcol= mcol_mul( *colorig, paintcol, alpha);
676                 else if(brush->vertexpaint_tool==VP_LIGHTEN)  testcol= mcol_lighten( *colorig, paintcol, alpha);
677                 else if(brush->vertexpaint_tool==VP_DARKEN)   testcol= mcol_darken( *colorig, paintcol, alpha);
678                 
679                 cp= (char *)col;
680                 ct= (char *)&testcol;
681                 co= (char *)colorig;
682                 
683                 for(a=0; a<4; a++) {
684                         if( ct[a]<co[a] ) {
685                                 if( cp[a]<ct[a] ) cp[a]= ct[a];
686                                 else if( cp[a]>co[a] ) cp[a]= co[a];
687                         }
688                         else {
689                                 if( cp[a]<co[a] ) cp[a]= co[a];
690                                 else if( cp[a]>ct[a] ) cp[a]= ct[a];
691                         }
692                 }
693         }
694 }
695
696
697 static int sample_backbuf_area(ViewContext *vc, int *indexar, int totface, int x, int y, float size)
698 {
699         struct ImBuf *ibuf;
700         int a, tot=0, index;
701         
702         /* brecht: disabled this because it obviously failes for
703            brushes with size > 64, why is this here? */
704         /*if(size>64.0) size= 64.0;*/
705         
706         ibuf= view3d_read_backbuf(vc, x-size, y-size, x+size, y+size);
707         if(ibuf) {
708                 unsigned int *rt= ibuf->rect;
709
710                 memset(indexar, 0, sizeof(int)*(totface+1));
711                 
712                 size= ibuf->x*ibuf->y;
713                 while(size--) {
714                                 
715                         if(*rt) {
716                                 index= WM_framebuffer_to_index(*rt);
717                                 if(index>0 && index<=totface)
718                                         indexar[index] = 1;
719                         }
720                 
721                         rt++;
722                 }
723                 
724                 for(a=1; a<=totface; a++) {
725                         if(indexar[a]) indexar[tot++]= a;
726                 }
727
728                 IMB_freeImBuf(ibuf);
729         }
730         
731         return tot;
732 }
733
734 static float calc_vp_alpha_dl(VPaint *vp, ViewContext *vc, float vpimat[][3], float *vert_nor, const float mval[2], float pressure)
735 {
736         Brush *brush = paint_brush(&vp->paint);
737         float fac, fac_2, size, dx, dy;
738         float alpha;
739         int vertco[2];
740         const int radius= brush_size(brush);
741
742         project_int_noclip(vc->ar, vert_nor, vertco);
743         dx= mval[0]-vertco[0];
744         dy= mval[1]-vertco[1];
745         
746         if (brush_use_size_pressure(brush))
747                 size = pressure * radius;
748         else
749                 size = radius;
750         
751         fac_2= dx*dx + dy*dy;
752         if(fac_2 > size*size) return 0.f;
753         fac = sqrtf(fac_2);
754         
755         alpha= brush_alpha(brush) * brush_curve_strength_clamp(brush, fac, size);
756         
757         if (brush_use_alpha_pressure(brush))
758                 alpha *= pressure;
759                 
760         if(vp->flag & VP_NORMALS) {
761                 float *no= vert_nor+3;
762                 
763                 /* transpose ! */
764                 fac= vpimat[2][0]*no[0]+vpimat[2][1]*no[1]+vpimat[2][2]*no[2];
765                 if(fac > 0.0f) {
766                         dx= vpimat[0][0]*no[0]+vpimat[0][1]*no[1]+vpimat[0][2]*no[2];
767                         dy= vpimat[1][0]*no[0]+vpimat[1][1]*no[1]+vpimat[1][2]*no[2];
768                         
769                         alpha*= fac/sqrtf(dx*dx + dy*dy + fac*fac);
770                 }
771                 else return 0.f;
772         }
773         
774         return alpha;
775 }
776
777 static void wpaint_blend(VPaint *wp, MDeformWeight *dw, MDeformWeight *uw, float alpha, float paintval, int flip)
778 {
779         Brush *brush = paint_brush(&wp->paint);
780         int tool = brush->vertexpaint_tool;
781         
782         if(dw==NULL || uw==NULL) return;
783         
784         if (flip) {
785                 switch(tool) {
786                         case VP_MIX:
787                                 paintval = 1.f - paintval; break;
788                         case VP_ADD:
789                                 tool= VP_SUB; break;
790                         case VP_SUB:
791                                 tool= VP_ADD; break;
792                         case VP_LIGHTEN:
793                                 tool= VP_DARKEN; break;
794                         case VP_DARKEN:
795                                 tool= VP_LIGHTEN; break;
796                 }
797         }
798         
799         if(tool==VP_MIX || tool==VP_BLUR)
800                 dw->weight = paintval*alpha + dw->weight*(1.0f-alpha);
801         else if(tool==VP_ADD)
802                 dw->weight += paintval*alpha;
803         else if(tool==VP_SUB) 
804                 dw->weight -= paintval*alpha;
805         else if(tool==VP_MUL) 
806                 /* first mul, then blend the fac */
807                 dw->weight = ((1.0f-alpha) + alpha*paintval)*dw->weight;
808         else if(tool==VP_LIGHTEN) {
809                 if (dw->weight < paintval)
810                         dw->weight = paintval*alpha + dw->weight*(1.0f-alpha);
811         } else if(tool==VP_DARKEN) {
812                 if (dw->weight > paintval)
813                         dw->weight = paintval*alpha + dw->weight*(1.0f-alpha);
814         }
815         CLAMP(dw->weight, 0.0f, 1.0f);
816         
817         /* if no spray, clip result with orig weight & orig alpha */
818         if((wp->flag & VP_SPRAY)==0) {
819                 float testw=0.0f;
820                 
821                 alpha= brush_alpha(brush);
822                 if(tool==VP_MIX || tool==VP_BLUR)
823                         testw = paintval*alpha + uw->weight*(1.0f-alpha);
824                 else if(tool==VP_ADD)
825                         testw = uw->weight + paintval*alpha;
826                 else if(tool==VP_SUB) 
827                         testw = uw->weight - paintval*alpha;
828                 else if(tool==VP_MUL) 
829                         /* first mul, then blend the fac */
830                         testw = ((1.0f-alpha) + alpha*paintval)*uw->weight;
831                 else if(tool==VP_LIGHTEN) {
832                         if (uw->weight < paintval)
833                                 testw = paintval*alpha + uw->weight*(1.0f-alpha);
834                         else
835                                 testw = uw->weight;
836                 } else if(tool==VP_DARKEN) {
837                         if (uw->weight > paintval)
838                                 testw = paintval*alpha + uw->weight*(1.0f-alpha);
839                         else
840                                 testw = uw->weight;
841                 }
842                 CLAMP(testw, 0.0f, 1.0f);
843                 
844                 if( testw<uw->weight ) {
845                         if(dw->weight < testw) dw->weight= testw;
846                         else if(dw->weight > uw->weight) dw->weight= uw->weight;
847                 }
848                 else {
849                         if(dw->weight > testw) dw->weight= testw;
850                         else if(dw->weight < uw->weight) dw->weight= uw->weight;
851                 }
852         }
853         
854 }
855
856 /* ----------------------------------------------------- */
857
858 /* used for 3d view, on active object, assumes me->dvert exists */
859 /* if mode==1: */
860 /*     samples cursor location, and gives menu with vertex groups to activate */
861 /* else */
862 /*     sets wp->weight to the closest weight value to vertex */
863 /*     note: we cant sample frontbuf, weight colors are interpolated too unpredictable */
864 static void sample_wpaint(Scene *scene, ARegion *ar, View3D *UNUSED(v3d), int mode)
865 {
866         ViewContext vc;
867         ToolSettings *ts= scene->toolsettings;
868         Object *ob= OBACT;
869         Mesh *me= get_mesh(ob);
870         int index;
871         int mval[2] = {0, 0}, sco[2];
872         int vgroup= ob->actdef-1;
873
874         if (!me) return;
875         
876 //      getmouseco_areawin(mval);
877         index= view3d_sample_backbuf(&vc, mval[0], mval[1]);
878         
879         if(index && index<=me->totface) {
880                 MFace *mface;
881                 
882                 mface= ((MFace *)me->mface) + index-1;
883                 
884                 if(mode==1) {   /* sampe which groups are in here */
885                         MDeformVert *dv;
886                         int a, totgroup;
887                         
888                         totgroup= BLI_countlist(&ob->defbase);
889                         if(totgroup) {
890                                 int totmenu=0;
891                                 int *groups=MEM_callocN(totgroup*sizeof(int), "groups");
892                                 
893                                 dv= me->dvert+mface->v1;
894                                 for(a=0; a<dv->totweight; a++) {
895                                         if (dv->dw[a].def_nr<totgroup)
896                                                 groups[dv->dw[a].def_nr]= 1;
897                                 }
898                                 dv= me->dvert+mface->v2;
899                                 for(a=0; a<dv->totweight; a++) {
900                                         if (dv->dw[a].def_nr<totgroup)
901                                                 groups[dv->dw[a].def_nr]= 1;
902                                 }
903                                 dv= me->dvert+mface->v3;
904                                 for(a=0; a<dv->totweight; a++) {
905                                         if (dv->dw[a].def_nr<totgroup)
906                                                 groups[dv->dw[a].def_nr]= 1;
907                                 }
908                                 if(mface->v4) {
909                                         dv= me->dvert+mface->v4;
910                                         for(a=0; a<dv->totweight; a++) {
911                                                 if (dv->dw[a].def_nr<totgroup)
912                                                         groups[dv->dw[a].def_nr]= 1;
913                                         }
914                                 }
915                                 for(a=0; a<totgroup; a++)
916                                         if(groups[a]) totmenu++;
917                                 
918                                 if(totmenu==0) {
919                                         //notice("No Vertex Group Selected");
920                                 }
921                                 else {
922                                         bDeformGroup *dg;
923                                         short val;
924                                         char item[40], *str= MEM_mallocN(40*totmenu+40, "menu");
925                                         
926                                         strcpy(str, "Vertex Groups %t");
927                                         for(a=0, dg=ob->defbase.first; dg && a<totgroup; a++, dg= dg->next) {
928                                                 if(groups[a]) {
929                                                         sprintf(item, "|%s %%x%d", dg->name, a);
930                                                         strcat(str, item);
931                                                 }
932                                         }
933                                         
934                                         val= 0; // XXX pupmenu(str);
935                                         if(val>=0) {
936                                                 ob->actdef= val+1;
937                                                 DAG_id_tag_update(&me->id, 0);
938                                         }
939                                         MEM_freeN(str);
940                                 }
941                                 MEM_freeN(groups);
942                         }
943 //                      else notice("No Vertex Groups in Object");
944                 }
945                 else {
946                         DerivedMesh *dm;
947                         float w1, w2, w3, w4, co[3], fac;
948                         
949                         dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
950                         if(dm->getVertCo==NULL) {
951                                 //notice("Not supported yet");
952                         }
953                         else {
954                                 /* calc 3 or 4 corner weights */
955                                 dm->getVertCo(dm, mface->v1, co);
956                                 project_int_noclip(ar, co, sco);
957                                 w1= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
958                                 
959                                 dm->getVertCo(dm, mface->v2, co);
960                                 project_int_noclip(ar, co, sco);
961                                 w2= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
962                                 
963                                 dm->getVertCo(dm, mface->v3, co);
964                                 project_int_noclip(ar, co, sco);
965                                 w3= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
966                                 
967                                 if(mface->v4) {
968                                         dm->getVertCo(dm, mface->v4, co);
969                                         project_int_noclip(ar, co, sco);
970                                         w4= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
971                                 }
972                                 else w4= 1.0e10;
973                                 
974                                 fac= MIN4(w1, w2, w3, w4);
975                                 if(w1==fac) {
976                                         ts->vgroup_weight= defvert_find_weight(me->dvert+mface->v1, vgroup);
977                                 }
978                                 else if(w2==fac) {
979                                         ts->vgroup_weight= defvert_find_weight(me->dvert+mface->v2, vgroup);
980                                 }
981                                 else if(w3==fac) {
982                                         ts->vgroup_weight= defvert_find_weight(me->dvert+mface->v3, vgroup);
983                                 }
984                                 else if(w4==fac) {
985                                         if(mface->v4) {
986                                                 ts->vgroup_weight= defvert_find_weight(me->dvert+mface->v4, vgroup);
987                                         }
988                                 }
989                         }
990                         dm->release(dm);
991                 }               
992                 
993         }
994         
995 }
996
997 static void do_weight_paint_auto_normalize(MDeformVert *dvert, 
998                                            int paint_nr, char *map)
999 {
1000 //      MDeformWeight *dw = dvert->dw;
1001         float sum=0.0f, fac=0.0f, paintw=0.0f;
1002         int i, tot=0;
1003
1004         if (!map)
1005                 return;
1006
1007         for (i=0; i<dvert->totweight; i++) {
1008                 if (dvert->dw[i].def_nr == paint_nr)
1009                         paintw = dvert->dw[i].weight;
1010
1011                 if (map[dvert->dw[i].def_nr]) {
1012                         tot += 1;
1013                         if (dvert->dw[i].def_nr != paint_nr)
1014                                 sum += dvert->dw[i].weight;
1015                 }
1016         }
1017         
1018         if (!tot || sum <= (1.0f - paintw))
1019                 return;
1020
1021         fac = sum / (1.0f - paintw);
1022         fac = fac==0.0f ? 1.0f : 1.0f / fac;
1023
1024         for (i=0; i<dvert->totweight; i++) {
1025                 if (map[dvert->dw[i].def_nr]) {
1026                         if (dvert->dw[i].def_nr != paint_nr)
1027                                 dvert->dw[i].weight *= fac;
1028                 }
1029         }
1030 }
1031 /* Jason was here 
1032 this function will handle normalize with locked groups 
1033 it assumes that the current ratios (of locked groups)
1034 are the intended ratios.
1035
1036 also, it does not attempt to force them to add up to 1, that would destroy intergroup weight ratios,
1037 it simply makes the highest weight sum add up to one
1038
1039 the Mesh is needed to change the ratios across the group
1040
1041 I need to resolve a precision error issue, however:
1042 dividing can cause the weights to drop to 0
1043 */
1044 static void do_wp_auto_normalize_locked_groups(Mesh *me, MDeformVert *dvert, char* map)
1045 {
1046         float highestSum = 0.0f;
1047         float currentSum;
1048         int cnt;
1049         int i, k;
1050         int totvert = me->totvert;
1051         MDeformVert dv;
1052
1053         if(!map) {
1054                 return;
1055         }
1056
1057         for(i = 0; i < totvert; i++) {
1058                 dv = dvert[i];
1059                 cnt = dv.totweight;
1060                 currentSum = 0.0f;
1061                 for(k = 0; k < cnt; k++) {
1062                         if(map[dv.dw->def_nr]) {
1063                                 //printf("group %d considered\n", dv.dw->def_nr);
1064                                 currentSum += (dv.dw+k)->weight;
1065                         }
1066                 }
1067                 if(highestSum < currentSum) {
1068                         highestSum = currentSum;
1069                 }
1070         }
1071         if(highestSum == 1.0f) {
1072                 return;
1073         }
1074         for(i = 0; i < totvert; i++) {
1075                 dv = dvert[i];
1076                 cnt = dv.totweight;
1077
1078                 for(k = 0; k < cnt; k++) {
1079                         if(map[dv.dw->def_nr]) {
1080                                 (dv.dw+k)->weight /= highestSum;
1081                         }
1082                 }
1083         }
1084 }
1085 /* Jason was here */
1086 /*static char get_locked_flag(Object *ob, int vgroup)
1087 {
1088         int i;
1089         bDeformGroup *defgroup = ob->defbase.first;
1090         for(i = 0; i < vgroup && defgroup; i++) {
1091                 defgroup = defgroup->next;
1092         }
1093         if(defgroup) {
1094                 return defgroup->flag;
1095         }
1096         return 0;
1097 }*/
1098 /* Jason was here */
1099 static int locked_group_exists(Object *ob)
1100 {
1101         bDeformGroup *defgroup = ob->defbase.first;
1102         while(defgroup) {
1103                 if(defgroup->flag){
1104                         return TRUE;
1105                 }
1106                 defgroup = defgroup->next;
1107         }
1108         return FALSE;
1109 }
1110 /* Jason was here */
1111 /*
1112 See if the current deform group has a locked group
1113 */
1114 static char has_locked_group(MDeformVert *dvert, char *flags)
1115 {
1116         int i;
1117         for(i = 0; i < dvert->totweight; i++) {
1118                 if(flags[(dvert->dw+i)->def_nr] && (dvert->dw+i)->weight > 0.0f) {
1119                         return TRUE;
1120                 }
1121         }
1122         return FALSE;
1123 }
1124 /*Jason was here
1125 not sure where the prototypes belong at the moment
1126 static char* gen_lck_flags(Object* ob);
1127
1128 gen_lck_flags gets the status of "flag" for each bDeformGroup
1129 in ob->defbase and returns an array containing them
1130 */
1131 static char* gen_lck_flags(Object* ob, int defcnt)
1132 {
1133         char is_locked = FALSE;
1134         int i;
1135         //int defcnt = BLI_countlist(&ob->defbase);
1136         char *flags = MEM_mallocN(defcnt*sizeof(char), "defflags");
1137         bDeformGroup *defgroup = ob->defbase.first;
1138         for(i = 0; i < defcnt && defgroup; i++) {
1139                 flags[i] = defgroup->flag;
1140                 defgroup = defgroup->next;
1141                 if(flags[i]) {
1142                         is_locked = TRUE;
1143                 }
1144         }
1145         if(is_locked){
1146                 return flags;
1147         }
1148         // don't forget to free it
1149         MEM_freeN(flags);
1150         return NULL;
1151 }
1152 /*Jason was here
1153 this alters the weights in order to maintain the ratios to match with the change in weights of pnt_dw
1154 This was not the intended solution evidently
1155 */
1156 /*static void fix_weight_ratios(Mesh *me, MDeformWeight *pnt_dw, float oldw)
1157 {
1158         int i, k, totvert, cnt;
1159         float scaledown = 1.0f;
1160         float neww = pnt_dw->weight;
1161         int defgroup = pnt_dw->def_nr;
1162         MDeformVert *dvert;
1163         MDeformVert dv;
1164         MDeformWeight *dw;
1165         totvert = me->totvert;
1166         pnt_dw->weight = oldw;
1167
1168         if(oldw == 0 || neww == 0){
1169                 return;
1170         }
1171         dvert = me->dvert;
1172         for(i = 0; i < totvert; i++) {
1173                 dv = dvert[i];
1174                 cnt = dv.totweight;
1175                 for(k = 0; k < cnt; k++) {
1176                         dw = dv.dw+k;
1177                         if(dw->def_nr == defgroup){
1178                                 dw->weight = neww * (dw->weight / oldw);
1179                                 if(dw->weight > scaledown){
1180                                         scaledown = dw->weight;
1181                                 }
1182                                 break;
1183                         }
1184                 }
1185         }
1186         if(scaledown > 1.0f) {
1187                 for(i = 0; i < totvert; i++) {
1188                         dv = dvert[i];
1189                         cnt = dv.totweight;
1190                         for(k = 0; k < cnt; k++) {
1191                                 dw = dv.dw+k;
1192                                 if(dw->def_nr == defgroup){
1193                                         dw->weight /= scaledown;
1194                                         break;
1195                                 }
1196                         }
1197                 }
1198         }
1199 }*/
1200 /* Jason was here */
1201 /*
1202 The idea behind this function is to get the difference in weight for pnt_dw,
1203 and to redistribute that weight to the unlocked groups
1204 (if it has to, then it will put some/all of the change
1205 back onto the original group)
1206 */
1207 static void redistribute_weight_change(MDeformVert *dvert, MDeformWeight *pnt_dw, float oldw, char* flags, int defcnt, char *map)
1208 {
1209         int i;
1210         float change_left = oldw - pnt_dw->weight;
1211         float change;
1212         char was_a_change;
1213         int groups_left_that_can_change = 0;
1214         char* change_status = MEM_mallocN(defcnt*sizeof(char), "defflags");
1215         MDeformWeight *dw;
1216         //printf("start\n");
1217         for(i = 0; i < defcnt; i++) {
1218                 if(pnt_dw->def_nr == i || !map[i]) {
1219                         change_status[i] = FALSE;
1220                 } else {
1221                         change_status[i] = !flags[i];
1222                 }
1223                 if(change_status[i]) {
1224                         groups_left_that_can_change++;
1225                 }
1226                 //printf("group %d, change status: %d flag: %d active?: %d\n", i, change_status[i], flags[i], pnt_dw->def_nr == i);
1227         }
1228         //printf("\n");
1229         if(groups_left_that_can_change > 0) {
1230                 do {
1231                         was_a_change = FALSE;
1232                         for(i = 0; i < dvert->totweight; i++) {
1233                                 dw = (dvert->dw+i);
1234                                 if(!change_status[dw->def_nr]) {
1235                                         continue;
1236                                 }
1237
1238                                 change = change_left/groups_left_that_can_change;
1239
1240                                 dw->weight += change;
1241                                 change_left -= change;
1242                                 //printf("group %d, change: %f weight: %f groups left: %d\n", dw->def_nr, change, dw->weight, groups_left_that_can_change);
1243
1244                                 if(dw->weight >= 1.0f) {
1245
1246                                         change_left += dw->weight-1.0f;
1247                                         dw->weight = 1.0f;
1248                                         groups_left_that_can_change--;
1249                                         change_status[dw->def_nr] = FALSE;
1250
1251                                 }else if(dw->weight <= 0.0f) {
1252
1253                                         change_left += dw->weight;
1254                                         dw->weight = 0.0f;
1255                                         groups_left_that_can_change--;
1256                                         change_status[dw->def_nr] = FALSE;
1257                                 }
1258                                 was_a_change = TRUE;
1259                         }
1260                 } while(groups_left_that_can_change > 0 && change_left != 0.0f && was_a_change);
1261         }
1262         // add any remaining change back to the original weight
1263         pnt_dw->weight += change_left;
1264         MEM_freeN(change_status);
1265         //printf("done\n");
1266 }
1267 /* Jason */
1268 static void check_locks_and_normalize(Mesh *me, int index, int vgroup, MDeformWeight *dw, float oldw, char *validmap, char *flags, int defcnt, char *bone_groups)
1269 {
1270         if(flags && has_locked_group(me->dvert+index, flags)) {
1271                 if(flags[dw->def_nr]) {
1272                         // cannot change locked groups!
1273                         dw->weight = oldw;
1274                 } else if(bone_groups[dw->def_nr]) {
1275                         redistribute_weight_change(me->dvert+index, dw, oldw, flags, defcnt, bone_groups);
1276                         do_wp_auto_normalize_locked_groups(me, me->dvert, validmap);
1277                 }
1278         } else {
1279                 do_weight_paint_auto_normalize(me->dvert+index, vgroup, validmap);
1280         }
1281 }
1282 // Jason
1283 static char *wpaint_make_validmap(Object *ob);
1284
1285 static void do_weight_paint_vertex(VPaint *wp, Object *ob, int index, 
1286                                    float alpha, float paintweight, int flip, 
1287                                    int vgroup_mirror, char *validmap)
1288 {
1289         Mesh *me= ob->data;
1290         MDeformWeight *dw, *uw;
1291         int vgroup= ob->actdef-1;
1292         
1293         /* Jason was here */
1294         char* flags;
1295         char* bone_groups;
1296         float oldw;
1297         int defcnt;
1298         if(validmap) {
1299                 bone_groups = validmap;
1300         }else {
1301                 bone_groups = wpaint_make_validmap(ob);
1302         }
1303
1304         if(wp->flag & VP_ONLYVGROUP) {
1305                 dw= defvert_find_index(me->dvert+index, vgroup);
1306                 uw= defvert_find_index(wp->wpaint_prev+index, vgroup);
1307         }
1308         else {
1309                 dw= defvert_verify_index(me->dvert+index, vgroup);
1310                 uw= defvert_verify_index(wp->wpaint_prev+index, vgroup);
1311         }
1312         if(dw==NULL || uw==NULL)
1313                 return;
1314         /* Jason was here */
1315         flags = gen_lck_flags(ob, defcnt = BLI_countlist(&ob->defbase));
1316         oldw = dw->weight;
1317
1318         wpaint_blend(wp, dw, uw, alpha, paintweight, flip);
1319
1320         /* Jason was here */
1321         check_locks_and_normalize(me, index, vgroup, dw, oldw, validmap, flags, defcnt, bone_groups);
1322         if(me->editflag & ME_EDIT_MIRROR_X) {   /* x mirror painting */
1323                 int j= mesh_get_x_mirror_vert(ob, index);
1324                 if(j>=0) {
1325                         /* copy, not paint again */
1326                         if(vgroup_mirror != -1)
1327                                 uw= defvert_verify_index(me->dvert+j, vgroup_mirror);
1328                         else
1329                                 uw= defvert_verify_index(me->dvert+j, vgroup);
1330                         /* Jason */
1331                         oldw = uw->weight;
1332
1333                         uw->weight= dw->weight;
1334                         /* Jason */
1335                         check_locks_and_normalize(me, j, vgroup, dw, oldw, validmap, flags, defcnt, bone_groups);
1336                 }
1337         }
1338         /* Jason */
1339         if(flags) {
1340                 MEM_freeN(flags);
1341         }
1342         if(!validmap) {
1343                 MEM_freeN(bone_groups);
1344         }
1345 }
1346
1347
1348 /* *************** set wpaint operator ****************** */
1349
1350 static int set_wpaint(bContext *C, wmOperator *UNUSED(op))              /* toggle */
1351 {               
1352         Object *ob= CTX_data_active_object(C);
1353         Scene *scene= CTX_data_scene(C);
1354         VPaint *wp= scene->toolsettings->wpaint;
1355         Mesh *me;
1356         
1357         me= get_mesh(ob);
1358         if(ob->id.lib || me==NULL) return OPERATOR_PASS_THROUGH;
1359         
1360         if(ob->mode & OB_MODE_WEIGHT_PAINT) ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1361         else ob->mode |= OB_MODE_WEIGHT_PAINT;
1362         
1363         
1364         /* Weightpaint works by overriding colors in mesh,
1365                 * so need to make sure we recalc on enter and
1366                 * exit (exit needs doing regardless because we
1367                                 * should redeform).
1368                 */
1369         DAG_id_tag_update(&me->id, 0);
1370         
1371         if(ob->mode & OB_MODE_WEIGHT_PAINT) {
1372                 Object *par;
1373                 
1374                 if(wp==NULL)
1375                         wp= scene->toolsettings->wpaint= new_vpaint(1);
1376
1377                 paint_init(&wp->paint, PAINT_CURSOR_WEIGHT_PAINT);
1378                 paint_cursor_start(C, weight_paint_poll);
1379                 
1380                 mesh_octree_table(ob, NULL, NULL, 's');
1381                 
1382                 /* verify if active weight group is also active bone */
1383                 par= modifiers_isDeformedByArmature(ob);
1384                 if(par && (par->mode & OB_MODE_POSE)) {
1385                         bArmature *arm= par->data;
1386
1387                         if(arm->act_bone)
1388                                 ED_vgroup_select_by_name(ob, arm->act_bone->name);
1389                 }
1390         }
1391         else {
1392                 mesh_octree_table(NULL, NULL, NULL, 'e');
1393                 mesh_mirrtopo_table(NULL, 'e');
1394         }
1395         
1396         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1397         
1398         return OPERATOR_FINISHED;
1399 }
1400
1401 /* for switching to/from mode */
1402 static int paint_poll_test(bContext *C)
1403 {
1404         if(CTX_data_edit_object(C))
1405                 return 0;
1406         if(CTX_data_active_object(C)==NULL)
1407                 return 0;
1408         return 1;
1409 }
1410
1411 void PAINT_OT_weight_paint_toggle(wmOperatorType *ot)
1412 {
1413         
1414         /* identifiers */
1415         ot->name= "Weight Paint Mode";
1416         ot->idname= "PAINT_OT_weight_paint_toggle";
1417         
1418         /* api callbacks */
1419         ot->exec= set_wpaint;
1420         ot->poll= paint_poll_test;
1421         
1422         /* flags */
1423         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1424         
1425 }
1426
1427 /* ************ weight paint operator ********** */
1428
1429 struct WPaintData {
1430         ViewContext vc;
1431         int *indexar;
1432         int vgroup_mirror;
1433         float *vertexcosnos;
1434         float wpimat[3][3];
1435         
1436         /*variables for auto normalize*/
1437         int auto_normalize;
1438         char *vgroup_validmap; /*stores if vgroups tie to deforming bones or not*/
1439 };
1440
1441 static char *wpaint_make_validmap(Object *ob)
1442 {
1443         bDeformGroup *dg;
1444         ModifierData *md;
1445         char *validmap;
1446         bPose *pose;
1447         bPoseChannel *chan;
1448         ArmatureModifierData *amd;
1449         GHash *gh = BLI_ghash_new(BLI_ghashutil_strhash, BLI_ghashutil_strcmp, "wpaint_make_validmap gh");
1450         int i = 0, step1=1;
1451
1452         /*add all names to a hash table*/
1453         for (dg=ob->defbase.first, i=0; dg; dg=dg->next, i++) {
1454                 BLI_ghash_insert(gh, dg->name, NULL);
1455         }
1456
1457         if (!i)
1458                 return NULL;
1459
1460         validmap = MEM_callocN(i, "wpaint valid map");
1461
1462         /*now loop through the armature modifiers and identify deform bones*/
1463         for (md = ob->modifiers.first; md; md= !md->next && step1 ? (step1=0), modifiers_getVirtualModifierList(ob) : md->next) {
1464                 if (!(md->mode & (eModifierMode_Realtime|eModifierMode_Virtual)))
1465                         continue;
1466
1467                 if (md->type == eModifierType_Armature) 
1468                 {
1469                         amd = (ArmatureModifierData*) md;
1470
1471                         if(amd->object && amd->object->pose) {
1472                                 pose = amd->object->pose;
1473                                 
1474                                 for (chan=pose->chanbase.first; chan; chan=chan->next) {
1475                                         if (chan->bone->flag & BONE_NO_DEFORM)
1476                                                 continue;
1477
1478                                         if (BLI_ghash_haskey(gh, chan->name)) {
1479                                                 BLI_ghash_remove(gh, chan->name, NULL, NULL);
1480                                                 BLI_ghash_insert(gh, chan->name, SET_INT_IN_POINTER(1));
1481                                         }
1482                                 }
1483                         }
1484                 }
1485         }
1486         
1487         /*add all names to a hash table*/
1488         for (dg=ob->defbase.first, i=0; dg; dg=dg->next, i++) {
1489                 if (BLI_ghash_lookup(gh, dg->name) != NULL) {
1490                         validmap[i] = 1;
1491                 }
1492         }
1493
1494         BLI_ghash_free(gh, NULL, NULL);
1495
1496         return validmap;
1497 }
1498
1499 static int wpaint_stroke_test_start(bContext *C, wmOperator *op, wmEvent *UNUSED(event))
1500 {
1501         Scene *scene= CTX_data_scene(C);
1502         struct PaintStroke *stroke = op->customdata;
1503         ToolSettings *ts= CTX_data_tool_settings(C);
1504         VPaint *wp= ts->wpaint;
1505         Object *ob= CTX_data_active_object(C);
1506         struct WPaintData *wpd;
1507         Mesh *me;
1508         float mat[4][4], imat[4][4];
1509
1510         if(scene->obedit) return OPERATOR_CANCELLED;
1511         
1512         me= get_mesh(ob);
1513         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1514         
1515         /* if nothing was added yet, we make dverts and a vertex deform group */
1516         if (!me->dvert) {
1517                 ED_vgroup_data_create(&me->id);
1518                 WM_event_add_notifier(C, NC_GEOM|ND_DATA, me);
1519         }
1520
1521
1522         /* make mode data storage */
1523         wpd= MEM_callocN(sizeof(struct WPaintData), "WPaintData");
1524         paint_stroke_set_mode_data(stroke, wpd);
1525         view3d_set_viewcontext(C, &wpd->vc);
1526         wpd->vgroup_mirror= -1;
1527         
1528         /*set up auto-normalize, and generate map for detecting which
1529           vgroups affect deform bones*/
1530         wpd->auto_normalize = ts->auto_normalize;
1531         if (wpd->auto_normalize)
1532                 wpd->vgroup_validmap = wpaint_make_validmap(ob);
1533         
1534         //      if(qual & LR_CTRLKEY) {
1535         //              sample_wpaint(scene, ar, v3d, 0);
1536         //              return;
1537         //      }
1538         //      if(qual & LR_SHIFTKEY) {
1539         //              sample_wpaint(scene, ar, v3d, 1);
1540         //              return;
1541         //      }
1542         
1543         /* ALLOCATIONS! no return after this line */
1544         /* painting on subsurfs should give correct points too, this returns me->totvert amount */
1545         wpd->vertexcosnos= mesh_get_mapped_verts_nors(scene, ob);
1546         wpd->indexar= get_indexarray(me);
1547         copy_wpaint_prev(wp, me->dvert, me->totvert);
1548         
1549         /* this happens on a Bone select, when no vgroup existed yet */
1550         if(ob->actdef<=0) {
1551                 Object *modob;
1552                 if((modob = modifiers_isDeformedByArmature(ob))) {
1553                         Bone *actbone= ((bArmature *)modob->data)->act_bone;
1554                         if(actbone) {
1555                                 bPoseChannel *pchan= get_pose_channel(modob->pose, actbone->name);
1556
1557                                 if(pchan) {
1558                                         bDeformGroup *dg= defgroup_find_name(ob, pchan->name);
1559                                         if(dg==NULL)
1560                                                 dg= ED_vgroup_add_name(ob, pchan->name);        /* sets actdef */
1561                                         else
1562                                                 ob->actdef= 1 + defgroup_find_index(ob, dg);
1563                                 }
1564                         }
1565                 }
1566         }
1567         if(ob->defbase.first==NULL) {
1568                 ED_vgroup_add(ob);
1569         }
1570         
1571         //      if(ob->lay & v3d->lay); else error("Active object is not in this layer");
1572         
1573         /* imat for normals */
1574         mul_m4_m4m4(mat, ob->obmat, wpd->vc.rv3d->viewmat);
1575         invert_m4_m4(imat, mat);
1576         copy_m3_m4(wpd->wpimat, imat);
1577         
1578         /* if mirror painting, find the other group */
1579         if(me->editflag & ME_EDIT_MIRROR_X) {
1580                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
1581                 if(defgroup) {
1582                         bDeformGroup *curdef;
1583                         int actdef= 0;
1584                         char name[32];
1585
1586                         flip_side_name(name, defgroup->name, FALSE);
1587
1588                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
1589                                 if (!strcmp(curdef->name, name))
1590                                         break;
1591                         if(curdef==NULL) {
1592                                 int olddef= ob->actdef; /* tsk, ED_vgroup_add sets the active defgroup */
1593                                 curdef= ED_vgroup_add_name (ob, name);
1594                                 ob->actdef= olddef;
1595                         }
1596                         
1597                         if(curdef && curdef!=defgroup)
1598                                 wpd->vgroup_mirror= actdef;
1599                 }
1600         }
1601
1602         return 1;
1603 }
1604
1605 static void wpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1606 {
1607         ToolSettings *ts= CTX_data_tool_settings(C);
1608         VPaint *wp= ts->wpaint;
1609         Brush *brush = paint_brush(&wp->paint);
1610         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1611         ViewContext *vc;
1612         Object *ob;
1613         Mesh *me;
1614         float mat[4][4];
1615         float paintweight;
1616         int *indexar;
1617         int totindex, index, totw, flip;
1618         float alpha;
1619         float mval[2], pressure;
1620         
1621         /* cannot paint if there is no stroke data */
1622         if (wpd == NULL) {
1623                 // XXX: force a redraw here, since even though we can't paint, 
1624                 // at least view won't freeze until stroke ends
1625                 ED_region_tag_redraw(CTX_wm_region(C));
1626                 return;
1627         }
1628                 
1629         vc= &wpd->vc;
1630         ob= vc->obact;
1631         me= ob->data;
1632         indexar= wpd->indexar;
1633         
1634         view3d_operator_needs_opengl(C);
1635                 
1636         /* load projection matrix */
1637         mul_m4_m4m4(mat, ob->obmat, vc->rv3d->persmat);
1638
1639         flip = RNA_boolean_get(itemptr, "pen_flip");
1640         pressure = RNA_float_get(itemptr, "pressure");
1641         RNA_float_get_array(itemptr, "mouse", mval);
1642         mval[0]-= vc->ar->winrct.xmin;
1643         mval[1]-= vc->ar->winrct.ymin;
1644                         
1645         swap_m4m4(wpd->vc.rv3d->persmat, mat);
1646                         
1647         /* which faces are involved */
1648         if(wp->flag & VP_AREA) {
1649                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush_size(brush));
1650         }
1651         else {
1652                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1653                 if(indexar[0]) totindex= 1;
1654                 else totindex= 0;
1655         }
1656                         
1657         if(wp->flag & VP_COLINDEX) {
1658                 for(index=0; index<totindex; index++) {
1659                         if(indexar[index] && indexar[index]<=me->totface) {
1660                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1661                                                 
1662                                 if(mface->mat_nr!=ob->actcol-1) {
1663                                         indexar[index]= 0;
1664                                 }
1665                         }
1666                 }
1667         }
1668                         
1669         if((me->editflag & ME_EDIT_PAINT_MASK) && me->mface) {
1670                 for(index=0; index<totindex; index++) {
1671                         if(indexar[index] && indexar[index]<=me->totface) {
1672                                 MFace *mface= ((MFace *)me->mface) + (indexar[index]-1);
1673                                                 
1674                                 if((mface->flag & ME_FACE_SEL)==0) {
1675                                         indexar[index]= 0;
1676                                 }
1677                         }                                       
1678                 }
1679         }
1680                         
1681         /* make sure each vertex gets treated only once */
1682         /* and calculate filter weight */
1683         totw= 0;
1684         if(brush->vertexpaint_tool==VP_BLUR) 
1685                 paintweight= 0.0f;
1686         else
1687                 paintweight= ts->vgroup_weight;
1688                         
1689         for(index=0; index<totindex; index++) {
1690                 if(indexar[index] && indexar[index]<=me->totface) {
1691                         MFace *mface= me->mface + (indexar[index]-1);
1692                                         
1693                         (me->dvert+mface->v1)->flag= 1;
1694                         (me->dvert+mface->v2)->flag= 1;
1695                         (me->dvert+mface->v3)->flag= 1;
1696                         if(mface->v4) (me->dvert+mface->v4)->flag= 1;
1697                                         
1698                         if(brush->vertexpaint_tool==VP_BLUR) {
1699                                 MDeformWeight *dw, *(*dw_func)(MDeformVert *, const int);
1700                                                 
1701                                 if(wp->flag & VP_ONLYVGROUP)
1702                                         dw_func= (MDeformWeight *(*)(MDeformVert *, const int))defvert_find_index;
1703                                 else
1704                                         dw_func= defvert_verify_index;
1705                                                 
1706                                 dw= dw_func(me->dvert+mface->v1, ob->actdef-1);
1707                                 if(dw) {paintweight+= dw->weight; totw++;}
1708                                 dw= dw_func(me->dvert+mface->v2, ob->actdef-1);
1709                                 if(dw) {paintweight+= dw->weight; totw++;}
1710                                 dw= dw_func(me->dvert+mface->v3, ob->actdef-1);
1711                                 if(dw) {paintweight+= dw->weight; totw++;}
1712                                 if(mface->v4) {
1713                                         dw= dw_func(me->dvert+mface->v4, ob->actdef-1);
1714                                         if(dw) {paintweight+= dw->weight; totw++;}
1715                                 }
1716                         }
1717                 }
1718         }
1719                         
1720         if(brush->vertexpaint_tool==VP_BLUR) 
1721                 paintweight/= (float)totw;
1722                         
1723         for(index=0; index<totindex; index++) {
1724                                 
1725                 if(indexar[index] && indexar[index]<=me->totface) {
1726                         MFace *mface= me->mface + (indexar[index]-1);
1727                                         
1728                         if((me->dvert+mface->v1)->flag) {
1729                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v1, mval, pressure);
1730                                 if(alpha) {
1731                                         do_weight_paint_vertex(wp, ob, mface->v1, 
1732                                                 alpha, paintweight, flip, wpd->vgroup_mirror, 
1733                                                 wpd->vgroup_validmap);
1734                                 }
1735                                 (me->dvert+mface->v1)->flag= 0;
1736                         }
1737                                         
1738                         if((me->dvert+mface->v2)->flag) {
1739                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v2, mval, pressure);
1740                                 if(alpha) {
1741                                         do_weight_paint_vertex(wp, ob, mface->v2, 
1742                                                 alpha, paintweight, flip, wpd->vgroup_mirror, 
1743                                                 wpd->vgroup_validmap);
1744                                 }
1745                                 (me->dvert+mface->v2)->flag= 0;
1746                         }
1747                                         
1748                         if((me->dvert+mface->v3)->flag) {
1749                                 alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v3, mval, pressure);
1750                                 if(alpha) {
1751                                         do_weight_paint_vertex(wp, ob, mface->v3, 
1752                                                 alpha, paintweight, flip, wpd->vgroup_mirror, 
1753                                                 wpd->vgroup_validmap);
1754                                 }
1755                                 (me->dvert+mface->v3)->flag= 0;
1756                         }
1757                                         
1758                         if((me->dvert+mface->v4)->flag) {
1759                                 if(mface->v4) {
1760                                         alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*mface->v4, mval, pressure);
1761                                         if(alpha) {
1762                                                 do_weight_paint_vertex(wp, ob, mface->v4, 
1763                                                         alpha, paintweight, flip, wpd->vgroup_mirror,
1764                                                         wpd->vgroup_validmap);
1765                                         }
1766                                         (me->dvert+mface->v4)->flag= 0;
1767                                 }
1768                         }
1769                 }
1770         }
1771                         
1772         swap_m4m4(vc->rv3d->persmat, mat);
1773                         
1774         DAG_id_tag_update(ob->data, 0);
1775         ED_region_tag_redraw(vc->ar);
1776 }
1777
1778 static void wpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
1779 {
1780         ToolSettings *ts= CTX_data_tool_settings(C);
1781         Object *ob= CTX_data_active_object(C);
1782         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1783         
1784         if(wpd) {
1785                 if(wpd->vertexcosnos)
1786                         MEM_freeN(wpd->vertexcosnos);
1787                 MEM_freeN(wpd->indexar);
1788                 
1789                 if (wpd->vgroup_validmap)
1790                         MEM_freeN(wpd->vgroup_validmap);
1791                 
1792                 MEM_freeN(wpd);
1793         }
1794         
1795         /* frees prev buffer */
1796         copy_wpaint_prev(ts->wpaint, NULL, 0);
1797         
1798         /* and particles too */
1799         if(ob->particlesystem.first) {
1800                 ParticleSystem *psys;
1801                 int i;
1802                 
1803                 for(psys= ob->particlesystem.first; psys; psys= psys->next) {
1804                         for(i=0; i<PSYS_TOT_VG; i++) {
1805                                 if(psys->vgroup[i]==ob->actdef) {
1806                                         psys->recalc |= PSYS_RECALC_RESET;
1807                                         break;
1808                                 }
1809                         }
1810                 }
1811         }
1812         
1813         DAG_id_tag_update(ob->data, 0);
1814 }
1815
1816
1817 static int wpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1818 {
1819         
1820         op->customdata = paint_stroke_new(C, NULL, wpaint_stroke_test_start,
1821                                           wpaint_stroke_update_step,
1822                                           wpaint_stroke_done, event->type);
1823         
1824         /* add modal handler */
1825         WM_event_add_modal_handler(C, op);
1826
1827         op->type->modal(C, op, event);
1828         
1829         return OPERATOR_RUNNING_MODAL;
1830 }
1831
1832 void PAINT_OT_weight_paint(wmOperatorType *ot)
1833 {
1834         
1835         /* identifiers */
1836         ot->name= "Weight Paint";
1837         ot->idname= "PAINT_OT_weight_paint";
1838         
1839         /* api callbacks */
1840         ot->invoke= wpaint_invoke;
1841         ot->modal= paint_stroke_modal;
1842         /* ot->exec= vpaint_exec; <-- needs stroke property */
1843         ot->poll= weight_paint_poll;
1844         
1845         /* flags */
1846         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1847
1848         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1849 }
1850
1851 static int weight_paint_set_exec(bContext *C, wmOperator *UNUSED(op))
1852 {
1853         struct Scene *scene= CTX_data_scene(C);
1854         Object *obact = CTX_data_active_object(C);
1855
1856         wpaint_fill(scene->toolsettings->wpaint, obact, scene->toolsettings->vgroup_weight);
1857         ED_region_tag_redraw(CTX_wm_region(C)); // XXX - should redraw all 3D views
1858         return OPERATOR_FINISHED;
1859 }
1860
1861 void PAINT_OT_weight_set(wmOperatorType *ot)
1862 {
1863         /* identifiers */
1864         ot->name= "Set Weight";
1865         ot->idname= "PAINT_OT_weight_set";
1866
1867         /* api callbacks */
1868         ot->exec= weight_paint_set_exec;
1869         ot->poll= facemask_paint_poll;
1870
1871         /* flags */
1872         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1873 }
1874
1875 /* ************ set / clear vertex paint mode ********** */
1876
1877
1878 static int set_vpaint(bContext *C, wmOperator *op)              /* toggle */
1879 {       
1880         Object *ob= CTX_data_active_object(C);
1881         Scene *scene= CTX_data_scene(C);
1882         VPaint *vp= scene->toolsettings->vpaint;
1883         Mesh *me;
1884         
1885         me= get_mesh(ob);
1886         
1887         if(me==NULL || object_data_is_libdata(ob)) {
1888                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1889                 return OPERATOR_PASS_THROUGH;
1890         }
1891         
1892         if(me && me->mcol==NULL) make_vertexcol(ob);
1893         
1894         /* toggle: end vpaint */
1895         if(ob->mode & OB_MODE_VERTEX_PAINT) {
1896                 
1897                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1898         }
1899         else {
1900                 ob->mode |= OB_MODE_VERTEX_PAINT;
1901                 /* Turn off weight painting */
1902                 if (ob->mode & OB_MODE_WEIGHT_PAINT)
1903                         set_wpaint(C, op);
1904                 
1905                 if(vp==NULL)
1906                         vp= scene->toolsettings->vpaint= new_vpaint(0);
1907                 
1908                 paint_cursor_start(C, vertex_paint_poll);
1909
1910                 paint_init(&vp->paint, PAINT_CURSOR_VERTEX_PAINT);
1911         }
1912         
1913         if (me)
1914                 /* update modifier stack for mapping requirements */
1915                 DAG_id_tag_update(&me->id, 0);
1916         
1917         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1918         
1919         return OPERATOR_FINISHED;
1920 }
1921
1922 void PAINT_OT_vertex_paint_toggle(wmOperatorType *ot)
1923 {
1924         
1925         /* identifiers */
1926         ot->name= "Vertex Paint Mode";
1927         ot->idname= "PAINT_OT_vertex_paint_toggle";
1928         
1929         /* api callbacks */
1930         ot->exec= set_vpaint;
1931         ot->poll= paint_poll_test;
1932         
1933         /* flags */
1934         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1935 }
1936
1937
1938
1939 /* ********************** vertex paint operator ******************* */
1940
1941 /* Implementation notes:
1942
1943 Operator->invoke()
1944   - validate context (add mcol)
1945   - create customdata storage
1946   - call paint once (mouse click)
1947   - add modal handler 
1948
1949 Operator->modal()
1950   - for every mousemove, apply vertex paint
1951   - exit on mouse release, free customdata
1952         (return OPERATOR_FINISHED also removes handler and operator)
1953
1954 For future:
1955   - implement a stroke event (or mousemove with past positons)
1956   - revise whether op->customdata should be added in object, in set_vpaint
1957
1958 */
1959
1960 typedef struct VPaintData {
1961         ViewContext vc;
1962         unsigned int paintcol;
1963         int *indexar;
1964         float *vertexcosnos;
1965         float vpimat[3][3];
1966 } VPaintData;
1967
1968 static int vpaint_stroke_test_start(bContext *C, struct wmOperator *op, wmEvent *UNUSED(event))
1969 {
1970         ToolSettings *ts= CTX_data_tool_settings(C);
1971         struct PaintStroke *stroke = op->customdata;
1972         VPaint *vp= ts->vpaint;
1973         struct VPaintData *vpd;
1974         Object *ob= CTX_data_active_object(C);
1975         Mesh *me;
1976         float mat[4][4], imat[4][4];
1977
1978         /* context checks could be a poll() */
1979         me= get_mesh(ob);
1980         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1981         
1982         if(me->mcol==NULL) make_vertexcol(ob);
1983         if(me->mcol==NULL) return OPERATOR_CANCELLED;
1984         
1985         /* make mode data storage */
1986         vpd= MEM_callocN(sizeof(struct VPaintData), "VPaintData");
1987         paint_stroke_set_mode_data(stroke, vpd);
1988         view3d_set_viewcontext(C, &vpd->vc);
1989         
1990         vpd->vertexcosnos= mesh_get_mapped_verts_nors(vpd->vc.scene, ob);
1991         vpd->indexar= get_indexarray(me);
1992         vpd->paintcol= vpaint_get_current_col(vp);
1993         
1994         /* for filtering */
1995         copy_vpaint_prev(vp, (unsigned int *)me->mcol, me->totface);
1996         
1997         /* some old cruft to sort out later */
1998         mul_m4_m4m4(mat, ob->obmat, vpd->vc.rv3d->viewmat);
1999         invert_m4_m4(imat, mat);
2000         copy_m3_m4(vpd->vpimat, imat);
2001
2002         return 1;
2003 }
2004
2005 static void vpaint_paint_face(VPaint *vp, VPaintData *vpd, Object *ob, int index, const float mval[2], float pressure, int UNUSED(flip))
2006 {
2007         ViewContext *vc = &vpd->vc;
2008         Brush *brush = paint_brush(&vp->paint);
2009         Mesh *me = get_mesh(ob);
2010         MFace *mface= ((MFace*)me->mface) + index;
2011         unsigned int *mcol= ((unsigned int*)me->mcol) + 4*index;
2012         unsigned int *mcolorig= ((unsigned int*)vp->vpaint_prev) + 4*index;
2013         float alpha;
2014         int i;
2015         
2016         if((vp->flag & VP_COLINDEX && mface->mat_nr!=ob->actcol-1) ||
2017            ((me->editflag & ME_EDIT_PAINT_MASK) && !(mface->flag & ME_FACE_SEL)))
2018                 return;
2019
2020         if(brush->vertexpaint_tool==VP_BLUR) {
2021                 unsigned int fcol1= mcol_blend( mcol[0], mcol[1], 128);
2022                 if(mface->v4) {
2023                         unsigned int fcol2= mcol_blend( mcol[2], mcol[3], 128);
2024                         vpd->paintcol= mcol_blend( fcol1, fcol2, 128);
2025                 }
2026                 else {
2027                         vpd->paintcol= mcol_blend( mcol[2], fcol1, 170);
2028                 }
2029                 
2030         }
2031
2032         for(i = 0; i < (mface->v4 ? 4 : 3); ++i) {
2033                 alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*(&mface->v1)[i], mval, pressure);
2034                 if(alpha)
2035                         vpaint_blend(vp, mcol+i, mcolorig+i, vpd->paintcol, (int)(alpha*255.0f));
2036         }
2037 }
2038
2039 static void vpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
2040 {
2041         ToolSettings *ts= CTX_data_tool_settings(C);
2042         struct VPaintData *vpd = paint_stroke_mode_data(stroke);
2043         VPaint *vp= ts->vpaint;
2044         Brush *brush = paint_brush(&vp->paint);
2045         ViewContext *vc= &vpd->vc;
2046         Object *ob= vc->obact;
2047         Mesh *me= ob->data;
2048         float mat[4][4];
2049         int *indexar= vpd->indexar;
2050         int totindex, index, flip;
2051         float pressure, mval[2];
2052
2053         RNA_float_get_array(itemptr, "mouse", mval);
2054         flip = RNA_boolean_get(itemptr, "pen_flip");
2055         pressure = RNA_float_get(itemptr, "pressure");
2056                         
2057         view3d_operator_needs_opengl(C);
2058                         
2059         /* load projection matrix */
2060         mul_m4_m4m4(mat, ob->obmat, vc->rv3d->persmat);
2061
2062         mval[0]-= vc->ar->winrct.xmin;
2063         mval[1]-= vc->ar->winrct.ymin;
2064
2065                         
2066         /* which faces are involved */
2067         if(vp->flag & VP_AREA) {
2068                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush_size(brush));
2069         }
2070         else {
2071                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
2072                 if(indexar[0]) totindex= 1;
2073                 else totindex= 0;
2074         }
2075                         
2076         swap_m4m4(vc->rv3d->persmat, mat);
2077                         
2078         for(index=0; index<totindex; index++) {                         
2079                 if(indexar[index] && indexar[index]<=me->totface)
2080                         vpaint_paint_face(vp, vpd, ob, indexar[index]-1, mval, pressure, flip);
2081         }
2082                                                 
2083         swap_m4m4(vc->rv3d->persmat, mat);
2084
2085         /* was disabled because it is slow, but necessary for blur */
2086         if(brush->vertexpaint_tool == VP_BLUR)
2087                 do_shared_vertexcol(me);
2088                         
2089         ED_region_tag_redraw(vc->ar);
2090                         
2091         DAG_id_tag_update(ob->data, 0);
2092 }
2093
2094 static void vpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
2095 {
2096         ToolSettings *ts= CTX_data_tool_settings(C);
2097         struct VPaintData *vpd= paint_stroke_mode_data(stroke);
2098         
2099         if(vpd->vertexcosnos)
2100                 MEM_freeN(vpd->vertexcosnos);
2101         MEM_freeN(vpd->indexar);
2102         
2103         /* frees prev buffer */
2104         copy_vpaint_prev(ts->vpaint, NULL, 0);
2105         
2106         MEM_freeN(vpd);
2107 }
2108
2109 static int vpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
2110 {
2111         
2112         op->customdata = paint_stroke_new(C, NULL, vpaint_stroke_test_start,
2113                                           vpaint_stroke_update_step,
2114                                           vpaint_stroke_done, event->type);
2115         
2116         /* add modal handler */
2117         WM_event_add_modal_handler(C, op);
2118
2119         op->type->modal(C, op, event);
2120         
2121         return OPERATOR_RUNNING_MODAL;
2122 }
2123
2124 void PAINT_OT_vertex_paint(wmOperatorType *ot)
2125 {
2126         /* identifiers */
2127         ot->name= "Vertex Paint";
2128         ot->idname= "PAINT_OT_vertex_paint";
2129         
2130         /* api callbacks */
2131         ot->invoke= vpaint_invoke;
2132         ot->modal= paint_stroke_modal;
2133         /* ot->exec= vpaint_exec; <-- needs stroke property */
2134         ot->poll= vertex_paint_poll;
2135         
2136         /* flags */
2137         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
2138
2139         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
2140 }
2141
2142 /* ********************** weight from bones operator ******************* */
2143
2144 static int weight_from_bones_poll(bContext *C)
2145 {
2146         Object *ob= CTX_data_active_object(C);
2147
2148         return (ob && (ob->mode & OB_MODE_WEIGHT_PAINT) && modifiers_isDeformedByArmature(ob));
2149 }
2150
2151 static int weight_from_bones_exec(bContext *C, wmOperator *op)
2152 {
2153         Scene *scene= CTX_data_scene(C);
2154         Object *ob= CTX_data_active_object(C);
2155         Object *armob= modifiers_isDeformedByArmature(ob);
2156         Mesh *me= ob->data;
2157         int type= RNA_enum_get(op->ptr, "type");
2158
2159         create_vgroups_from_armature(op->reports, scene, ob, armob, type, (me->editflag & ME_EDIT_MIRROR_X));
2160
2161         DAG_id_tag_update(&me->id, 0);
2162         WM_event_add_notifier(C, NC_GEOM|ND_DATA, me);
2163
2164         return OPERATOR_FINISHED;
2165 }
2166
2167 void PAINT_OT_weight_from_bones(wmOperatorType *ot)
2168 {
2169         static EnumPropertyItem type_items[]= {
2170                 {ARM_GROUPS_AUTO, "AUTOMATIC", 0, "Automatic", "Automatic weights froms bones"},
2171                 {ARM_GROUPS_ENVELOPE, "ENVELOPES", 0, "From Envelopes", "Weights from envelopes with user defined radius"},
2172                 {0, NULL, 0, NULL, NULL}};
2173
2174         /* identifiers */
2175         ot->name= "Weight from Bones";
2176         ot->idname= "PAINT_OT_weight_from_bones";
2177         
2178         /* api callbacks */
2179         ot->exec= weight_from_bones_exec;
2180         ot->invoke= WM_menu_invoke;
2181         ot->poll= weight_from_bones_poll;
2182         
2183         /* flags */
2184         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
2185
2186         /* properties */
2187         ot->prop= RNA_def_enum(ot->srna, "type", type_items, 0, "Type", "Method to use for assigning weights.");
2188 }
2189