ok, apparently didn't commit this either. apparently includes a merge with trunk...
[blender-staging.git] / source / blender / editors / sculpt_paint / paint_vertex.c
1 /**
2  * $Id$
3  *
4  * ***** BEGIN GPL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
19  *
20  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
21  * All rights reserved.
22  *
23  * The Original Code is: all of this file.
24  *
25  * Contributor(s): none yet.
26  *
27  * ***** END GPL LICENSE BLOCK *****
28  */
29
30 #include <math.h>
31 #include <string.h>
32
33 #ifdef WIN32
34 #include <io.h>
35 #else
36 #include <unistd.h>
37 #endif   
38
39 #include "MEM_guardedalloc.h"
40
41 #include "IMB_imbuf.h"
42 #include "IMB_imbuf_types.h"
43
44 #include "BLI_blenlib.h"
45 #include "BLI_math.h"
46 #include "BLI_memarena.h"
47 #include "BLI_cellalloc.h"
48 #include "BLI_ghash.h"
49
50 #include "DNA_anim_types.h"
51 #include "DNA_action_types.h"
52 #include "DNA_armature_types.h"
53 #include "DNA_brush_types.h"
54 #include "DNA_cloth_types.h"
55 #include "DNA_mesh_types.h"
56 #include "DNA_meshdata_types.h"
57 #include "DNA_modifier_types.h"
58 #include "DNA_object_types.h"
59 #include "DNA_object_force.h"
60 #include "DNA_particle_types.h"
61 #include "DNA_screen_types.h"
62 #include "DNA_space_types.h"
63 #include "DNA_scene_types.h"
64 #include "DNA_view3d_types.h"
65 #include "DNA_userdef_types.h"
66
67 #include "RNA_access.h"
68 #include "RNA_define.h"
69
70 #include "BKE_armature.h"
71 #include "BKE_brush.h"
72 #include "BKE_DerivedMesh.h"
73 #include "BKE_cloth.h"
74 #include "BKE_context.h"
75 #include "BKE_customdata.h"
76 #include "BKE_depsgraph.h"
77 #include "BKE_deform.h"
78 #include "BKE_displist.h"
79 #include "BKE_global.h"
80 #include "BKE_mesh.h"
81 #include "BKE_modifier.h"
82 #include "BKE_object.h"
83 #include "BKE_paint.h"
84 #include "BKE_utildefines.h"
85
86 #include "WM_api.h"
87 #include "WM_types.h"
88
89 #include "BIF_gl.h"
90 #include "BIF_glutil.h"
91
92 #include "ED_mesh.h"
93 #include "ED_object.h"
94 #include "ED_screen.h"
95 #include "ED_util.h"
96 #include "ED_view3d.h"
97
98 #include "paint_intern.h"
99
100         /* vp->mode */
101 #define VP_MIX  0
102 #define VP_ADD  1
103 #define VP_SUB  2
104 #define VP_MUL  3
105 #define VP_BLUR 4
106 #define VP_LIGHTEN      5
107 #define VP_DARKEN       6
108
109 #define MAXINDEX        512000
110
111 /* XXX */
112 static void error() {}
113
114 /* polling - retrieve whether cursor should be set or operator should be done */
115
116
117 /* Returns true if vertex paint mode is active */
118 int vertex_paint_mode_poll(bContext *C)
119 {
120         Object *ob = CTX_data_active_object(C);
121
122         return ob && ob->mode == OB_MODE_VERTEX_PAINT;
123 }
124
125 int vertex_paint_poll(bContext *C)
126 {
127         if(vertex_paint_mode_poll(C) && 
128            paint_brush(&CTX_data_tool_settings(C)->vpaint->paint)) {
129                 ScrArea *sa= CTX_wm_area(C);
130                 if(sa->spacetype==SPACE_VIEW3D) {
131                         ARegion *ar= CTX_wm_region(C);
132                         if(ar->regiontype==RGN_TYPE_WINDOW)
133                                 return 1;
134                 }
135         }
136         return 0;
137 }
138
139 int weight_paint_poll(bContext *C)
140 {
141         Object *ob = CTX_data_active_object(C);
142
143         if(ob && ob->mode & OB_MODE_WEIGHT_PAINT &&
144            paint_brush(&CTX_data_tool_settings(C)->wpaint->paint)) {
145                 ScrArea *sa= CTX_wm_area(C);
146                 if(sa->spacetype==SPACE_VIEW3D) {
147                         ARegion *ar= CTX_wm_region(C);
148                         if(ar->regiontype==RGN_TYPE_WINDOW)
149                                 return 1;
150                 }
151         }
152         return 0;
153 }
154
155 static VPaint *new_vpaint(int wpaint)
156 {
157         VPaint *vp= MEM_callocN(sizeof(VPaint), "VPaint");
158         
159         vp->flag= VP_AREA+VP_SPRAY;
160         
161         if(wpaint)
162                 vp->flag= VP_AREA;
163
164         return vp;
165 }
166
167 static int *get_indexarray(void)
168 {
169         return MEM_mallocN(sizeof(int)*MAXINDEX + 2, "vertexpaint");
170 }
171
172
173 /* in contradiction to cpack drawing colors, the MCOL colors (vpaint colors) are per byte! 
174    so not endian sensitive. Mcol = ABGR!!! so be cautious with cpack calls */
175
176 unsigned int rgba_to_mcol(float r, float g, float b, float a)
177 {
178         int ir, ig, ib, ia;
179         unsigned int col;
180         char *cp;
181         
182         ir= floor(255.0*r);
183         if(ir<0) ir= 0; else if(ir>255) ir= 255;
184         ig= floor(255.0*g);
185         if(ig<0) ig= 0; else if(ig>255) ig= 255;
186         ib= floor(255.0*b);
187         if(ib<0) ib= 0; else if(ib>255) ib= 255;
188         ia= floor(255.0*a);
189         if(ia<0) ia= 0; else if(ia>255) ia= 255;
190         
191         cp= (char *)&col;
192         cp[0]= ia;
193         cp[1]= ib;
194         cp[2]= ig;
195         cp[3]= ir;
196         
197         return col;
198         
199 }
200
201 unsigned int vpaint_get_current_col(VPaint *vp)
202 {
203         Brush *brush = paint_brush(&vp->paint);
204         return rgba_to_mcol(brush->rgb[0], brush->rgb[1], brush->rgb[2], 1.0f);
205 }
206
207 static void do_shared_vertex_tesscol(Mesh *me)
208 {
209         /* if no mcol: do not do */
210         /* if tface: only the involved faces, otherwise all */
211         MFace *mface;
212         MTFace *tface;
213         int a;
214         short *scolmain, *scol;
215         char *mcol;
216         
217         if(me->mcol==0 || me->totvert==0 || me->totface==0) return;
218         
219         scolmain= MEM_callocN(4*sizeof(short)*me->totvert, "colmain");
220         
221         tface= me->mtface;
222         mface= me->mface;
223         mcol= (char *)me->mcol;
224         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
225                 if((tface && tface->mode & TF_SHAREDCOL) || (me->editflag & ME_EDIT_PAINT_MASK)==0) {
226                         scol= scolmain+4*mface->v1;
227                         scol[0]++; scol[1]+= mcol[1]; scol[2]+= mcol[2]; scol[3]+= mcol[3];
228                         scol= scolmain+4*mface->v2;
229                         scol[0]++; scol[1]+= mcol[5]; scol[2]+= mcol[6]; scol[3]+= mcol[7];
230                         scol= scolmain+4*mface->v3;
231                         scol[0]++; scol[1]+= mcol[9]; scol[2]+= mcol[10]; scol[3]+= mcol[11];
232                         if(mface->v4) {
233                                 scol= scolmain+4*mface->v4;
234                                 scol[0]++; scol[1]+= mcol[13]; scol[2]+= mcol[14]; scol[3]+= mcol[15];
235                         }
236                 }
237                 if(tface) tface++;
238         }
239         
240         a= me->totvert;
241         scol= scolmain;
242         while(a--) {
243                 if(scol[0]>1) {
244                         scol[1]/= scol[0];
245                         scol[2]/= scol[0];
246                         scol[3]/= scol[0];
247                 }
248                 scol+= 4;
249         }
250         
251         tface= me->mtface;
252         mface= me->mface;
253         mcol= (char *)me->mcol;
254         for(a=me->totface; a>0; a--, mface++, mcol+=16) {
255                 if((tface && tface->mode & TF_SHAREDCOL) || (me->editflag & ME_EDIT_PAINT_MASK)==0) {
256                         scol= scolmain+4*mface->v1;
257                         mcol[1]= scol[1]; mcol[2]= scol[2]; mcol[3]= scol[3];
258                         scol= scolmain+4*mface->v2;
259                         mcol[5]= scol[1]; mcol[6]= scol[2]; mcol[7]= scol[3];
260                         scol= scolmain+4*mface->v3;
261                         mcol[9]= scol[1]; mcol[10]= scol[2]; mcol[11]= scol[3];
262                         if(mface->v4) {
263                                 scol= scolmain+4*mface->v4;
264                                 mcol[13]= scol[1]; mcol[14]= scol[2]; mcol[15]= scol[3];
265                         }
266                 }
267                 if(tface) tface++;
268         }
269
270         MEM_freeN(scolmain);
271 }
272
273 void do_shared_vertexcol(Mesh *me)
274 {
275         MLoop *ml = me->mloop;
276         MLoopCol *lcol = me->mloopcol;
277         MTexPoly *mtp = me->mtpoly;
278         MPoly *mp = me->mpoly;
279         float (*scol)[5];
280         int i;
281
282         /* if no mloopcol: do not do */
283         /* if mtexpoly: only the involved faces, otherwise all */
284
285         if(me->mloopcol==0 || me->totvert==0 || me->totpoly==0) return;
286
287         scol = MEM_callocN(sizeof(float)*me->totvert*5, "scol");
288
289         for (i=0; i<me->totloop; i++, ml++, lcol++) {
290                 if (i >= mp->loopstart + mp->totloop) {
291                         mp++;
292                         if (mtp) mtp++;
293                 }
294
295                 if (mtp && !(mtp->mode & TF_SHAREDCOL))
296                         continue;
297
298                 scol[ml->v][0] += lcol->r;
299                 scol[ml->v][1] += lcol->g;
300                 scol[ml->v][2] += lcol->b;
301                 scol[ml->v][3] += lcol->a;
302                 scol[ml->v][4] += 1.0;
303         }
304         
305         for (i=0; i<me->totvert; i++) {
306                 if (!scol[i][4]) continue;
307
308                 scol[i][0] /= scol[i][4];
309                 scol[i][1] /= scol[i][4];
310                 scol[i][2] /= scol[i][4];
311                 scol[i][3] /= scol[i][4];
312         }
313         
314         ml = me->mloop;
315         lcol = me->mloopcol;
316         for (i=0; i<me->totloop; i++, ml++, lcol++) {
317                 if (!scol[ml->v][4]) continue;
318
319                 lcol->r = scol[ml->v][0];
320                 lcol->g = scol[ml->v][1];
321                 lcol->b = scol[ml->v][2];
322                 lcol->a = scol[ml->v][3];
323         }
324
325         MEM_freeN(scol);
326
327         do_shared_vertex_tesscol(me);
328 }
329
330 static void make_vertexcol(Object *ob)  /* single ob */
331 {
332         Mesh *me;
333         if(!ob || ob->id.lib) return;
334         me= get_mesh(ob);
335         if(me==0) return;
336         if(me->edit_btmesh) return;
337
338         /* copies from shadedisplist to mcol */
339         if(!me->mcol)
340                 CustomData_add_layer(&me->fdata, CD_MCOL, CD_DEFAULT, NULL, me->totface);
341         if (!me->mloopcol)
342                 CustomData_add_layer(&me->ldata, CD_MLOOPCOL, CD_DEFAULT, NULL, me->totloop);
343         
344         mesh_update_customdata_pointers(me);
345
346         //if(shade)
347         //      shadeMeshMCol(scene, ob, me);
348         //else
349
350         memset(me->mcol, 255, 4*sizeof(MCol)*me->totface);
351
352         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
353         
354 }
355
356 static void copy_vpaint_prev(VPaint *vp, unsigned int *lcol, int tot)
357 {
358         if(vp->vpaint_prev) {
359                 MEM_freeN(vp->vpaint_prev);
360                 vp->vpaint_prev= NULL;
361         }
362         vp->tot= tot;   
363         
364         if(lcol==NULL || tot==0) return;
365         
366         vp->vpaint_prev= MEM_mallocN(sizeof(int)*tot, "vpaint_prev");
367         memcpy(vp->vpaint_prev, lcol, sizeof(int)*tot);
368         
369 }
370
371 static void copy_wpaint_prev (VPaint *wp, MDeformVert *dverts, int dcount)
372 {
373         if (wp->wpaint_prev) {
374                 free_dverts(wp->wpaint_prev, wp->tot);
375                 wp->wpaint_prev= NULL;
376         }
377         
378         if(dverts && dcount) {
379                 
380                 wp->wpaint_prev = MEM_mallocN (sizeof(MDeformVert)*dcount, "wpaint prev");
381                 wp->tot = dcount;
382                 copy_dverts (wp->wpaint_prev, dverts, dcount);
383         }
384 }
385
386
387 void vpaint_fill(Object *ob, unsigned int paintcol)
388 {
389         Mesh *me;
390         MFace *mf;
391         MPoly *mp;
392         MLoopCol *lcol;
393         unsigned int *mcol;
394         int i, j, selected;
395
396         me= get_mesh(ob);
397         if(me==0 || me->totface==0) return;
398
399         if(!me->mcol)
400                 make_vertexcol(ob);
401
402         selected= (me->editflag & ME_EDIT_PAINT_MASK);
403
404         mf = me->mface;
405         mcol = (unsigned int*)me->mcol;
406         for (i = 0; i < me->totface; i++, mf++, mcol+=4) {
407                 if (!selected || mf->flag & ME_FACE_SEL) {
408                         mcol[0] = paintcol;
409                         mcol[1] = paintcol;
410                         mcol[2] = paintcol;
411                         mcol[3] = paintcol;
412                 }
413         }
414
415         mp = me->mpoly;
416         lcol = me->mloopcol;
417         for (i=0; i<me->totpoly; i++, mp++) {
418                 if (!(!selected || mp->flag & ME_FACE_SEL))
419                         continue;
420
421                 lcol = me->mloopcol + mp->loopstart;
422                 for (j=0; j<mp->totloop; j++, lcol++) {
423                         *(int*)lcol = paintcol;
424                 }
425         }
426         
427         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
428 }
429
430
431 /* fills in the selected faces with the current weight and vertex group */
432 void wpaint_fill(VPaint *wp, Object *ob, float paintweight)
433 {
434         Mesh *me;
435         MFace *mface;
436         MDeformWeight *dw, *uw;
437         int *indexar;
438         int index, vgroup;
439         unsigned int faceverts[5]={0,0,0,0,0};
440         unsigned char i;
441         int vgroup_mirror= -1;
442         int selected;
443         
444         me= ob->data;
445         if(me==0 || me->totface==0 || me->dvert==0 || !me->mface) return;
446         
447         selected= (me->editflag & ME_EDIT_PAINT_MASK);
448
449         indexar= get_indexarray();
450         if(selected) {
451                 for(index=0, mface=me->mface; index<me->totface; index++, mface++) {
452                         if((mface->flag & ME_FACE_SEL)==0)
453                                 indexar[index]= 0;
454                         else
455                                 indexar[index]= index+1;
456                 }
457         }
458         else {
459                 for(index=0; index<me->totface; index++)
460                         indexar[index]= index+1;
461         }
462         
463         vgroup= ob->actdef-1;
464         
465         /* directly copied from weight_paint, should probaby split into a separate function */
466         /* if mirror painting, find the other group */          
467         if(me->editflag & ME_EDIT_MIRROR_X) {
468                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
469                 if(defgroup) {
470                         bDeformGroup *curdef;
471                         int actdef= 0;
472                         char name[32];
473
474                         BLI_strncpy(name, defgroup->name, 32);
475                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
476                         
477                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
478                                 if (!strcmp(curdef->name, name))
479                                         break;
480                         if(curdef==NULL) {
481                                 int olddef= ob->actdef; /* tsk, ED_vgroup_add sets the active defgroup */
482                                 curdef= ED_vgroup_add_name (ob, name);
483                                 ob->actdef= olddef;
484                         }
485                         
486                         if(curdef && curdef!=defgroup)
487                                 vgroup_mirror= actdef;
488                 }
489         }
490         /* end copy from weight_paint*/
491         
492         copy_wpaint_prev(wp, me->dvert, me->totvert);
493         
494         for(index=0; index<me->totface; index++) {
495                 if(indexar[index] && indexar[index]<=me->totface) {
496                         mface= me->mface + (indexar[index]-1);
497                         /* just so we can loop through the verts */
498                         faceverts[0]= mface->v1;
499                         faceverts[1]= mface->v2;
500                         faceverts[2]= mface->v3;
501                         faceverts[3]= mface->v4;
502                         for (i=0; i<3 || faceverts[i]; i++) {
503                                 if(!((me->dvert+faceverts[i])->flag)) {
504                                         dw= ED_vgroup_weight_verify(me->dvert+faceverts[i], vgroup);
505                                         if(dw) {
506                                                 uw= ED_vgroup_weight_verify(wp->wpaint_prev+faceverts[i], vgroup);
507                                                 uw->weight= dw->weight; /* set the undo weight */
508                                                 dw->weight= paintweight;
509                                                 
510                                                 if(me->editflag & ME_EDIT_MIRROR_X) {   /* x mirror painting */
511                                                         int j= mesh_get_x_mirror_vert(ob, faceverts[i]);
512                                                         if(j>=0) {
513                                                                 /* copy, not paint again */
514                                                                 if(vgroup_mirror != -1) {
515                                                                         dw= ED_vgroup_weight_verify(me->dvert+j, vgroup_mirror);
516                                                                         uw= ED_vgroup_weight_verify(wp->wpaint_prev+j, vgroup_mirror);
517                                                                 } else {
518                                                                         dw= ED_vgroup_weight_verify(me->dvert+j, vgroup);
519                                                                         uw= ED_vgroup_weight_verify(wp->wpaint_prev+j, vgroup);
520                                                                 }
521                                                                 uw->weight= dw->weight; /* set the undo weight */
522                                                                 dw->weight= paintweight;
523                                                         }
524                                                 }
525                                         }
526                                         (me->dvert+faceverts[i])->flag= 1;
527                                 }
528                         }
529                 }
530         }
531         
532         index=0;
533         while (index<me->totvert) {
534                 (me->dvert+index)->flag= 0;
535                 index++;
536         }
537         
538         MEM_freeN(indexar);
539         copy_wpaint_prev(wp, NULL, 0);
540
541         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
542 }
543
544 /* XXX: should be re-implemented as a vertex/weight paint 'colour correct' operator
545  
546 void vpaint_dogamma(Scene *scene)
547 {
548         VPaint *vp= scene->toolsettings->vpaint;
549         Mesh *me;
550         Object *ob;
551         float igam, fac;
552         int a, temp;
553         unsigned char *cp, gamtab[256];
554
555         ob= OBACT;
556         me= get_mesh(ob);
557
558         if(!(ob->mode & OB_MODE_VERTEX_PAINT)) return;
559         if(me==0 || me->mcol==0 || me->totface==0) return;
560
561         igam= 1.0/vp->gamma;
562         for(a=0; a<256; a++) {
563                 
564                 fac= ((float)a)/255.0;
565                 fac= vp->mul*pow( fac, igam);
566                 
567                 temp= 255.9*fac;
568                 
569                 if(temp<=0) gamtab[a]= 0;
570                 else if(temp>=255) gamtab[a]= 255;
571                 else gamtab[a]= temp;
572         }
573
574         a= 4*me->totface;
575         cp= (unsigned char *)me->mcol;
576         while(a--) {
577                 
578                 cp[1]= gamtab[ cp[1] ];
579                 cp[2]= gamtab[ cp[2] ];
580                 cp[3]= gamtab[ cp[3] ];
581                 
582                 cp+= 4;
583         }
584 }
585  */
586
587 static unsigned int mcol_blend(unsigned int col1, unsigned int col2, int fac)
588 {
589         char *cp1, *cp2, *cp;
590         int mfac;
591         unsigned int col=0;
592         
593         if(fac==0) return col1;
594         if(fac>=255) return col2;
595
596         mfac= 255-fac;
597         
598         cp1= (char *)&col1;
599         cp2= (char *)&col2;
600         cp=  (char *)&col;
601         
602         cp[0]= 255;
603         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
604         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
605         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
606         
607         return col;
608 }
609
610 static unsigned int mcol_add(unsigned int col1, unsigned int col2, int fac)
611 {
612         char *cp1, *cp2, *cp;
613         int temp;
614         unsigned int col=0;
615         
616         if(fac==0) return col1;
617         
618         cp1= (char *)&col1;
619         cp2= (char *)&col2;
620         cp=  (char *)&col;
621         
622         cp[0]= 255;
623         temp= cp1[1] + ((fac*cp2[1])/255);
624         if(temp>254) cp[1]= 255; else cp[1]= temp;
625         temp= cp1[2] + ((fac*cp2[2])/255);
626         if(temp>254) cp[2]= 255; else cp[2]= temp;
627         temp= cp1[3] + ((fac*cp2[3])/255);
628         if(temp>254) cp[3]= 255; else cp[3]= temp;
629         
630         return col;
631 }
632
633 static unsigned int mcol_sub(unsigned int col1, unsigned int col2, int fac)
634 {
635         char *cp1, *cp2, *cp;
636         int temp;
637         unsigned int col=0;
638         
639         if(fac==0) return col1;
640         
641         cp1= (char *)&col1;
642         cp2= (char *)&col2;
643         cp=  (char *)&col;
644         
645         cp[0]= 255;
646         temp= cp1[1] - ((fac*cp2[1])/255);
647         if(temp<0) cp[1]= 0; else cp[1]= temp;
648         temp= cp1[2] - ((fac*cp2[2])/255);
649         if(temp<0) cp[2]= 0; else cp[2]= temp;
650         temp= cp1[3] - ((fac*cp2[3])/255);
651         if(temp<0) cp[3]= 0; else cp[3]= temp;
652         
653         return col;
654 }
655
656 static unsigned int mcol_mul(unsigned int col1, unsigned int col2, int fac)
657 {
658         char *cp1, *cp2, *cp;
659         int mfac;
660         unsigned int col=0;
661         
662         if(fac==0) return col1;
663
664         mfac= 255-fac;
665         
666         cp1= (char *)&col1;
667         cp2= (char *)&col2;
668         cp=  (char *)&col;
669         
670         /* first mul, then blend the fac */
671         cp[0]= 255;
672         cp[1]= (mfac*cp1[1] + fac*((cp2[1]*cp1[1])/255)  )/255;
673         cp[2]= (mfac*cp1[2] + fac*((cp2[2]*cp1[2])/255)  )/255;
674         cp[3]= (mfac*cp1[3] + fac*((cp2[3]*cp1[3])/255)  )/255;
675
676         
677         return col;
678 }
679
680 static unsigned int mcol_lighten(unsigned int col1, unsigned int col2, int fac)
681 {
682         char *cp1, *cp2, *cp;
683         int mfac;
684         unsigned int col=0;
685         
686         if(fac==0) return col1;
687         if(fac>=255) return col2;
688
689         mfac= 255-fac;
690         
691         cp1= (char *)&col1;
692         cp2= (char *)&col2;
693         cp=  (char *)&col;
694         
695         /* See if are lighter, if so mix, else dont do anything.
696         if the paint col is darker then the original, then ignore */
697         if (cp1[1]+cp1[2]+cp1[3] > cp2[1]+cp2[2]+cp2[3])
698                 return col1;
699         
700         cp[0]= 255;
701         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
702         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
703         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
704         
705         return col;
706 }
707
708 static unsigned int mcol_darken(unsigned int col1, unsigned int col2, int fac)
709 {
710         char *cp1, *cp2, *cp;
711         int mfac;
712         unsigned int col=0;
713         
714         if(fac==0) return col1;
715         if(fac>=255) return col2;
716
717         mfac= 255-fac;
718         
719         cp1= (char *)&col1;
720         cp2= (char *)&col2;
721         cp=  (char *)&col;
722         
723         /* See if were darker, if so mix, else dont do anything.
724         if the paint col is brighter then the original, then ignore */
725         if (cp1[1]+cp1[2]+cp1[3] < cp2[1]+cp2[2]+cp2[3])
726                 return col1;
727         
728         cp[0]= 255;
729         cp[1]= (mfac*cp1[1]+fac*cp2[1])/255;
730         cp[2]= (mfac*cp1[2]+fac*cp2[2])/255;
731         cp[3]= (mfac*cp1[3]+fac*cp2[3])/255;
732         return col;
733 }
734
735 static void vpaint_blend(VPaint *vp, unsigned int *col, unsigned int *colorig, unsigned int paintcol, int alpha)
736 {
737         Brush *brush = paint_brush(&vp->paint);
738
739         if(vp->mode==VP_MIX || vp->mode==VP_BLUR) *col= mcol_blend( *col, paintcol, alpha);
740         else if(vp->mode==VP_ADD) *col= mcol_add( *col, paintcol, alpha);
741         else if(vp->mode==VP_SUB) *col= mcol_sub( *col, paintcol, alpha);
742         else if(vp->mode==VP_MUL) *col= mcol_mul( *col, paintcol, alpha);
743         else if(vp->mode==VP_LIGHTEN) *col= mcol_lighten( *col, paintcol, alpha);
744         else if(vp->mode==VP_DARKEN) *col= mcol_darken( *col, paintcol, alpha);
745         
746         /* if no spray, clip color adding with colorig & orig alpha */
747         if((vp->flag & VP_SPRAY)==0) {
748                 unsigned int testcol=0, a;
749                 char *cp, *ct, *co;
750                 
751                 alpha= (int)(255.0*brush->alpha);
752                 
753                 if(vp->mode==VP_MIX || vp->mode==VP_BLUR) testcol= mcol_blend( *colorig, paintcol, alpha);
754                 else if(vp->mode==VP_ADD) testcol= mcol_add( *colorig, paintcol, alpha);
755                 else if(vp->mode==VP_SUB) testcol= mcol_sub( *colorig, paintcol, alpha);
756                 else if(vp->mode==VP_MUL) testcol= mcol_mul( *colorig, paintcol, alpha);
757                 else if(vp->mode==VP_LIGHTEN)  testcol= mcol_lighten( *colorig, paintcol, alpha);
758                 else if(vp->mode==VP_DARKEN)   testcol= mcol_darken( *colorig, paintcol, alpha);
759                 
760                 cp= (char *)col;
761                 ct= (char *)&testcol;
762                 co= (char *)colorig;
763                 
764                 for(a=0; a<4; a++) {
765                         if( ct[a]<co[a] ) {
766                                 if( cp[a]<ct[a] ) cp[a]= ct[a];
767                                 else if( cp[a]>co[a] ) cp[a]= co[a];
768                         }
769                         else {
770                                 if( cp[a]<co[a] ) cp[a]= co[a];
771                                 else if( cp[a]>ct[a] ) cp[a]= ct[a];
772                         }
773                 }
774         }
775 }
776
777
778 static int sample_backbuf_area(ViewContext *vc, int *indexar, int totface, int x, int y, float size)
779 {
780         struct ImBuf *ibuf;
781         int a, tot=0, index;
782         
783         if(totface+4>=MAXINDEX) return 0;
784         
785         /* brecht: disabled this because it obviously failes for
786            brushes with size > 64, why is this here? */
787         /*if(size>64.0) size= 64.0;*/
788         
789         ibuf= view3d_read_backbuf(vc, x-size, y-size, x+size, y+size);
790         if(ibuf) {
791                 unsigned int *rt= ibuf->rect;
792
793                 memset(indexar, 0, sizeof(int)*totface+4);      /* plus 2! first element is total, +2 was giving valgrind errors, +4 seems ok */
794                 
795                 size= ibuf->x*ibuf->y;
796                 while(size--) {
797                                 
798                         if(*rt) {
799                                 index= WM_framebuffer_to_index(*rt);
800                                 if(index>0 && index<=totface)
801                                         indexar[index] = 1;
802                         }
803                 
804                         rt++;
805                 }
806                 
807                 for(a=1; a<=totface; a++) {
808                         if(indexar[a]) indexar[tot++]= a;
809                 }
810
811                 IMB_freeImBuf(ibuf);
812         }
813         
814         return tot;
815 }
816
817 static float calc_vp_alpha_dl(VPaint *vp, ViewContext *vc, float vpimat[][3], float *vert_nor, float *mval, float pressure)
818 {
819         Brush *brush = paint_brush(&vp->paint);
820         float fac, fac_2, size, dx, dy;
821         float alpha;
822         short vertco[2];
823         
824         project_short_noclip(vc->ar, vert_nor, vertco);
825         dx= mval[0]-vertco[0];
826         dy= mval[1]-vertco[1];
827         
828         if (brush->flag & BRUSH_SIZE_PRESSURE)
829                 size = pressure * brush->size;
830         else
831                 size = brush->size;
832         
833         fac_2= dx*dx + dy*dy;
834         if(fac_2 > size*size) return 0.f;
835         fac = sqrtf(fac_2);
836         
837         alpha= brush->alpha * brush_curve_strength_clamp(brush, fac, size);
838         
839         if (brush->flag & BRUSH_ALPHA_PRESSURE)
840                 alpha *= pressure;
841                 
842         if(vp->flag & VP_NORMALS) {
843                 float *no= vert_nor+3;
844                 
845                 /* transpose ! */
846                 fac= vpimat[2][0]*no[0]+vpimat[2][1]*no[1]+vpimat[2][2]*no[2];
847                 if(fac>0.0) {
848                         dx= vpimat[0][0]*no[0]+vpimat[0][1]*no[1]+vpimat[0][2]*no[2];
849                         dy= vpimat[1][0]*no[0]+vpimat[1][1]*no[1]+vpimat[1][2]*no[2];
850                         
851                         alpha*= fac/sqrtf(dx*dx + dy*dy + fac*fac);
852                 }
853                 else return 0.f;
854         }
855         
856         return alpha;
857 }
858
859 static void wpaint_blend(VPaint *wp, MDeformWeight *dw, MDeformWeight *uw, float alpha, float paintval, int flip)
860 {
861         Brush *brush = paint_brush(&wp->paint);
862         int mode = wp->mode;
863         
864         if(dw==NULL || uw==NULL) return;
865         
866         if (flip) {
867                 switch(mode) {
868                         case VP_MIX:
869                                 paintval = 1.f - paintval; break;
870                         case VP_ADD:
871                                 mode= VP_SUB; break;
872                         case VP_SUB:
873                                 mode= VP_ADD; break;
874                         case VP_LIGHTEN:
875                                 mode= VP_DARKEN; break;
876                         case VP_DARKEN:
877                                 mode= VP_LIGHTEN; break;
878                 }
879         }
880         
881         if(mode==VP_MIX || mode==VP_BLUR)
882                 dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
883         else if(mode==VP_ADD)
884                 dw->weight += paintval*alpha;
885         else if(mode==VP_SUB) 
886                 dw->weight -= paintval*alpha;
887         else if(mode==VP_MUL) 
888                 /* first mul, then blend the fac */
889                 dw->weight = ((1.0-alpha) + alpha*paintval)*dw->weight;
890         else if(mode==VP_LIGHTEN) {
891                 if (dw->weight < paintval)
892                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
893         } else if(mode==VP_DARKEN) {
894                 if (dw->weight > paintval)
895                         dw->weight = paintval*alpha + dw->weight*(1.0-alpha);
896         }
897         CLAMP(dw->weight, 0.0f, 1.0f);
898         
899         /* if no spray, clip result with orig weight & orig alpha */
900         if((wp->flag & VP_SPRAY)==0) {
901                 float testw=0.0f;
902                 
903                 alpha= brush->alpha;
904                 if(mode==VP_MIX || mode==VP_BLUR)
905                         testw = paintval*alpha + uw->weight*(1.0-alpha);
906                 else if(mode==VP_ADD)
907                         testw = uw->weight + paintval*alpha;
908                 else if(mode==VP_SUB) 
909                         testw = uw->weight - paintval*alpha;
910                 else if(mode==VP_MUL) 
911                         /* first mul, then blend the fac */
912                         testw = ((1.0-alpha) + alpha*paintval)*uw->weight;              
913                 else if(mode==VP_LIGHTEN) {
914                         if (uw->weight < paintval)
915                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
916                         else
917                                 testw = uw->weight;
918                 } else if(mode==VP_DARKEN) {
919                         if (uw->weight > paintval)
920                                 testw = paintval*alpha + uw->weight*(1.0-alpha);
921                         else
922                                 testw = uw->weight;
923                 }
924                 CLAMP(testw, 0.0f, 1.0f);
925                 
926                 if( testw<uw->weight ) {
927                         if(dw->weight < testw) dw->weight= testw;
928                         else if(dw->weight > uw->weight) dw->weight= uw->weight;
929                 }
930                 else {
931                         if(dw->weight > testw) dw->weight= testw;
932                         else if(dw->weight < uw->weight) dw->weight= uw->weight;
933                 }
934         }
935         
936 }
937
938 /* ----------------------------------------------------- */
939
940 /* used for 3d view, on active object, assumes me->dvert exists */
941 /* if mode==1: */
942 /*     samples cursor location, and gives menu with vertex groups to activate */
943 /* else */
944 /*     sets wp->weight to the closest weight value to vertex */
945 /*     note: we cant sample frontbuf, weight colors are interpolated too unpredictable */
946 #if 0
947 void sample_wpaint(Scene *scene, ARegion *ar, View3D *v3d, int mode)
948 {
949         ViewContext vc;
950         ToolSettings *ts= scene->toolsettings;
951         Object *ob= OBACT;
952         Mesh *me= get_mesh(ob);
953         int index, i;
954         short mval[2] = {0, 0}, sco[2];
955
956         if (!me) return;
957         
958 //      getmouseco_areawin(mval);
959         index= view3d_sample_backbuf(&vc, mval[0], mval[1]);
960         
961         if(index && index<=me->totpoly) {
962                 MPoly *mpoly;
963                 
964                 mpoly= ((MPoly *)me->mpoly) + index-1;          
965                 if(mode==1) {   /* sampe which groups are in here */
966                         MDeformVert *dv;
967                         MLoop *ml;
968                         int a, totgroup;
969                         
970                         totgroup= BLI_countlist(&ob->defbase);
971                         if(totgroup) {
972                                 int totmenu=0;
973                                 int *groups=MEM_callocN(totgroup*sizeof(int), "groups");
974                                 
975                                 ml = me->mloop+mpoly->loopstart;
976                                 for (i=0; i<mpoly->totloop; i++, ml++) {
977                                         dv = me->dvert+ml->v;
978                                         if (dv->dw[a].def_nr<totgroup)
979                                                 groups[dv->dw[a].def_nr]= 1;
980                                         
981                                 }
982
983                                 for(a=0; a<totgroup; a++)
984                                         if(groups[a]) totmenu++;
985                                 
986                                 if(totmenu==0) {
987                                         //notice("No Vertex Group Selected");
988                                 }
989                                 else {
990                                         bDeformGroup *dg;
991                                         short val;
992                                         char item[40], *str= MEM_mallocN(40*totmenu+40, "menu");
993                                         
994                                         strcpy(str, "Vertex Groups %t");
995                                         for(a=0, dg=ob->defbase.first; dg && a<totgroup; a++, dg= dg->next) {
996                                                 if(groups[a]) {
997                                                         sprintf(item, "|%s %%x%d", dg->name, a);
998                                                         strcat(str, item);
999                                                 }
1000                                         }
1001                                         
1002                                         val= 0; // XXX pupmenu(str);
1003                                         if(val>=0) {
1004                                                 ob->actdef= val+1;
1005                                                 DAG_id_flush_update(&me->id, OB_RECALC_DATA);
1006                                         }
1007                                         MEM_freeN(str);
1008                                 }
1009                                 MEM_freeN(groups);
1010                         }
1011 //                      else notice("No Vertex Groups in Object");
1012                 }
1013                 else {
1014                         DerivedMesh *dm;
1015                         MDeformWeight *dw;
1016                         MLoop *ml;
1017                         float w, co[3], fmin = 10000000.0f;
1018                         int loopi = mpoly->loopstart;
1019                         
1020                         dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
1021                         if(dm->getVertCo==NULL) {
1022                                 //notice("Not supported yet");
1023                         }
1024                         else {
1025                                 /* find minimum distance */
1026                                 ml = me->mloop + mpoly->loopstart;
1027                                 for (i=0; i<mpoly->totloop; i++, ml++) {
1028                                         dm->getVertCo(dm, ml->v, co);
1029                                         project_short_noclip(ar, co, sco);
1030                                         w4= ((mval[0]-sco[0])*(mval[0]-sco[0]) + (mval[1]-sco[1])*(mval[1]-sco[1]));
1031                                 }
1032                                 else w4= 1.0e10;
1033                                 
1034                                 fac= MIN4(w1, w2, w3, w4);
1035                                 if(w1==fac) {
1036                                         dw= ED_vgroup_weight_get(me->dvert+mface->v1, ob->actdef-1);
1037                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
1038                                 }
1039                                 else if(w2==fac) {
1040                                         dw= ED_vgroup_weight_get(me->dvert+mface->v2, ob->actdef-1);
1041                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
1042                                 }
1043                                 else if(w3==fac) {
1044                                         dw= ED_vgroup_weight_get(me->dvert+mface->v3, ob->actdef-1);
1045                                         if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
1046                                 }
1047                                 else if(w4==fac) {
1048                                         if(mface->v4) {
1049                                                 dw= ED_vgroup_weight_get(me->dvert+mface->v4, ob->actdef-1);
1050                                                 if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
1051 >>>>>>> .merge-right.r23146
1052                                         }
1053                                 }
1054
1055                                 ml = me->mloop + loopi;
1056                                 dw= get_defweight(me->dvert+ml->v, ob->actdef-1);
1057                                 if(dw) ts->vgroup_weight= dw->weight; else ts->vgroup_weight= 0.0f;
1058                         }
1059                         dm->release(dm);
1060                 }               
1061                 
1062         }
1063         
1064 }
1065 #endif
1066
1067 static void do_weight_paint_auto_normalize(MDeformVert *dvert, 
1068                                            int paint_nr, char *map)
1069 {
1070 //      MDeformWeight *dw = dvert->dw;
1071         float sum=0.0f, fac=0.0f, paintw=0.0f;
1072         int i, tot=0;
1073
1074         if (!map)
1075                 return;
1076
1077         for (i=0; i<dvert->totweight; i++) {
1078                 if (dvert->dw[i].def_nr == paint_nr)
1079                         paintw = dvert->dw[i].weight;
1080
1081                 if (map[dvert->dw[i].def_nr]) {
1082                         tot += 1;
1083                         if (dvert->dw[i].def_nr != paint_nr)
1084                                 sum += dvert->dw[i].weight;
1085                 }
1086         }
1087         
1088         if (!tot || sum <= (1.0f - paintw))
1089                 return;
1090
1091         fac = sum / (1.0f - paintw);
1092         fac = fac==0.0f ? 1.0f : 1.0f / fac;
1093
1094         for (i=0; i<dvert->totweight; i++) {
1095                 if (map[dvert->dw[i].def_nr]) {
1096                         if (dvert->dw[i].def_nr != paint_nr)
1097                                 dvert->dw[i].weight *= fac;
1098                 }
1099         }
1100 }
1101
1102 static void do_weight_paint_vertex(VPaint *wp, Object *ob, int index, 
1103                                    float alpha, float paintweight, int flip, 
1104                                    int vgroup_mirror, char *validmap)
1105 {
1106         Mesh *me= ob->data;
1107         MDeformWeight *dw, *uw;
1108         int vgroup= ob->actdef-1;
1109         
1110         if(wp->flag & VP_ONLYVGROUP) {
1111                 dw= ED_vgroup_weight_get(me->dvert+index, vgroup);
1112                 uw= ED_vgroup_weight_get(wp->wpaint_prev+index, vgroup);
1113         }
1114         else {
1115                 dw= ED_vgroup_weight_verify(me->dvert+index, vgroup);
1116                 uw= ED_vgroup_weight_verify(wp->wpaint_prev+index, vgroup);
1117         }
1118         if(dw==NULL || uw==NULL)
1119                 return;
1120         
1121         wpaint_blend(wp, dw, uw, alpha, paintweight, flip);
1122         do_weight_paint_auto_normalize(me->dvert+index, vgroup, validmap);
1123
1124         if(me->editflag & ME_EDIT_MIRROR_X) {   /* x mirror painting */
1125                 int j= mesh_get_x_mirror_vert(ob, index);
1126                 if(j>=0) {
1127                         /* copy, not paint again */
1128                         if(vgroup_mirror != -1)
1129                                 uw= ED_vgroup_weight_verify(me->dvert+j, vgroup_mirror);
1130                         else
1131                                 uw= ED_vgroup_weight_verify(me->dvert+j, vgroup);
1132                                 
1133                         uw->weight= dw->weight;
1134
1135                         do_weight_paint_auto_normalize(me->dvert+j, vgroup, validmap);
1136                 }
1137         }
1138 }
1139
1140
1141 /* *************** set wpaint operator ****************** */
1142
1143 static int set_wpaint(bContext *C, wmOperator *op)              /* toggle */
1144 {               
1145         Object *ob= CTX_data_active_object(C);
1146         Scene *scene= CTX_data_scene(C);
1147         VPaint *wp= scene->toolsettings->wpaint;
1148         Mesh *me;
1149         
1150         me= get_mesh(ob);
1151         if(ob->id.lib || me==NULL) return OPERATOR_PASS_THROUGH;
1152         
1153         if(me && me->totface>=MAXINDEX) {
1154                 error("Maximum number of faces: %d", MAXINDEX-1);
1155                 ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1156                 return OPERATOR_CANCELLED;
1157         }
1158         
1159         if(ob->mode & OB_MODE_WEIGHT_PAINT) ob->mode &= ~OB_MODE_WEIGHT_PAINT;
1160         else ob->mode |= OB_MODE_WEIGHT_PAINT;
1161         
1162         
1163         /* Weightpaint works by overriding colors in mesh,
1164                 * so need to make sure we recalc on enter and
1165                 * exit (exit needs doing regardless because we
1166                                 * should redeform).
1167                 */
1168         DAG_id_flush_update(&me->id, OB_RECALC_DATA);
1169         
1170         if(ob->mode & OB_MODE_WEIGHT_PAINT) {
1171                 Object *par;
1172                 
1173                 if(wp==NULL)
1174                         wp= scene->toolsettings->wpaint= new_vpaint(1);
1175
1176                 paint_init(&wp->paint, PAINT_CURSOR_WEIGHT_PAINT);
1177                 paint_cursor_start(C, weight_paint_poll);
1178                 
1179                 mesh_octree_table(ob, NULL, NULL, 's');
1180                 
1181                 /* verify if active weight group is also active bone */
1182                 par= modifiers_isDeformedByArmature(ob);
1183                 if(par && (par->mode & OB_MODE_POSE)) {
1184                         bArmature *arm= ob->data;
1185
1186                         if(arm->act_bone)
1187                                 ED_vgroup_select_by_name(ob, arm->act_bone->name);
1188                 }
1189         }
1190         else {
1191                 mesh_octree_table(ob, NULL, NULL, 'e');
1192         }
1193         
1194         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1195         
1196         return OPERATOR_FINISHED;
1197 }
1198
1199 /* for switching to/from mode */
1200 static int paint_poll_test(bContext *C)
1201 {
1202         if(ED_operator_view3d_active(C)==0)
1203                 return 0;
1204         if(CTX_data_edit_object(C))
1205                 return 0;
1206         if(CTX_data_active_object(C)==NULL)
1207                 return 0;
1208         return 1;
1209 }
1210
1211 void PAINT_OT_weight_paint_toggle(wmOperatorType *ot)
1212 {
1213         
1214         /* identifiers */
1215         ot->name= "Weight Paint Mode";
1216         ot->idname= "PAINT_OT_weight_paint_toggle";
1217         
1218         /* api callbacks */
1219         ot->exec= set_wpaint;
1220         ot->poll= paint_poll_test;
1221         
1222         /* flags */
1223         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1224         
1225 }
1226
1227 /* ************ paint radial controls *************/
1228
1229 static int vpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1230 {
1231         Paint *p = paint_get_active(CTX_data_scene(C));
1232         Brush *brush = paint_brush(p);
1233         
1234         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1235         p->paint_cursor = NULL;
1236         brush_radial_control_invoke(op, brush, 1);
1237         return WM_radial_control_invoke(C, op, event);
1238 }
1239
1240 static int vpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1241 {
1242         int ret = WM_radial_control_modal(C, op, event);
1243         if(ret != OPERATOR_RUNNING_MODAL)
1244                 paint_cursor_start(C, vertex_paint_poll);
1245         return ret;
1246 }
1247
1248 static int vpaint_radial_control_exec(bContext *C, wmOperator *op)
1249 {
1250         Brush *brush = paint_brush(&CTX_data_scene(C)->toolsettings->vpaint->paint);
1251         int ret = brush_radial_control_exec(op, brush, 1);
1252         
1253         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1254         
1255         return ret;
1256 }
1257
1258 static int wpaint_radial_control_invoke(bContext *C, wmOperator *op, wmEvent *event)
1259 {
1260         Paint *p = paint_get_active(CTX_data_scene(C));
1261         Brush *brush = paint_brush(p);
1262         
1263         WM_paint_cursor_end(CTX_wm_manager(C), p->paint_cursor);
1264         p->paint_cursor = NULL;
1265         brush_radial_control_invoke(op, brush, 1);
1266         return WM_radial_control_invoke(C, op, event);
1267 }
1268
1269 static int wpaint_radial_control_modal(bContext *C, wmOperator *op, wmEvent *event)
1270 {
1271         int ret = WM_radial_control_modal(C, op, event);
1272         if(ret != OPERATOR_RUNNING_MODAL)
1273                 paint_cursor_start(C, weight_paint_poll);
1274         return ret;
1275 }
1276
1277 static int wpaint_radial_control_exec(bContext *C, wmOperator *op)
1278 {
1279         Brush *brush = paint_brush(&CTX_data_scene(C)->toolsettings->wpaint->paint);
1280         int ret = brush_radial_control_exec(op, brush, 1);
1281         
1282         WM_event_add_notifier(C, NC_BRUSH|NA_EDITED, brush);
1283         
1284         return ret;
1285 }
1286
1287 void PAINT_OT_weight_paint_radial_control(wmOperatorType *ot)
1288 {
1289         WM_OT_radial_control_partial(ot);
1290
1291         ot->name= "Weight Paint Radial Control";
1292         ot->idname= "PAINT_OT_weight_paint_radial_control";
1293
1294         ot->invoke= wpaint_radial_control_invoke;
1295         ot->modal= wpaint_radial_control_modal;
1296         ot->exec= wpaint_radial_control_exec;
1297         ot->poll= weight_paint_poll;
1298         
1299         /* flags */
1300         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1301 }
1302
1303 void PAINT_OT_vertex_paint_radial_control(wmOperatorType *ot)
1304 {
1305         WM_OT_radial_control_partial(ot);
1306
1307         ot->name= "Vertex Paint Radial Control";
1308         ot->idname= "PAINT_OT_vertex_paint_radial_control";
1309
1310         ot->invoke= vpaint_radial_control_invoke;
1311         ot->modal= vpaint_radial_control_modal;
1312         ot->exec= vpaint_radial_control_exec;
1313         ot->poll= vertex_paint_poll;
1314         
1315         /* flags */
1316         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1317 }
1318
1319 /* ************ weight paint operator ********** */
1320
1321 struct WPaintData {
1322         ViewContext vc;
1323         int *indexar;
1324         int vgroup_mirror;
1325         float *vertexcosnos;
1326         float wpimat[3][3];
1327         
1328         /*variables for auto normalize*/
1329         int auto_normalize;
1330         char *vgroup_validmap; /*stores if vgroups tie to deforming bones or not*/
1331 };
1332
1333 static char *wpaint_make_validmap(Mesh *me, Object *ob)
1334 {
1335         bDeformGroup *dg;
1336         ModifierData *md;
1337         char *validmap;
1338         bPose *pose;
1339         bPoseChannel *chan;
1340         ArmatureModifierData *amd;
1341         GHash *gh = BLI_ghash_new(BLI_ghashutil_strhash, BLI_ghashutil_strcmp);
1342         int i = 0, step1=1;
1343
1344         /*add all names to a hash table*/
1345         for (dg=ob->defbase.first, i=0; dg; dg=dg->next, i++) {
1346                 BLI_ghash_insert(gh, dg->name, NULL);
1347         }
1348
1349         if (!i)
1350                 return NULL;
1351
1352         validmap = MEM_callocN(i, "wpaint valid map");
1353
1354         /*now loop through the armature modifiers and identify deform bones*/
1355         for (md = ob->modifiers.first; md; md= !md->next && step1 ? (step1=0), modifiers_getVirtualModifierList(ob) : md->next) {
1356                 if (!(md->mode & (eModifierMode_Realtime|eModifierMode_Virtual)))
1357                         continue;
1358
1359                 if (md->type == eModifierType_Armature) 
1360                 {
1361                         amd = (ArmatureModifierData*) md;
1362                         pose = amd->object->pose;
1363                         
1364                         for (chan=pose->chanbase.first; chan; chan=chan->next) {
1365                                 if (chan->bone->flag & BONE_NO_DEFORM)
1366                                         continue;
1367
1368                                 if (BLI_ghash_haskey(gh, chan->name)) {
1369                                         BLI_ghash_remove(gh, chan->name, NULL, NULL);
1370                                         BLI_ghash_insert(gh, chan->name, SET_INT_IN_POINTER(1));
1371                                 }
1372                         }
1373                 }
1374         }
1375         
1376         /*add all names to a hash table*/
1377         for (dg=ob->defbase.first, i=0; dg; dg=dg->next, i++) {
1378                 if (BLI_ghash_lookup(gh, dg->name) != NULL) {
1379                         validmap[i] = 1;
1380                 }
1381         }
1382
1383         BLI_ghash_free(gh, NULL, NULL);
1384
1385         return validmap;
1386 }
1387
1388 static int wpaint_stroke_test_start(bContext *C, wmOperator *op, wmEvent *event)
1389 {
1390         Scene *scene= CTX_data_scene(C);
1391         struct PaintStroke *stroke = op->customdata;
1392         ToolSettings *ts= CTX_data_tool_settings(C);
1393         VPaint *wp= ts->wpaint;
1394         Object *ob= CTX_data_active_object(C);
1395         struct WPaintData *wpd;
1396         Mesh *me;
1397         float mat[4][4], imat[4][4];
1398         
1399         if(scene->obedit) return OPERATOR_CANCELLED;
1400         
1401         me= get_mesh(ob);
1402         if(me==NULL || me->totface==0) return OPERATOR_PASS_THROUGH;
1403         
1404         /*we can't assume mfaces have a correct origindex layer that indices to mpolys.
1405           so instead we have to regenerate the tesselation faces altogether.*/
1406         me->totface = mesh_recalcTesselation(&me->fdata, &me->ldata, &me->pdata, 
1407                 me->mvert, me->totface, me->totloop, me->totpoly, 1);
1408         mesh_update_customdata_pointers(me);
1409         makeDerivedMesh(scene, ob, NULL, CD_MASK_BAREMESH);
1410
1411         /* if nothing was added yet, we make dverts and a vertex deform group */
1412         if (!me->dvert)
1413                 ED_vgroup_data_create(&me->id);
1414         
1415         /* make mode data storage */
1416         wpd= MEM_callocN(sizeof(struct WPaintData), "WPaintData");
1417         paint_stroke_set_mode_data(stroke, wpd);
1418         view3d_set_viewcontext(C, &wpd->vc);
1419         wpd->vgroup_mirror= -1;
1420         
1421         /*set up auto-normalize, and generate map for detecting which
1422           vgroups affect deform bones*/
1423         wpd->auto_normalize = ts->auto_normalize;
1424         if (wpd->auto_normalize)
1425                 wpd->vgroup_validmap = wpaint_make_validmap(me, ob);
1426         
1427         //      if(qual & LR_CTRLKEY) {
1428         //              sample_wpaint(scene, ar, v3d, 0);
1429         //              return;
1430         //      }
1431         //      if(qual & LR_SHIFTKEY) {
1432         //              sample_wpaint(scene, ar, v3d, 1);
1433         //              return;
1434         //      }
1435         
1436         /* ALLOCATIONS! no return after this line */
1437         /* painting on subsurfs should give correct points too, this returns me->totvert amount */
1438         wpd->vertexcosnos= mesh_get_mapped_verts_nors(scene, ob);
1439         wpd->indexar= get_indexarray();
1440         copy_wpaint_prev(wp, me->dvert, me->totvert);
1441         
1442         /* this happens on a Bone select, when no vgroup existed yet */
1443         if(ob->actdef<=0) {
1444                 Object *modob;
1445                 if((modob = modifiers_isDeformedByArmature(ob))) {
1446                         bPoseChannel *pchan;
1447                         for(pchan= modob->pose->chanbase.first; pchan; pchan= pchan->next)
1448                                 if(pchan->bone->flag & SELECT)
1449                                         break;
1450                         if(pchan) {
1451                                 bDeformGroup *dg= get_named_vertexgroup(ob, pchan->name);
1452                                 if(dg==NULL)
1453                                         dg= ED_vgroup_add_name(ob, pchan->name);        /* sets actdef */
1454                                 else
1455                                         ob->actdef= get_defgroup_num(ob, dg);
1456                         }
1457                 }
1458         }
1459         if(ob->defbase.first==NULL) {
1460                 ED_vgroup_add(ob);
1461         }       
1462         
1463         //      if(ob->lay & v3d->lay); else error("Active object is not in this layer");
1464         
1465         /* imat for normals */
1466         mul_m4_m4m4(mat, ob->obmat, wpd->vc.rv3d->viewmat);
1467         invert_m4_m4(imat, mat);
1468         copy_m3_m4(wpd->wpimat, imat);
1469         
1470         /* if mirror painting, find the other group */
1471         if(me->editflag & ME_EDIT_MIRROR_X) {
1472                 bDeformGroup *defgroup= BLI_findlink(&ob->defbase, ob->actdef-1);
1473                 if(defgroup) {
1474                         bDeformGroup *curdef;
1475                         int actdef= 0;
1476                         char name[32];
1477                         
1478                         BLI_strncpy(name, defgroup->name, 32);
1479                         bone_flip_name(name, 0);                /* 0 = don't strip off number extensions */
1480                         
1481                         for (curdef = ob->defbase.first; curdef; curdef=curdef->next, actdef++)
1482                                 if (!strcmp(curdef->name, name))
1483                                         break;
1484                         if(curdef==NULL) {
1485                                 int olddef= ob->actdef; /* tsk, ED_vgroup_add sets the active defgroup */
1486                                 curdef= ED_vgroup_add_name (ob, name);
1487                                 ob->actdef= olddef;
1488                         }
1489                         
1490                         if(curdef && curdef!=defgroup)
1491                                 wpd->vgroup_mirror= actdef;
1492                 }
1493         }
1494         
1495         return 1;
1496 }
1497
1498 static void wpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1499 {
1500         ToolSettings *ts= CTX_data_tool_settings(C);
1501         VPaint *wp= ts->wpaint;
1502         Brush *brush = paint_brush(&wp->paint);
1503         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1504         ViewContext *vc= &wpd->vc;
1505         Object *ob= vc->obact;
1506         Mesh *me= ob->data;
1507         float mat[4][4];
1508         float paintweight= ts->vgroup_weight;
1509         int *indexar= wpd->indexar;
1510         int totindex, index, totw, flip;
1511         float alpha;
1512         float mval[2], pressure;
1513
1514         view3d_operator_needs_opengl(C);
1515                         
1516         /* load projection matrix */
1517         wmMultMatrix(ob->obmat);
1518         wmGetSingleMatrix(mat);
1519         wmLoadMatrix(wpd->vc.rv3d->viewmat);
1520
1521         flip = RNA_boolean_get(itemptr, "flip");
1522         pressure = RNA_float_get(itemptr, "pressure");
1523         RNA_float_get_array(itemptr, "mouse", mval);
1524         mval[0]-= vc->ar->winrct.xmin;
1525         mval[1]-= vc->ar->winrct.ymin;
1526                         
1527         swap_m4m4(wpd->vc.rv3d->persmat, mat);
1528                         
1529         /* which faces are involved */
1530         if(wp->flag & VP_AREA) {
1531                 totindex= sample_backbuf_area(vc, indexar, me->totface, mval[0], mval[1], brush->size);
1532         }
1533         else {
1534                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1535                 if(indexar[0]) totindex= 1;
1536                 else totindex= 0;
1537         }
1538                         
1539         if(wp->flag & VP_COLINDEX) {
1540                 for(index=0; index<totindex; index++) {
1541                         if(indexar[index] && indexar[index]<=me->totpoly) {
1542                                 MPoly *mpoly= ((MPoly *)me->mpoly) + (indexar[index]-1);
1543                                                 
1544                                 if(mpoly->mat_nr!=ob->actcol-1) {
1545                                         indexar[index]= 0;
1546                                 }
1547                         }
1548                 }
1549         }
1550                         
1551         if((me->editflag & ME_EDIT_PAINT_MASK) && me->mpoly) {
1552                 for(index=0; index<totindex; index++) {
1553                         if(indexar[index] && indexar[index]<=me->totpoly) {
1554                                 MPoly *mpoly= ((MPoly *)me->mpoly) + (indexar[index]-1);
1555                                                 
1556                                 if((mpoly->flag & ME_FACE_SEL)==0) {
1557                                         indexar[index]= 0;
1558                                 }
1559                         }                                       
1560                 }
1561         }
1562                         
1563         /* make sure each vertex gets treated only once */
1564         /* and calculate filter weight */
1565         totw= 0;
1566         if(wp->mode==VP_BLUR) 
1567                 paintweight= 0.0f;
1568         else
1569                 paintweight= ts->vgroup_weight;
1570                         
1571         for(index=0; index<totindex; index++) {
1572                 if(indexar[index] && indexar[index]<=me->totpoly) {
1573                         MPoly *mpoly= me->mpoly + (indexar[index]-1);
1574                         MLoop *ml = me->mloop + mpoly->loopstart;
1575                         int i;
1576
1577                         for (i=0; i<mpoly->totloop; i++, ml++) {
1578                                 (me->dvert+ml->v)->flag = 1;
1579                         }
1580                                         
1581                         if(wp->mode==VP_BLUR) {
1582                                 MDeformWeight *dw, *(*dw_func)(MDeformVert *, int) = ED_vgroup_weight_verify;
1583                                                 
1584                                 if(wp->flag & VP_ONLYVGROUP)
1585                                         dw_func= ED_vgroup_weight_get;
1586                                                 
1587                                 ml = me->mloop + mpoly->loopstart;
1588                                 for (i=0; i<mpoly->totloop; i++, ml++) {
1589                                         dw = dw_func(me->dvert+ml->v, ob->actdef-1);
1590                                         if (dw) {
1591                                                 paintweight += dw->weight;
1592                                                 totw++;
1593                                         }                                       
1594                                 }
1595                         }
1596                 }
1597         }
1598                         
1599         if(wp->mode==VP_BLUR) 
1600                 paintweight/= (float)totw;
1601                         
1602         for(index=0; index<totindex; index++) {
1603                 if(indexar[index] && indexar[index]<=me->totpoly) {
1604                         MPoly *mpoly= me->mpoly + (indexar[index]-1);
1605                         MLoop *ml=me->mloop+mpoly->loopstart;
1606                         int i;
1607
1608                         for (i=0; i<mpoly->totloop; i++, ml++) {
1609                                 if ((me->dvert+ml->v)->flag) {
1610                                         alpha= calc_vp_alpha_dl(wp, vc, wpd->wpimat, wpd->vertexcosnos+6*ml->v, mval, pressure);
1611                                         if(alpha) {
1612                                                 do_weight_paint_vertex(wp, ob, ml->v, 
1613                                                         alpha, paintweight, flip, wpd->vgroup_mirror, 
1614                                                         wpd->vgroup_validmap);
1615                                         }
1616                                         (me->dvert+ml->v)->flag= 0;
1617                                 }
1618                         }
1619                 }
1620         }
1621                         
1622         swap_m4m4(vc->rv3d->persmat, mat);
1623                         
1624         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1625         ED_region_tag_redraw(vc->ar);
1626 }
1627
1628 static void wpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
1629 {
1630         ToolSettings *ts= CTX_data_tool_settings(C);
1631         Object *ob= CTX_data_active_object(C);
1632         struct WPaintData *wpd= paint_stroke_mode_data(stroke);
1633         
1634         if(wpd->vertexcosnos)
1635                 MEM_freeN(wpd->vertexcosnos);
1636         MEM_freeN(wpd->indexar);
1637         
1638         if (wpd->vgroup_validmap)
1639                 MEM_freeN(wpd->vgroup_validmap);
1640
1641         /* frees prev buffer */
1642         copy_wpaint_prev(ts->wpaint, NULL, 0);
1643         
1644         /* and particles too */
1645         if(ob->particlesystem.first) {
1646                 ParticleSystem *psys;
1647                 int i;
1648                 
1649                 for(psys= ob->particlesystem.first; psys; psys= psys->next) {
1650                         for(i=0; i<PSYS_TOT_VG; i++) {
1651                                 if(psys->vgroup[i]==ob->actdef) {
1652                                         psys->recalc |= PSYS_RECALC_RESET;
1653                                         break;
1654                                 }
1655                         }
1656                 }
1657         }
1658         
1659         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
1660         
1661         MEM_freeN(wpd);
1662 }
1663
1664
1665 static int wpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
1666 {
1667         
1668         op->customdata = paint_stroke_new(C, wpaint_stroke_test_start,
1669                                           wpaint_stroke_update_step,
1670                                           wpaint_stroke_done);
1671         
1672         /* add modal handler */
1673         WM_event_add_modal_handler(C, op);
1674
1675         op->type->modal(C, op, event);
1676         
1677         return OPERATOR_RUNNING_MODAL;
1678 }
1679
1680 void PAINT_OT_weight_paint(wmOperatorType *ot)
1681 {
1682         
1683         /* identifiers */
1684         ot->name= "Weight Paint";
1685         ot->idname= "PAINT_OT_weight_paint";
1686         
1687         /* api callbacks */
1688         ot->invoke= wpaint_invoke;
1689         ot->modal= paint_stroke_modal;
1690         /* ot->exec= vpaint_exec; <-- needs stroke property */
1691         ot->poll= weight_paint_poll;
1692         
1693         /* flags */
1694         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
1695
1696         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
1697 }
1698
1699 static int weight_paint_set_exec(bContext *C, wmOperator *op)
1700 {
1701         struct Scene *scene= CTX_data_scene(C);
1702         Object *obact = CTX_data_active_object(C);
1703
1704         wpaint_fill(scene->toolsettings->wpaint, obact, scene->toolsettings->vgroup_weight);
1705         ED_region_tag_redraw(CTX_wm_region(C)); // XXX - should redraw all 3D views
1706         return OPERATOR_FINISHED;
1707 }
1708
1709 void PAINT_OT_weight_set(wmOperatorType *ot)
1710 {
1711         /* identifiers */
1712         ot->name= "Weight Set";
1713         ot->idname= "PAINT_OT_weight_set";
1714
1715         /* api callbacks */
1716         ot->exec= weight_paint_set_exec;
1717         ot->poll= facemask_paint_poll;
1718
1719         /* flags */
1720         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1721
1722         RNA_def_boolean(ot->srna, "extend", 0, "Extend", "Extend selection instead of deselecting everything first.");
1723 }
1724
1725 /* ************ set / clear vertex paint mode ********** */
1726
1727
1728 static int set_vpaint(bContext *C, wmOperator *op)              /* toggle */
1729 {       
1730         Object *ob= CTX_data_active_object(C);
1731         Scene *scene= CTX_data_scene(C);
1732         VPaint *vp= scene->toolsettings->vpaint;
1733         Mesh *me;
1734         
1735         me= get_mesh(ob);
1736         
1737         if(me==NULL || object_data_is_libdata(ob)) {
1738                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1739                 return OPERATOR_PASS_THROUGH;
1740         }
1741         
1742         if(me && me->totface>=MAXINDEX) {
1743                 error("Maximum number of faces: %d", MAXINDEX-1);
1744                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1745                 return OPERATOR_FINISHED;
1746         }
1747         
1748         if(me && me->mloopcol==NULL) make_vertexcol(ob);
1749         
1750         /* toggle: end vpaint */
1751         if(ob->mode & OB_MODE_VERTEX_PAINT) {
1752                 
1753                 ob->mode &= ~OB_MODE_VERTEX_PAINT;
1754         }
1755         else {
1756                 ob->mode |= OB_MODE_VERTEX_PAINT;
1757                 /* Turn off weight painting */
1758                 if (ob->mode & OB_MODE_WEIGHT_PAINT)
1759                         set_wpaint(C, op);
1760                 
1761                 if(vp==NULL)
1762                         vp= scene->toolsettings->vpaint= new_vpaint(0);
1763                 
1764                 paint_cursor_start(C, vertex_paint_poll);
1765
1766                 paint_init(&vp->paint, PAINT_CURSOR_VERTEX_PAINT);
1767         }
1768         
1769         if (me)
1770                 /* update modifier stack for mapping requirements */
1771                 DAG_id_flush_update(&me->id, OB_RECALC_DATA);
1772         
1773         WM_event_add_notifier(C, NC_SCENE|ND_MODE, scene);
1774         
1775         return OPERATOR_FINISHED;
1776 }
1777
1778 void PAINT_OT_vertex_paint_toggle(wmOperatorType *ot)
1779 {
1780         
1781         /* identifiers */
1782         ot->name= "Vertex Paint Mode";
1783         ot->idname= "PAINT_OT_vertex_paint_toggle";
1784         
1785         /* api callbacks */
1786         ot->exec= set_vpaint;
1787         ot->poll= paint_poll_test;
1788         
1789         /* flags */
1790         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
1791         
1792 }
1793
1794
1795
1796 /* ********************** vertex paint operator ******************* */
1797
1798 /* Implementation notes:
1799
1800 Operator->invoke()
1801   - validate context (add mcol)
1802   - create customdata storage
1803   - call paint once (mouse click)
1804   - add modal handler 
1805
1806 Operator->modal()
1807   - for every mousemove, apply vertex paint
1808   - exit on mouse release, free customdata
1809     (return OPERATOR_FINISHED also removes handler and operator)
1810
1811 For future:
1812   - implement a stroke event (or mousemove with past positons)
1813   - revise whether op->customdata should be added in object, in set_vpaint
1814
1815 */
1816
1817 typedef struct polyfacemap_e {
1818         struct polyfacemap_e *next, *prev;
1819         int facenr;
1820 } polyfacemap_e;
1821
1822 typedef struct VPaintData {
1823         ViewContext vc;
1824         unsigned int paintcol;
1825         int *indexar;
1826         float *vertexcosnos;
1827         float vpimat[3][3];
1828         
1829         /*mpoly -> mface mapping*/
1830         MemArena *arena;
1831         ListBase *polyfacemap;
1832 } VPaintData;
1833
1834 static void vpaint_build_poly_facemap(struct VPaintData *vd, Mesh *me,
1835                                       Object *ob, Scene *scene)
1836 {
1837         MFace *mf;
1838         polyfacemap_e *e;
1839         int *origIndex;
1840         int i;
1841
1842         vd->arena = BLI_memarena_new(1<<13);
1843         BLI_memarena_use_calloc(vd->arena);
1844
1845         vd->polyfacemap = BLI_memarena_alloc(vd->arena, sizeof(ListBase)*me->totpoly);
1846
1847         /*we can't assume mfaces have a correct origindex layer that indices to mpolys.
1848           so instead we have to regenerate the tesselation faces altogether.*/
1849         me->totface = mesh_recalcTesselation(&me->fdata, &me->ldata, &me->pdata, 
1850                 me->mvert, me->totface, me->totloop, me->totpoly, 1);
1851         mesh_update_customdata_pointers(me);
1852         makeDerivedMesh(scene, ob, NULL, CD_MASK_BAREMESH);
1853
1854         origIndex = CustomData_get_layer(&me->fdata, CD_ORIGINDEX);
1855         mf = me->mface;
1856
1857         if (!origIndex)
1858                 return;
1859
1860         for (i=0; i<me->totface; i++, mf++, origIndex++) {
1861                 if (*origIndex == ORIGINDEX_NONE)
1862                         continue;
1863
1864                 e = BLI_memarena_alloc(vd->arena, sizeof(polyfacemap_e));
1865                 e->facenr = i;
1866                 
1867                 BLI_addtail(&vd->polyfacemap[*origIndex], e);
1868         }
1869 }
1870
1871 static int vpaint_stroke_test_start(bContext *C, struct wmOperator *op, wmEvent *event)
1872 {
1873         ToolSettings *ts= CTX_data_tool_settings(C);
1874         struct PaintStroke *stroke = op->customdata;
1875         VPaint *vp= ts->vpaint;
1876         struct VPaintData *vpd;
1877         Object *ob= CTX_data_active_object(C);
1878         Scene *scene = CTX_data_scene(C);
1879         Mesh *me;
1880         float mat[4][4], imat[4][4];
1881
1882         /* context checks could be a poll() */
1883         me= get_mesh(ob);
1884         if(me==NULL || me->totpoly==0)
1885                 return OPERATOR_PASS_THROUGH;
1886         
1887         if(me->mloopcol==NULL)
1888                 make_vertexcol(ob);
1889         if(me->mloopcol==NULL)
1890                 return OPERATOR_CANCELLED;
1891
1892         /* make mode data storage */
1893         vpd= MEM_callocN(sizeof(struct VPaintData), "VPaintData");
1894         paint_stroke_set_mode_data(stroke, vpd);
1895         view3d_set_viewcontext(C, &vpd->vc);
1896         
1897         vpd->vertexcosnos= mesh_get_mapped_verts_nors(vpd->vc.scene, ob);
1898         vpd->indexar= get_indexarray();
1899         vpd->paintcol= vpaint_get_current_col(vp);
1900         vpaint_build_poly_facemap(vpd, me, ob, scene);
1901         
1902         /* for filtering */
1903         copy_vpaint_prev(vp, (unsigned int *)me->mloopcol, me->totloop);
1904         
1905         /* some old cruft to sort out later */
1906         mul_m4_m4m4(mat, ob->obmat, vpd->vc.rv3d->viewmat);
1907         invert_m4_m4(imat, mat);
1908         copy_m3_m4(vpd->vpimat, imat);
1909
1910         return 1;
1911 }
1912
1913 #if 0
1914 static void vpaint_paint_face(VPaint *vp, VPaintData *vpd, Object *ob, int index, float mval[2], float pressure, int flip)
1915 {
1916         ViewContext *vc = &vpd->vc;
1917         Mesh *me = get_mesh(ob);
1918         MFace *mface= ((MFace*)me->mface) + index;
1919         unsigned int *mcol= ((unsigned int*)me->mcol) + 4*index;
1920         unsigned int *mcolorig= ((unsigned int*)vp->vpaint_prev) + 4*index;
1921         float alpha;
1922         int i;
1923         
1924         if((vp->flag & VP_COLINDEX && mface->mat_nr!=ob->actcol-1) ||
1925            ((me->editflag & ME_EDIT_PAINT_MASK) && !(mface->flag & ME_FACE_SEL)))
1926                 return;
1927
1928         if(vp->mode==VP_BLUR) {
1929                 unsigned int fcol1= mcol_blend( mcol[0], mcol[1], 128);
1930                 if(mface->v4) {
1931                         unsigned int fcol2= mcol_blend( mcol[2], mcol[3], 128);
1932                         vpd->paintcol= mcol_blend( fcol1, fcol2, 128);
1933                 }
1934                 else {
1935                         vpd->paintcol= mcol_blend( mcol[2], fcol1, 170);
1936                 }
1937                 
1938         }
1939
1940         for(i = 0; i < (mface->v4 ? 4 : 3); ++i) {
1941                 alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*(&mface->v1)[i], mval, pressure);
1942                 if(alpha)
1943                         vpaint_blend(vp, mcol+i, mcolorig+i, vpd->paintcol, (int)(alpha*255.0));
1944         }
1945 }
1946 #endif
1947
1948 static void vpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerRNA *itemptr)
1949 {
1950         ToolSettings *ts= CTX_data_tool_settings(C);
1951         struct VPaintData *vpd = paint_stroke_mode_data(stroke);
1952         VPaint *vp= ts->vpaint;
1953         Brush *brush = paint_brush(&vp->paint);
1954         ViewContext *vc= &vpd->vc;
1955         Object *ob= vc->obact;
1956         polyfacemap_e *e;
1957         Mesh *me= ob->data;
1958         float mat[4][4];
1959         int *indexar= vpd->indexar;
1960         int totindex, index, flip;
1961         float pressure, mval[2];
1962
1963         RNA_float_get_array(itemptr, "mouse", mval);
1964         flip = RNA_boolean_get(itemptr, "flip");
1965         pressure = RNA_float_get(itemptr, "pressure");
1966                         
1967         view3d_operator_needs_opengl(C);
1968                         
1969         /* load projection matrix */
1970         wmMultMatrix(ob->obmat);
1971         wmGetSingleMatrix(mat);
1972         wmLoadMatrix(vc->rv3d->viewmat);
1973
1974         mval[0]-= vc->ar->winrct.xmin;
1975         mval[1]-= vc->ar->winrct.ymin;
1976
1977                         
1978         /* which faces are involved */
1979         if(vp->flag & VP_AREA) {
1980                 totindex= sample_backbuf_area(vc, indexar, me->totpoly, mval[0], mval[1], brush->size);
1981         }
1982         else {
1983                 indexar[0]= view3d_sample_backbuf(vc, mval[0], mval[1]);
1984                 if(indexar[0]) totindex= 1;
1985                 else totindex= 0;
1986         }
1987                         
1988         swap_m4m4(vc->rv3d->persmat, mat);
1989                         
1990         if(vp->flag & VP_COLINDEX) {
1991                 for(index=0; index<totindex; index++) {
1992                         if(indexar[index] && indexar[index]<=me->totpoly) {
1993                                 MPoly *mpoly= ((MPoly *)me->mpoly) + (indexar[index]-1);
1994                                                 
1995                                 if(mpoly->mat_nr!=ob->actcol-1) {
1996                                         indexar[index]= 0;
1997                                 }
1998                         }                                       
1999                 }
2000         }
2001
2002         if((me->editflag & ME_EDIT_PAINT_MASK) && me->mpoly) {
2003                 for(index=0; index<totindex; index++) {
2004                         if(indexar[index] && indexar[index]<=me->totpoly) {
2005                                 MPoly *mpoly= ((MPoly *)me->mpoly) + (indexar[index]-1);
2006                                                 
2007                                 if((mpoly->flag & ME_FACE_SEL)==0)
2008                                         indexar[index]= 0;
2009                         }                                       
2010                 }
2011         }
2012         
2013         swap_m4m4(vc->rv3d->persmat, mat);
2014
2015         /* was disabled because it is slow, but necessary for blur */
2016         if(vp->mode == VP_BLUR)
2017                 do_shared_vertexcol(me);
2018                         
2019         for(index=0; index<totindex; index++) {
2020                                 
2021                 if(indexar[index] && indexar[index]<=me->totpoly) {
2022                         MPoly *mpoly= ((MPoly *)me->mpoly) + (indexar[index]-1);
2023                         MFace *mf;
2024                         MCol *mc;
2025                         MLoop *ml;
2026                         MLoopCol *mlc;
2027                         unsigned int *lcol = ((unsigned int*)me->mloopcol) + mpoly->loopstart;
2028                         unsigned int *lcolorig = ((unsigned int*)vp->vpaint_prev) + mpoly->loopstart;
2029                         int i, j, alpha;
2030                                         
2031                         if(vp->mode==VP_BLUR) {
2032                                 unsigned int blend[5] = {0};
2033                                 char *col;
2034
2035                                 for (j=0; j<mpoly->totloop; j += 2) {
2036                                         col = (char*)(lcol + j);
2037                                         blend[0] += col[0];
2038                                         blend[1] += col[1];
2039                                         blend[2] += col[2];
2040                                         blend[3] += col[3];
2041                                 }
2042
2043                                 blend[0] /= mpoly->totloop;
2044                                 blend[1] /= mpoly->totloop;
2045                                 blend[2] /= mpoly->totloop;
2046                                 blend[3] /= mpoly->totloop;
2047                                 col = (char*)(blend + 4);
2048                                 col[0] = blend[0];
2049                                 col[1] = blend[1];
2050                                 col[2] = blend[2];
2051                                 col[3] = blend[3];
2052
2053                                 vpd->paintcol = *((unsigned int*)col);
2054                         }
2055
2056                         ml = me->mloop + mpoly->loopstart;
2057                         for (i=0; i<mpoly->totloop; i++, ml++) {
2058                                 alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, 
2059                                            vpd->vertexcosnos+6*ml->v, mval, pressure);
2060                                 if(alpha) vpaint_blend(vp, lcol+i, lcolorig+i, vpd->paintcol, alpha);
2061                         }
2062         
2063                         #ifdef CPYCOL
2064                         #undef CPYCOL
2065                         #endif
2066                         #define CPYCOL(c, l) (c)->a = (l)->a, (c)->r = (l)->r, (c)->g = (l)->g, (c)->b = (l)->b
2067
2068                         /*update vertex colors for tesselations incrementally,
2069                           rather then regenerating the tesselation altogether.*/
2070                         for (e=vpd->polyfacemap[(indexar[index]-1)].first; e; e=e->next) {
2071                                 mf = me->mface + e->facenr;
2072                                 mc = me->mcol + e->facenr*4;
2073                                 
2074                                 ml = me->mloop + mpoly->loopstart;
2075                                 mlc = me->mloopcol + mpoly->loopstart;
2076                                 for (j=0; j<mpoly->totloop; j++, ml++, mlc++) {
2077                                         if (ml->v == mf->v1)
2078                                                 CPYCOL(mc, mlc);
2079                                         else if (ml->v == mf->v2)
2080                                                 CPYCOL(mc+1, mlc);
2081                                         else if (ml->v == mf->v3)
2082                                                 CPYCOL(mc+2, mlc);
2083                                         else if (mf->v4 && ml->v == mf->v4)
2084                                                 CPYCOL(mc+3, mlc);
2085
2086                                 }
2087                         }
2088                         #undef CPYCOL
2089                 }
2090         }
2091                 
2092         swap_m4m4(vc->rv3d->persmat, mat);
2093                         
2094         do_shared_vertexcol(me);
2095
2096         ED_region_tag_redraw(vc->ar);           
2097         DAG_id_flush_update(ob->data, OB_RECALC_DATA);
2098 }
2099
2100 static void vpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
2101 {
2102         ToolSettings *ts= CTX_data_tool_settings(C);
2103         struct VPaintData *vpd= paint_stroke_mode_data(stroke);
2104         
2105         if(vpd->vertexcosnos)
2106                 MEM_freeN(vpd->vertexcosnos);
2107         MEM_freeN(vpd->indexar);
2108         
2109         /* frees prev buffer */
2110         copy_vpaint_prev(ts->vpaint, NULL, 0);
2111         BLI_memarena_free(vpd->arena);
2112
2113         MEM_freeN(vpd);
2114 }
2115
2116 static int vpaint_invoke(bContext *C, wmOperator *op, wmEvent *event)
2117 {
2118         
2119         op->customdata = paint_stroke_new(C, vpaint_stroke_test_start,
2120                                           vpaint_stroke_update_step,
2121                                           vpaint_stroke_done);
2122         
2123         /* add modal handler */
2124         WM_event_add_modal_handler(C, op);
2125
2126         op->type->modal(C, op, event);
2127         
2128         return OPERATOR_RUNNING_MODAL;
2129 }
2130
2131 void PAINT_OT_vertex_paint(wmOperatorType *ot)
2132 {
2133         /* identifiers */
2134         ot->name= "Vertex Paint";
2135         ot->idname= "PAINT_OT_vertex_paint";
2136         
2137         /* api callbacks */
2138         ot->invoke= vpaint_invoke;
2139         ot->modal= paint_stroke_modal;
2140         /* ot->exec= vpaint_exec; <-- needs stroke property */
2141         ot->poll= vertex_paint_poll;
2142         
2143         /* flags */
2144         ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO|OPTYPE_BLOCKING;
2145
2146         RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
2147 }
2148