Fix buffer overflows in TIFF, PNG, IRIS, DPX, HDR and AVI loading.
[blender.git] / source / blender / imbuf / intern / iris.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
19  * All rights reserved.
20  *
21  * The Original Code is: all of this file.
22  *
23  * Contributor(s): none yet.
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  */
27
28 /** \file blender/imbuf/intern/iris.c
29  *  \ingroup imbuf
30  */
31
32
33 #include <string.h>
34
35 #include "BLI_utildefines.h"
36 #include "BLI_fileops.h"
37
38 #include "MEM_guardedalloc.h"
39
40 #include "imbuf.h"
41 #include "IMB_imbuf_types.h"
42 #include "IMB_imbuf.h"
43 #include "IMB_filetype.h"
44
45 #include "IMB_colormanagement.h"
46 #include "IMB_colormanagement_intern.h"
47
48 #define IMAGIC 0732
49
50 typedef struct {
51         ushort  imagic;      /* stuff saved on disk . . */
52         ushort  type;
53         ushort  dim;
54         ushort  xsize;
55         ushort  ysize;
56         ushort  zsize;
57         uint    min;
58         uint    max;
59         uchar   _pad1[4];
60         char    name[80];
61         uint    colormap;
62         uchar   _pad2[404];
63 } IMAGE;
64
65 #define HEADER_SIZE 512
66
67 BLI_STATIC_ASSERT(sizeof(IMAGE) == HEADER_SIZE, "Invalid header size");
68
69 #define RINTLUM (79)
70 #define GINTLUM (156)
71 #define BINTLUM (21)
72
73 #define ILUM(r, g, b)   ((int)(RINTLUM * (r) + GINTLUM * (g) + BINTLUM * (b)) >> 8)
74
75 #define OFFSET_R    0   /* this is byte order dependent */
76 #define OFFSET_G    1
77 #define OFFSET_B    2
78 // #define OFFSET_A    3
79
80 #define CHANOFFSET(z)   (3 - (z)) /* this is byte order dependent */
81
82 // #define TYPEMASK        0xff00
83 #define BPPMASK         0x00ff
84 // #define ITYPE_VERBATIM      0x0000 // UNUSED
85 #define ITYPE_RLE       0x0100
86 #define ISRLE(type)     (((type) & 0xff00) == ITYPE_RLE)
87 // #define ISVERBATIM(type)    (((type) & 0xff00) == ITYPE_VERBATIM)
88 #define BPP(type)       ((type) & BPPMASK)
89 #define RLE(bpp)        (ITYPE_RLE | (bpp))
90 // #define VERBATIM(bpp)       (ITYPE_VERBATIM | (bpp)) // UNUSED
91 // #define IBUFSIZE(pixels)    ((pixels + (pixels >> 6)) << 2) // UNUSED
92 // #define RLE_NOP         0x00
93
94 /* local struct for mem access */
95 typedef struct MFileOffset {
96         const uchar *_file_data;
97         uint _file_offset;
98 } MFileOffset;
99
100 #define MFILE_DATA(inf) ((void)0, ((inf)->_file_data + (inf)->_file_offset))
101 #define MFILE_STEP(inf, step) { (inf)->_file_offset += step; } ((void)0)
102 #define MFILE_SEEK(inf, pos)  { (inf)->_file_offset  = pos;  } ((void)0)
103
104 /* error flags */
105 #define DIRTY_FLAG_EOF (1 << 0)
106 #define DIRTY_FLAG_ENCODING (1 << 1)
107
108 /* funcs */
109 static void readheader(MFileOffset *inf, IMAGE *image);
110 static int writeheader(FILE *outf, IMAGE *image);
111
112 static ushort getshort(MFileOffset *inf);
113 static uint getlong(MFileOffset *inf);
114 static void putshort(FILE *outf, ushort val);
115 static int putlong(FILE *outf, uint val);
116 static int writetab(FILE *outf, uint *tab, int len);
117 static void readtab(MFileOffset *inf, uint *tab, int len);
118
119 static int expandrow(uchar *optr, const uchar *optr_end, const uchar *iptr, const uchar *iptr_end, int z);
120 static int expandrow2(float *optr, const float *optr_end, const uchar *iptr, const uchar *iptr_end, int z);
121 static void interleaverow(uchar *lptr, const uchar *cptr, int z, int n);
122 static void interleaverow2(float *lptr, const uchar *cptr, int z, int n);
123 static int compressrow(uchar *lbuf, uchar *rlebuf, int z, int cnt);
124 static void lumrow(uchar *rgbptr, uchar *lumptr, int n);
125
126 /*
127  *      byte order independent read/write of shorts and ints.
128  *
129  */
130
131 static ushort getshort(MFileOffset *inf)
132 {
133         const uchar *buf;
134
135         buf = MFILE_DATA(inf);
136         MFILE_STEP(inf, 2);
137
138         return ((ushort)buf[0] << 8) + ((ushort)buf[1] << 0);
139 }
140
141 static uint getlong(MFileOffset *mofs)
142 {
143         const uchar *buf;
144         
145         buf = MFILE_DATA(mofs);
146         MFILE_STEP(mofs, 4);
147
148         return ((uint)buf[0] << 24) + ((uint)buf[1] << 16) + ((uint)buf[2] << 8) + ((uint)buf[3] << 0);
149 }
150
151 static void putshort(FILE *outf, ushort val)
152 {
153         uchar buf[2];
154
155         buf[0] = (val >> 8);
156         buf[1] = (val >> 0);
157         fwrite(buf, 2, 1, outf);
158 }
159
160 static int putlong(FILE *outf, uint val)
161 {
162         uchar buf[4];
163
164         buf[0] = (val >> 24);
165         buf[1] = (val >> 16);
166         buf[2] = (val >> 8);
167         buf[3] = (val >> 0);
168         return fwrite(buf, 4, 1, outf);
169 }
170
171 static void readheader(MFileOffset *inf, IMAGE *image)
172 {
173         memset(image, 0, sizeof(IMAGE));
174         image->imagic = getshort(inf);
175         image->type = getshort(inf);
176         image->dim = getshort(inf);
177         image->xsize = getshort(inf);
178         image->ysize = getshort(inf);
179         image->zsize = getshort(inf);
180 }
181
182 static int writeheader(FILE *outf, IMAGE *image)
183 {
184         IMAGE t = {0};
185
186         fwrite(&t, sizeof(IMAGE), 1, outf);
187         fseek(outf, 0, SEEK_SET);
188         putshort(outf, image->imagic);
189         putshort(outf, image->type);
190         putshort(outf, image->dim);
191         putshort(outf, image->xsize);
192         putshort(outf, image->ysize);
193         putshort(outf, image->zsize);
194         putlong(outf, image->min);
195         putlong(outf, image->max);
196         putlong(outf, 0);
197         return fwrite("no name", 8, 1, outf);
198 }
199
200 static int writetab(FILE *outf, uint *tab, int len)
201 {
202         int r = 0;
203
204         while (len) {
205                 r = putlong(outf, *tab++);
206                 len -= 4;
207         }
208         return r;
209 }
210
211 static void readtab(MFileOffset *inf, uint *tab, int len)
212 {
213         while (len) {
214                 *tab++ = getlong(inf);
215                 len -= 4;
216         }
217 }
218
219 static void test_endian_zbuf(struct ImBuf *ibuf)
220 {
221         int len;
222         int *zval;
223         
224         if (BIG_LONG(1) == 1) return;
225         if (ibuf->zbuf == NULL) return;
226         
227         len = ibuf->x * ibuf->y;
228         zval = ibuf->zbuf;
229         
230         while (len--) {
231                 zval[0] = BIG_LONG(zval[0]);
232                 zval++;
233         }
234 }
235
236 /* from misc_util: flip the bytes from x  */
237 #define GS(x) (((uchar *)(x))[0] << 8 | ((uchar *)(x))[1])
238
239 /* this one is only def-ed once, strangely... */
240 #define GSS(x) (((uchar *)(x))[1] << 8 | ((uchar *)(x))[0])
241
242 int imb_is_a_iris(const uchar *mem)
243 {
244         return ((GS(mem) == IMAGIC) || (GSS(mem) == IMAGIC));
245 }
246
247 /*
248  *      longimagedata - 
249  *              read in a B/W RGB or RGBA iris image file and return a 
250  *      pointer to an array of ints.
251  *
252  */
253
254 struct ImBuf *imb_loadiris(const uchar *mem, size_t size, int flags, char colorspace[IM_MAX_SPACE])
255 {
256         uint *base, *lptr = NULL;
257         float *fbase, *fptr = NULL;
258         uint *zbase, *zptr;
259         const uchar *rledat;
260         const uchar *mem_end = mem + size;
261         MFileOffset _inf_data = {mem, 0}, *inf = &_inf_data;
262         IMAGE image;
263         int bpp, rle, cur, badorder;
264         ImBuf *ibuf;
265         uchar dirty_flag = 0;
266
267         if (size < HEADER_SIZE) {
268                 return NULL;
269         }
270
271         if (!imb_is_a_iris(mem)) {
272                 return NULL;
273         }
274
275         /* OCIO_TODO: only tested with 1 byte per pixel, not sure how to test with other settings */
276         colorspace_set_default_role(colorspace, IM_MAX_SPACE, COLOR_ROLE_DEFAULT_BYTE);
277
278         readheader(inf, &image);
279         if (image.imagic != IMAGIC) {
280                 fprintf(stderr, "longimagedata: bad magic number in image file\n");
281                 return(NULL);
282         }
283
284         rle = ISRLE(image.type);
285         bpp = BPP(image.type);
286         if (bpp != 1 && bpp != 2) {
287                 fprintf(stderr, "longimagedata: image must have 1 or 2 byte per pix chan\n");
288                 return(NULL);
289         }
290         if ((uint)image.zsize > 8) {
291                 fprintf(stderr, "longimagedata: channels over 8 not supported\n");
292                 return(NULL);
293         }
294
295         const int xsize = image.xsize;
296         const int ysize = image.ysize;
297         const int zsize = image.zsize;
298
299         if (flags & IB_test) {
300                 ibuf = IMB_allocImBuf(image.xsize, image.ysize, 8 * image.zsize, 0);
301                 if (ibuf) ibuf->ftype = IMB_FTYPE_IMAGIC;
302                 return(ibuf);
303         }
304         
305         if (rle) {
306                 size_t tablen = (size_t)ysize * (size_t)zsize * sizeof(int);
307                 MFILE_SEEK(inf, HEADER_SIZE);
308
309                 uint *starttab = MEM_mallocN(tablen, "iris starttab");
310                 uint *lengthtab = MEM_mallocN(tablen, "iris endtab");
311
312 #define MFILE_CAPACITY_AT_PTR_OK_OR_FAIL(p) \
313                 if (UNLIKELY((p) > mem_end)) { dirty_flag |= DIRTY_FLAG_EOF; goto fail_rle; } ((void)0)
314
315                 MFILE_CAPACITY_AT_PTR_OK_OR_FAIL(MFILE_DATA(inf) + ((4 * 2) * tablen));
316
317                 readtab(inf, starttab, tablen);
318                 readtab(inf, lengthtab, tablen);
319         
320                 /* check data order */
321                 cur = 0;
322                 badorder = 0;
323                 for (size_t y = 0; y < ysize; y++) {
324                         for (size_t z = 0; z < zsize; z++) {
325                                 if (starttab[y + z * ysize] < cur) {
326                                         badorder = 1;
327                                         break;
328                                 }
329                                 cur = starttab[y + z * ysize];
330                         }
331                         if (badorder)
332                                 break;
333                 }
334         
335                 if (bpp == 1) {
336                         
337                         ibuf = IMB_allocImBuf(xsize, ysize, 8 * zsize, IB_rect);
338                         if (!ibuf) {
339                                 goto fail_rle;
340                         }
341                         if (ibuf->planes > 32) ibuf->planes = 32;
342                         base = ibuf->rect;
343                         zbase = (uint *)ibuf->zbuf;
344                         
345                         if (badorder) {
346                                 for (size_t z = 0; z < zsize; z++) {
347                                         lptr = base;
348                                         for (size_t y = 0; y < ysize; y++) {
349                                                 MFILE_SEEK(inf, starttab[y + z * ysize]);
350                                                 rledat = MFILE_DATA(inf);
351                                                 MFILE_STEP(inf, lengthtab[y + z * ysize]);
352                                                 const uchar *rledat_next = MFILE_DATA(inf);
353                                                 uint *lptr_next = lptr + xsize;
354                                                 MFILE_CAPACITY_AT_PTR_OK_OR_FAIL(rledat_next);
355                                                 dirty_flag |= expandrow((uchar *)lptr, (uchar *)lptr_next, rledat, rledat_next, 3 - z);
356                                                 lptr = lptr_next;
357                                         }
358                                 }
359                         }
360                         else {
361                                 lptr = base;
362                                 zptr = zbase;
363                                 for (size_t y = 0; y < ysize; y++) {
364
365                                         uint *lptr_next = lptr + xsize;
366                                         uint *zptr_next = zptr + xsize;
367
368                                         for (size_t z = 0; z < zsize; z++) {
369                                                 MFILE_SEEK(inf, starttab[y + z * ysize]);
370                                                 rledat = MFILE_DATA(inf);
371                                                 MFILE_STEP(inf, lengthtab[y + z * ysize]);
372                                                 const uchar *rledat_next = MFILE_DATA(inf);
373                                                 MFILE_CAPACITY_AT_PTR_OK_OR_FAIL(rledat_next);
374                                                 if (z < 4) {
375                                                         dirty_flag |= expandrow((uchar *)lptr, (uchar *)lptr_next, rledat, rledat_next, 3 - z);
376                                                 }
377                                                 else if (z < 8) {
378                                                         dirty_flag |= expandrow((uchar *)zptr, (uchar *)zptr_next, rledat, rledat_next, 7 - z);
379                                                 }
380                                         }
381                                         lptr = lptr_next;
382                                         zptr = zptr_next;
383                                 }
384                         }
385                         
386
387                 }
388                 else {  /* bpp == 2 */
389                         
390                         ibuf = IMB_allocImBuf(xsize, ysize, 32, (flags & IB_rect) | IB_rectfloat);
391                         if (!ibuf) {
392                                 goto fail_rle;
393                         }
394
395                         fbase = ibuf->rect_float;
396                         
397                         if (badorder) {
398                                 for (size_t z = 0; z < zsize; z++) {
399                                         fptr = fbase;
400                                         for (size_t y = 0; y < ysize; y++) {
401                                                 MFILE_SEEK(inf, starttab[y + z * ysize]);
402                                                 rledat = MFILE_DATA(inf);
403                                                 MFILE_STEP(inf, lengthtab[y + z * ysize]);
404                                                 const uchar *rledat_next = MFILE_DATA(inf);
405                                                 MFILE_CAPACITY_AT_PTR_OK_OR_FAIL(rledat_next);
406                                                 float *fptr_next = fptr + (xsize * 4);
407                                                 dirty_flag |= expandrow2(fptr, fptr_next, rledat, rledat_next, 3 - z);
408                                                 fptr = fptr_next;
409                                         }
410                                 }
411                         }
412                         else {
413                                 fptr = fbase;
414                                 float *fptr_next = fptr + (xsize * 4);
415
416                                 for (size_t y = 0; y < ysize; y++) {
417                                 
418                                         for (size_t z = 0; z < zsize; z++) {
419                                                 MFILE_SEEK(inf, starttab[y + z * ysize]);
420                                                 rledat = MFILE_DATA(inf);
421                                                 MFILE_STEP(inf, lengthtab[y + z * ysize]);
422                                                 const uchar *rledat_next = MFILE_DATA(inf);
423                                                 MFILE_CAPACITY_AT_PTR_OK_OR_FAIL(rledat_next);
424                                                 dirty_flag |= expandrow2(fptr, fptr_next, rledat, rledat_next, 3 - z);
425                                         }
426                                         fptr = fptr_next;
427                                 }
428                         }
429                 }
430 #undef MFILE_CAPACITY_AT_PTR_OK_OR_FAIL
431 fail_rle:
432                 MEM_freeN(starttab);
433                 MEM_freeN(lengthtab);
434
435                 if (!ibuf) {
436                         return NULL;
437                 }
438         }
439         else {
440
441 #define MFILE_CAPACITY_AT_PTR_OK_OR_FAIL(p) \
442                 if (UNLIKELY((p) > mem_end)) { dirty_flag |= DIRTY_FLAG_EOF; goto fail_uncompressed; } ((void)0)
443
444                 if (bpp == 1) {
445                         
446                         ibuf = IMB_allocImBuf(xsize, ysize, 8 * zsize, IB_rect);
447                         if (!ibuf) {
448                                 goto fail_uncompressed;
449                         }
450                         if (ibuf->planes > 32) ibuf->planes = 32;
451
452                         base = ibuf->rect;
453                         zbase = (uint *)ibuf->zbuf;
454                         
455                         MFILE_SEEK(inf, HEADER_SIZE);
456                         rledat = MFILE_DATA(inf);
457                         
458                         for (size_t z = 0; z < zsize; z++) {
459                                 
460                                 if (z < 4) lptr = base;
461                                 else if (z < 8) lptr = zbase;
462
463                                 for (size_t y = 0; y < ysize; y++) {
464                                         const uchar *rledat_next = rledat + xsize;
465                                         const int z_ofs = 3 - z;
466                                         MFILE_CAPACITY_AT_PTR_OK_OR_FAIL(rledat_next + z_ofs);
467                                         interleaverow((uchar *)lptr, rledat, z_ofs, xsize);
468                                         rledat = rledat_next;
469                                         lptr += xsize;
470                                 }
471                         }
472                         
473                 }
474                 else {  /* bpp == 2 */
475                         
476                         ibuf = IMB_allocImBuf(xsize, ysize, 32, (flags & IB_rect) | IB_rectfloat);
477                         if (!ibuf) {
478                                 goto fail_uncompressed;
479                         }
480
481                         fbase = ibuf->rect_float;
482
483                         MFILE_SEEK(inf, HEADER_SIZE);
484                         rledat = MFILE_DATA(inf);
485                         
486                         for (size_t z = 0; z < zsize; z++) {
487                                 
488                                 fptr = fbase;
489
490                                 for (size_t y = 0; y < ysize; y++) {
491                                         const uchar *rledat_next = rledat + xsize * 2;
492                                         const int z_ofs = 3 - z;
493                                         MFILE_CAPACITY_AT_PTR_OK_OR_FAIL(rledat_next + z_ofs);
494                                         interleaverow2(fptr, rledat, z_ofs, xsize);
495                                         rledat = rledat_next;
496                                         fptr += xsize * 4;
497                                 }
498                         }
499                         
500                 }
501 #undef MFILE_CAPACITY_AT_PTR_OK_OR_FAIL
502 fail_uncompressed:
503                 if (!ibuf) {
504                         return NULL;
505                 }
506         }
507
508         if (bpp == 1) {
509                 uchar *rect;
510                 
511                 if (image.zsize == 1) {
512                         rect = (uchar *) ibuf->rect;
513                         for (size_t x = (size_t)ibuf->x * (size_t)ibuf->y; x > 0; x--) {
514                                 rect[0] = 255;
515                                 rect[1] = rect[2] = rect[3];
516                                 rect += 4;
517                         }
518                 }
519                 else if (image.zsize == 2) {
520                         /* grayscale with alpha */
521                         rect = (uchar *) ibuf->rect;
522                         for (size_t x = (size_t)ibuf->x * (size_t)ibuf->y; x > 0; x--) {
523                                 rect[0] = rect[2];
524                                 rect[1] = rect[2] = rect[3];
525                                 rect += 4;
526                         }
527                 }
528                 else if (image.zsize == 3) {
529                         /* add alpha */
530                         rect = (uchar *) ibuf->rect;
531                         for (size_t x = (size_t)ibuf->x * (size_t)ibuf->y; x > 0; x--) {
532                                 rect[0] = 255;
533                                 rect += 4;
534                         }
535                 }
536                 
537         }
538         else {  /* bpp == 2 */
539                 
540                 if (image.zsize == 1) {
541                         fbase = ibuf->rect_float;
542                         for (size_t x = (size_t)ibuf->x * (size_t)ibuf->y; x > 0; x--) {
543                                 fbase[0] = 1;
544                                 fbase[1] = fbase[2] = fbase[3];
545                                 fbase += 4;
546                         }
547                 }
548                 else if (image.zsize == 2) {
549                         /* grayscale with alpha */
550                         fbase = ibuf->rect_float;
551                         for (size_t x = (size_t)ibuf->x * (size_t)ibuf->y; x > 0; x--) {
552                                 fbase[0] = fbase[2];
553                                 fbase[1] = fbase[2] = fbase[3];
554                                 fbase += 4;
555                         }
556                 }
557                 else if (image.zsize == 3) {
558                         /* add alpha */
559                         fbase = ibuf->rect_float;
560                         for (size_t x = (size_t)ibuf->x * (size_t)ibuf->y; x > 0; x--) {
561                                 fbase[0] = 1;
562                                 fbase += 4;
563                         }
564                 }
565                 
566                 if (flags & IB_rect) {
567                         IMB_rect_from_float(ibuf);
568                 }
569                 
570         }
571
572         if (dirty_flag) {
573                 fprintf(stderr, "longimagedata: corrupt file content (%d)\n", dirty_flag);
574         }
575         ibuf->ftype = IMB_FTYPE_IMAGIC;
576
577         test_endian_zbuf(ibuf);
578
579         if (ibuf->rect) {
580                 IMB_convert_rgba_to_abgr(ibuf);
581         }
582
583         return(ibuf);
584 }
585
586 /* static utility functions for longimagedata */
587
588 static void interleaverow(uchar *lptr, const uchar *cptr, int z, int n)
589 {
590         lptr += z;
591         while (n--) {
592                 *lptr = *cptr++;
593                 lptr += 4;
594         }
595 }
596
597 static void interleaverow2(float *lptr, const uchar *cptr, int z, int n)
598 {
599         lptr += z;
600         while (n--) {
601                 *lptr = ((cptr[0] << 8) | (cptr[1] << 0)) / (float)0xFFFF;
602                 cptr += 2;
603                 lptr += 4;
604         }
605 }
606
607 static int expandrow2(
608         float *optr, const float *optr_end,
609         const uchar *iptr, const uchar *iptr_end, int z)
610 {
611         ushort pixel, count;
612         float pixel_f;
613
614 #define EXPAND_CAPACITY_AT_INPUT_OK_OR_FAIL(iptr_next) \
615         if (UNLIKELY(iptr_next > iptr_end)) { goto fail; }
616
617 #define EXPAND_CAPACITY_AT_OUTPUT_OK_OR_FAIL(optr_next) \
618         if (UNLIKELY(optr_next > optr_end)) { goto fail; }
619
620         optr += z;
621         optr_end += z;
622         while (1) {
623                 const uchar *iptr_next = iptr + 2;
624                 EXPAND_CAPACITY_AT_INPUT_OK_OR_FAIL(iptr_next);
625                 pixel = (iptr[0] << 8) | (iptr[1] << 0);
626                 iptr = iptr_next;
627
628                 if (!(count = (pixel & 0x7f)) )
629                         return false;
630                 const float *optr_next = optr + count;
631                 EXPAND_CAPACITY_AT_OUTPUT_OK_OR_FAIL(optr_next);
632                 if (pixel & 0x80) {
633                         iptr_next = iptr + (count * 2);
634                         EXPAND_CAPACITY_AT_INPUT_OK_OR_FAIL(iptr_next);
635                         while (count >= 8) {
636                                 optr[0 * 4] = ((iptr[0] << 8) | (iptr[1] << 0)) / (float)0xFFFF;
637                                 optr[1 * 4] = ((iptr[2] << 8) | (iptr[3] << 0)) / (float)0xFFFF;
638                                 optr[2 * 4] = ((iptr[4] << 8) | (iptr[5] << 0)) / (float)0xFFFF;
639                                 optr[3 * 4] = ((iptr[6] << 8) | (iptr[7] << 0)) / (float)0xFFFF;
640                                 optr[4 * 4] = ((iptr[8] << 8) | (iptr[9] << 0)) / (float)0xFFFF;
641                                 optr[5 * 4] = ((iptr[10] << 8) | (iptr[11] << 0)) / (float)0xFFFF;
642                                 optr[6 * 4] = ((iptr[12] << 8) | (iptr[13] << 0)) / (float)0xFFFF;
643                                 optr[7 * 4] = ((iptr[14] << 8) | (iptr[15] << 0)) / (float)0xFFFF;
644                                 optr += 8 * 4;
645                                 iptr += 8 * 2;
646                                 count -= 8;
647                         }
648                         while (count--) {
649                                 *optr = ((iptr[0] << 8) | (iptr[1] << 0)) / (float)0xFFFF;
650                                 iptr += 2;
651                                 optr += 4;
652                         }
653                         BLI_assert(iptr == iptr_next);
654                 }
655                 else {
656                         iptr_next = iptr + 2;
657                         EXPAND_CAPACITY_AT_INPUT_OK_OR_FAIL(iptr_next);
658                         pixel_f = ((iptr[0] << 8) | (iptr[1] << 0)) / (float)0xFFFF;
659                         iptr = iptr_next;
660
661                         while (count >= 8) {
662                                 optr[0 * 4] = pixel_f;
663                                 optr[1 * 4] = pixel_f;
664                                 optr[2 * 4] = pixel_f;
665                                 optr[3 * 4] = pixel_f;
666                                 optr[4 * 4] = pixel_f;
667                                 optr[5 * 4] = pixel_f;
668                                 optr[6 * 4] = pixel_f;
669                                 optr[7 * 4] = pixel_f;
670                                 optr += 8 * 4;
671                                 count -= 8;
672                         }
673                         while (count--) {
674                                 *optr = pixel_f;
675                                 optr += 4;
676                         }
677                         BLI_assert(iptr == iptr_next);
678                 }
679                 BLI_assert(optr == optr_next);
680         }
681         return false;
682
683 #undef EXPAND_CAPACITY_AT_INPUT_OK_OR_FAIL
684 #undef EXPAND_CAPACITY_AT_OUTPUT_OK_OR_FAIL
685 fail:
686         return DIRTY_FLAG_ENCODING;
687 }
688
689 static int expandrow(
690         uchar *optr, const uchar *optr_end,
691         const uchar *iptr, const uchar *iptr_end, int z)
692 {
693         uchar pixel, count;
694
695 #define EXPAND_CAPACITY_AT_INPUT_OK_OR_FAIL(iptr_next) \
696         if (UNLIKELY(iptr_next > iptr_end)) { goto fail; }
697
698 #define EXPAND_CAPACITY_AT_OUTPUT_OK_OR_FAIL(optr_next) \
699         if (UNLIKELY(optr_next > optr_end)) { goto fail; }
700
701         optr += z;
702         optr_end += z;
703         while (1) {
704                 const uchar *iptr_next = iptr + 1;
705                 EXPAND_CAPACITY_AT_INPUT_OK_OR_FAIL(iptr_next);
706                 pixel = *iptr;
707                 iptr = iptr_next;
708                 if (!(count = (pixel & 0x7f)) )
709                         return false;
710                 const uchar *optr_next = optr + ((int)count * 4);
711                 EXPAND_CAPACITY_AT_OUTPUT_OK_OR_FAIL(optr_next);
712
713                 if (pixel & 0x80) {
714                         iptr_next = iptr + count;
715                         EXPAND_CAPACITY_AT_INPUT_OK_OR_FAIL(iptr_next);
716                         while (count >= 8) {
717                                 optr[0 * 4] = iptr[0];
718                                 optr[1 * 4] = iptr[1];
719                                 optr[2 * 4] = iptr[2];
720                                 optr[3 * 4] = iptr[3];
721                                 optr[4 * 4] = iptr[4];
722                                 optr[5 * 4] = iptr[5];
723                                 optr[6 * 4] = iptr[6];
724                                 optr[7 * 4] = iptr[7];
725                                 optr += 8 * 4;
726                                 iptr += 8;
727                                 count -= 8;
728                         }
729                         while (count--) {
730                                 *optr = *iptr++;
731                                 optr += 4;
732                         }
733                         BLI_assert(iptr == iptr_next);
734                 }
735                 else {
736                         iptr_next = iptr + 1;
737                         EXPAND_CAPACITY_AT_INPUT_OK_OR_FAIL(iptr_next);
738                         pixel = *iptr++;
739                         while (count >= 8) {
740                                 optr[0 * 4] = pixel;
741                                 optr[1 * 4] = pixel;
742                                 optr[2 * 4] = pixel;
743                                 optr[3 * 4] = pixel;
744                                 optr[4 * 4] = pixel;
745                                 optr[5 * 4] = pixel;
746                                 optr[6 * 4] = pixel;
747                                 optr[7 * 4] = pixel;
748                                 optr += 8 * 4;
749                                 count -= 8;
750                         }
751                         while (count--) {
752                                 *optr = pixel;
753                                 optr += 4;
754                         }
755                         BLI_assert(iptr == iptr_next);
756                 }
757                 BLI_assert(optr == optr_next);
758         }
759
760         return false;
761
762 #undef EXPAND_CAPACITY_AT_INPUT_OK_OR_FAIL
763 #undef EXPAND_CAPACITY_AT_OUTPUT_OK_OR_FAIL
764 fail:
765         return DIRTY_FLAG_ENCODING;
766 }
767
768 /*
769  *      output_iris -
770  *              copy an array of ints to an iris image file.  Each int
771  *      represents one pixel.  xsize and ysize specify the dimensions of
772  *      the pixel array.  zsize specifies what kind of image file to
773  *      write out.  if zsize is 1, the luminance of the pixels are
774  *      calculated, and a single channel black and white image is saved.
775  *      If zsize is 3, an RGB image file is saved.  If zsize is 4, an
776  *      RGBA image file is saved.
777  *
778  *  Added: zbuf write
779  */
780
781 static int output_iris(uint *lptr, int xsize, int ysize, int zsize, const char *name, int *zptr)
782 {
783         FILE *outf;
784         IMAGE *image;
785         int tablen, y, z, pos, len = 0;
786         uint *starttab, *lengthtab;
787         uchar *rlebuf;
788         uint *lumbuf;
789         int rlebuflen, goodwrite;
790
791         goodwrite = 1;
792         outf = BLI_fopen(name, "wb");
793         if (!outf) return 0;
794
795         tablen = ysize * zsize * sizeof(int);
796
797         image = (IMAGE *)MEM_mallocN(sizeof(IMAGE), "iris image");
798         starttab = (uint *)MEM_mallocN(tablen, "iris starttab");
799         lengthtab = (uint *)MEM_mallocN(tablen, "iris lengthtab");
800         rlebuflen = 1.05 * xsize + 10;
801         rlebuf = (uchar *)MEM_mallocN(rlebuflen, "iris rlebuf");
802         lumbuf = (uint *)MEM_mallocN(xsize * sizeof(int), "iris lumbuf");
803
804         memset(image, 0, sizeof(IMAGE));
805         image->imagic = IMAGIC;
806         image->type = RLE(1);
807         if (zsize > 1)
808                 image->dim = 3;
809         else
810                 image->dim = 2;
811         image->xsize = xsize;
812         image->ysize = ysize;
813         image->zsize = zsize;
814         image->min = 0;
815         image->max = 255;
816         goodwrite *= writeheader(outf, image);
817         fseek(outf, HEADER_SIZE + (2 * tablen), SEEK_SET);
818         pos = HEADER_SIZE + (2 * tablen);
819         
820         for (y = 0; y < ysize; y++) {
821                 for (z = 0; z < zsize; z++) {
822                         
823                         if (zsize == 1) {
824                                 lumrow((uchar *)lptr, (uchar *)lumbuf, xsize);
825                                 len = compressrow((uchar *)lumbuf, rlebuf, CHANOFFSET(z), xsize);
826                         }
827                         else {
828                                 if (z < 4) {
829                                         len = compressrow((uchar *)lptr, rlebuf, CHANOFFSET(z), xsize);
830                                 }
831                                 else if (z < 8 && zptr) {
832                                         len = compressrow((uchar *)zptr, rlebuf, CHANOFFSET(z - 4), xsize);
833                                 }
834                         }
835                         if (len > rlebuflen) {
836                                 fprintf(stderr, "output_iris: rlebuf is too small - bad poop\n");
837                                 exit(1);
838                         }
839                         goodwrite *= fwrite(rlebuf, len, 1, outf);
840                         starttab[y + z * ysize] = pos;
841                         lengthtab[y + z * ysize] = len;
842                         pos += len;
843                 }
844                 lptr += xsize;
845                 if (zptr) zptr += xsize;
846         }
847
848         fseek(outf, HEADER_SIZE, SEEK_SET);
849         goodwrite *= writetab(outf, starttab, tablen);
850         goodwrite *= writetab(outf, lengthtab, tablen);
851         MEM_freeN(image);
852         MEM_freeN(starttab);
853         MEM_freeN(lengthtab);
854         MEM_freeN(rlebuf);
855         MEM_freeN(lumbuf);
856         fclose(outf);
857         if (goodwrite)
858                 return 1;
859         else {
860                 fprintf(stderr, "output_iris: not enough space for image!!\n");
861                 return 0;
862         }
863 }
864
865 /* static utility functions for output_iris */
866
867 static void lumrow(uchar *rgbptr, uchar *lumptr, int n)
868 {
869         lumptr += CHANOFFSET(0);
870         while (n--) {
871                 *lumptr = ILUM(rgbptr[OFFSET_R], rgbptr[OFFSET_G], rgbptr[OFFSET_B]);
872                 lumptr += 4;
873                 rgbptr += 4;
874         }
875 }
876
877 static int compressrow(uchar *lbuf, uchar *rlebuf, int z, int cnt)
878 {
879         uchar *iptr, *ibufend, *sptr, *optr;
880         short todo, cc;
881         int count;
882
883         lbuf += z;
884         iptr = lbuf;
885         ibufend = iptr + cnt * 4;
886         optr = rlebuf;
887
888         while (iptr < ibufend) {
889                 sptr = iptr;
890                 iptr += 8;
891                 while ((iptr < ibufend) && ((iptr[-8] != iptr[-4]) || (iptr[-4] != iptr[0])))
892                         iptr += 4;
893                 iptr -= 8;
894                 count = (iptr - sptr) / 4;
895                 while (count) {
896                         todo = count > 126 ? 126 : count;
897                         count -= todo;
898                         *optr++ = 0x80 | todo;
899                         while (todo > 8) {
900                                 optr[0] = sptr[0 * 4];
901                                 optr[1] = sptr[1 * 4];
902                                 optr[2] = sptr[2 * 4];
903                                 optr[3] = sptr[3 * 4];
904                                 optr[4] = sptr[4 * 4];
905                                 optr[5] = sptr[5 * 4];
906                                 optr[6] = sptr[6 * 4];
907                                 optr[7] = sptr[7 * 4];
908
909                                 optr += 8;
910                                 sptr += 8 * 4;
911                                 todo -= 8;
912                         }
913                         while (todo--) {
914                                 *optr++ = *sptr;
915                                 sptr += 4;
916                         }
917                 }
918                 sptr = iptr;
919                 cc = *iptr;
920                 iptr += 4;
921                 while ( (iptr < ibufend) && (*iptr == cc) )
922                         iptr += 4;
923                 count = (iptr - sptr) / 4;
924                 while (count) {
925                         todo = count > 126 ? 126 : count;
926                         count -= todo;
927                         *optr++ = todo;
928                         *optr++ = cc;
929                 }
930         }
931         *optr++ = 0;
932         return optr - (uchar *)rlebuf;
933 }
934
935 int imb_saveiris(struct ImBuf *ibuf, const char *name, int flags)
936 {
937         short zsize;
938         int ret;
939
940         zsize = (ibuf->planes + 7) >> 3;
941         if (flags & IB_zbuf &&  ibuf->zbuf != NULL) zsize = 8;
942         
943         IMB_convert_rgba_to_abgr(ibuf);
944         test_endian_zbuf(ibuf);
945
946         ret = output_iris(ibuf->rect, ibuf->x, ibuf->y, zsize, name, ibuf->zbuf);
947
948         /* restore! Quite clumsy, 2 times a switch... maybe better a malloc ? */
949         IMB_convert_rgba_to_abgr(ibuf);
950         test_endian_zbuf(ibuf);
951
952         return(ret);
953 }
954