Fix buffer overflows in TIFF, PNG, IRIS, DPX, HDR and AVI loading.
[blender.git] / source / blender / imbuf / intern / tiff.c
index 2630aebef3bf0f8103126b7bb90d9aa1853b8595..afd28bb570b0195dfa35f4038be9f4496b3d90ea 100644 (file)
@@ -53,9 +53,7 @@
 #include "IMB_imbuf_types.h"
 #include "IMB_imbuf.h"
 
-#include "IMB_allocimbuf.h"
 #include "IMB_filetype.h"
-#include "IMB_filter.h"
 
 #include "IMB_colormanagement.h"
 #include "IMB_colormanagement_intern.h"
@@ -81,9 +79,9 @@ static void    imb_tiff_DummyUnmapProc(thandle_t fd, tdata_t base, toff_t size);
 
 /* Structure for in-memory TIFF file. */
 typedef struct ImbTIFFMemFile {
-       unsigned char *mem; /* Location of first byte of TIFF file. */
-       toff_t offset;      /* Current offset within the file.      */
-       tsize_t size;       /* Size of the TIFF file.               */
+       const unsigned char *mem;   /* Location of first byte of TIFF file. */
+       toff_t offset;              /* Current offset within the file.      */
+       tsize_t size;               /* Size of the TIFF file.               */
 } ImbTIFFMemFile;
 #define IMB_TIFF_GET_MEMFILE(x) ((ImbTIFFMemFile *)(x))
 
@@ -276,7 +274,7 @@ static toff_t imb_tiff_SizeProc(thandle_t handle)
        return (toff_t)(mfile->size);
 }
 
-static TIFF *imb_tiff_client_open(ImbTIFFMemFile *memFile, unsigned char *mem, size_t size)
+static TIFF *imb_tiff_client_open(ImbTIFFMemFile *memFile, const unsigned char *mem, size_t size)
 {
        /* open the TIFF client layer interface to the in-memory file */
        memFile->mem = mem;
@@ -306,16 +304,16 @@ static TIFF *imb_tiff_client_open(ImbTIFFMemFile *memFile, unsigned char *mem, s
  * hence my manual comparison. - Jonathan Merritt (lancelet) 4th Sept 2005.
  */
 #define IMB_TIFF_NCB 4      /* number of comparison bytes used */
-int imb_is_a_tiff(unsigned char *mem)
+int imb_is_a_tiff(const unsigned char *mem)
 {
        char big_endian[IMB_TIFF_NCB] = { 0x4d, 0x4d, 0x00, 0x2a };
        char lil_endian[IMB_TIFF_NCB] = { 0x49, 0x49, 0x2a, 0x00 };
 
-       return ( (memcmp(big_endian, mem, IMB_TIFF_NCB) == 0) ||
-                (memcmp(lil_endian, mem, IMB_TIFF_NCB) == 0) );
+       return ((memcmp(big_endian, mem, IMB_TIFF_NCB) == 0) ||
+               (memcmp(lil_endian, mem, IMB_TIFF_NCB) == 0));
 }
 
-static void scanline_contig_16bit(float *rectf, unsigned short *sbuf, int scanline_w, int spp)
+static void scanline_contig_16bit(float *rectf, const unsigned short *sbuf, int scanline_w, int spp)
 {
        int i;
        for (i = 0; i < scanline_w; i++) {
@@ -326,7 +324,7 @@ static void scanline_contig_16bit(float *rectf, unsigned short *sbuf, int scanli
        }
 }
 
-static void scanline_contig_32bit(float *rectf, float *fbuf, int scanline_w, int spp)
+static void scanline_contig_32bit(float *rectf, const float *fbuf, int scanline_w, int spp)
 {
        int i;
        for (i = 0; i < scanline_w; i++) {
@@ -337,14 +335,14 @@ static void scanline_contig_32bit(float *rectf, float *fbuf, int scanline_w, int
        }
 }
 
-static void scanline_separate_16bit(float *rectf, unsigned short *sbuf, int scanline_w, int chan)
+static void scanline_separate_16bit(float *rectf, const unsigned short *sbuf, int scanline_w, int chan)
 {
        int i;
        for (i = 0; i < scanline_w; i++)
                rectf[i * 4 + chan] = sbuf[i] / 65535.0;
 }
 
-static void scanline_separate_32bit(float *rectf, float *fbuf, int scanline_w, int chan)
+static void scanline_separate_32bit(float *rectf, const float *fbuf, int scanline_w, int chan)
 {
        int i;
        for (i = 0; i < scanline_w; i++)
@@ -378,7 +376,7 @@ static void imb_read_tiff_resolution(ImBuf *ibuf, TIFF *image)
  */
 static int imb_read_tiff_pixels(ImBuf *ibuf, TIFF *image)
 {
-       ImBuf *tmpibuf;
+       ImBuf *tmpibuf = NULL;
        int success = 0;
        short bitspersample, spp, config;
        size_t scanline;
@@ -414,16 +412,25 @@ static int imb_read_tiff_pixels(ImBuf *ibuf, TIFF *image)
        if (bitspersample == 32) {
                ib_flag = IB_rectfloat;
                fbuf = (float *)_TIFFmalloc(scanline);
+               if (!fbuf) {
+                       goto cleanup;
+               }
        }
        else if (bitspersample == 16) {
                ib_flag = IB_rectfloat;
                sbuf = (unsigned short *)_TIFFmalloc(scanline);
+               if (!sbuf) {
+                       goto cleanup;
+               }
        }
        else {
                ib_flag = IB_rect;
        }
        
        tmpibuf = IMB_allocImBuf(ibuf->x, ibuf->y, ibuf->planes, ib_flag);
+       if (!tmpibuf) {
+               goto cleanup;
+       }
        
        /* simple RGBA image */
        if (!(bitspersample == 32 || bitspersample == 16)) {
@@ -432,7 +439,7 @@ static int imb_read_tiff_pixels(ImBuf *ibuf, TIFF *image)
        /* contiguous channels: RGBRGBRGB */
        else if (config == PLANARCONFIG_CONTIG) {
                for (row = 0; row < ibuf->y; row++) {
-                       int ib_offset = ibuf->x * ibuf->y * 4 - ibuf->x * 4 * (row + 1);
+                       size_t ib_offset = (size_t)ibuf->x * 4 * ((size_t)ibuf->y - ((size_t)row + 1));
                
                        if (bitspersample == 32) {
                                success |= TIFFReadScanline(image, fbuf, row, 0);
@@ -452,11 +459,11 @@ static int imb_read_tiff_pixels(ImBuf *ibuf, TIFF *image)
                 * but only fill in from the TIFF scanline where necessary. */
                for (chan = 0; chan < 4; chan++) {
                        for (row = 0; row < ibuf->y; row++) {
-                               int ib_offset = ibuf->x * ibuf->y * 4 - ibuf->x * 4 * (row + 1);
+                               size_t ib_offset = (size_t)ibuf->x * 4 * ((size_t)ibuf->y - ((size_t)row + 1));
                                
                                if (bitspersample == 32) {
                                        if (chan == 3 && spp == 3) /* fill alpha if only RGB TIFF */
-                                               fill_vn_fl(fbuf, ibuf->x, 1.0f);
+                                               copy_vn_fl(fbuf, ibuf->x, 1.0f);
                                        else if (chan >= spp) /* for grayscale, duplicate first channel into G and B */
                                                success |= TIFFReadScanline(image, fbuf, row, 0);
                                        else
@@ -466,7 +473,7 @@ static int imb_read_tiff_pixels(ImBuf *ibuf, TIFF *image)
                                }
                                else if (bitspersample == 16) {
                                        if (chan == 3 && spp == 3) /* fill alpha if only RGB TIFF */
-                                               fill_vn_ushort(sbuf, ibuf->x, 65535);
+                                               copy_vn_ushort(sbuf, ibuf->x, 65535);
                                        else if (chan >= spp) /* for grayscale, duplicate first channel into G and B */
                                                success |= TIFFReadScanline(image, fbuf, row, 0);
                                        else
@@ -477,11 +484,6 @@ static int imb_read_tiff_pixels(ImBuf *ibuf, TIFF *image)
                        }
                }
        }
-       
-       if (bitspersample == 32)
-               _TIFFfree(fbuf);
-       else if (bitspersample == 16)
-               _TIFFfree(sbuf);
 
        if (success) {
                /* Code seems to be not needed for 16 bits tif, on PPC G5 OSX (ton) */
@@ -500,6 +502,12 @@ static int imb_read_tiff_pixels(ImBuf *ibuf, TIFF *image)
                tmpibuf->mall &= ~ib_flag;
        }
 
+cleanup:
+       if (bitspersample == 32)
+               _TIFFfree(fbuf);
+       else if (bitspersample == 16)
+               _TIFFfree(sbuf);
+
        IMB_freeImBuf(tmpibuf);
        
        return success;
@@ -522,7 +530,7 @@ void imb_inittiff(void)
  *
  * \return: A newly allocated ImBuf structure if successful, otherwise NULL.
  */
-ImBuf *imb_loadtiff(unsigned char *mem, size_t size, int flags, char colorspace[IM_MAX_SPACE])
+ImBuf *imb_loadtiff(const unsigned char *mem, size_t size, int flags, char colorspace[IM_MAX_SPACE])
 {
        TIFF *image = NULL;
        ImBuf *ibuf = NULL, *hbuf;
@@ -532,6 +540,7 @@ ImBuf *imb_loadtiff(unsigned char *mem, size_t size, int flags, char colorspace[
        int level;
        short spp;
        int ib_depth;
+       int found;
 
        /* check whether or not we have a TIFF file */
        if (size < IMB_TIFF_NCB) {
@@ -560,7 +569,7 @@ ImBuf *imb_loadtiff(unsigned char *mem, size_t size, int flags, char colorspace[
        
        ibuf = IMB_allocImBuf(width, height, ib_depth, 0);
        if (ibuf) {
-               ibuf->ftype = TIF;
+               ibuf->ftype = IMB_FTYPE_TIF;
        }
        else {
                fprintf(stderr, 
@@ -575,10 +584,11 @@ ImBuf *imb_loadtiff(unsigned char *mem, size_t size, int flags, char colorspace[
                if (spp == 4) {
                        unsigned short extra, *extraSampleTypes;
 
-                       TIFFGetField(image, TIFFTAG_EXTRASAMPLES, &extra, &extraSampleTypes);
+                       found = TIFFGetField(image, TIFFTAG_EXTRASAMPLES, &extra, &extraSampleTypes);
 
-                       if (extraSampleTypes[0] == EXTRASAMPLE_ASSOCALPHA)
+                       if (found && (extraSampleTypes[0] == EXTRASAMPLE_ASSOCALPHA)) {
                                ibuf->flags |= IB_alphamode_premul;
+                       }
                }
        }
 
@@ -594,7 +604,7 @@ ImBuf *imb_loadtiff(unsigned char *mem, size_t size, int flags, char colorspace[
                format = NULL;
                TIFFGetField(image, TIFFTAG_PIXAR_TEXTUREFORMAT, &format);
 
-               if (format && strcmp(format, "Plain Texture") == 0 && TIFFIsTiled(image)) {
+               if (format && STREQ(format, "Plain Texture") && TIFFIsTiled(image)) {
                        int numlevel = TIFFNumberOfDirectories(image);
 
                        /* create empty mipmap levels in advance */
@@ -643,7 +653,7 @@ ImBuf *imb_loadtiff(unsigned char *mem, size_t size, int flags, char colorspace[
        return ibuf;
 }
 
-void imb_loadtiletiff(ImBuf *ibuf, unsigned char *mem, size_t size, int tx, int ty, unsigned int *rect)
+void imb_loadtiletiff(ImBuf *ibuf, const unsigned char *mem, size_t size, int tx, int ty, unsigned int *rect)
 {
        TIFF *image = NULL;
        uint32 width, height;
@@ -708,6 +718,7 @@ int imb_savetiff(ImBuf *ibuf, const char *name, int flags)
        float *fromf = NULL;
        float xres, yres;
        int x, y, from_i, to_i, i;
+       int compress_mode = COMPRESSION_NONE;
 
        /* check for a valid number of bytes per pixel.  Like the PNG writer,
         * the TIFF writer supports 1, 3 or 4 bytes per pixel, corresponding
@@ -720,11 +731,18 @@ int imb_savetiff(ImBuf *ibuf, const char *name, int flags)
                return (0);
        }
 
-       if ((ibuf->ftype & TIF_16BIT) && ibuf->rect_float)
+       if ((ibuf->foptions.flag & TIF_16BIT) && ibuf->rect_float)
                bitspersample = 16;
        else
                bitspersample = 8;
 
+       if (ibuf->foptions.flag & TIF_COMPRESS_DEFLATE)
+               compress_mode = COMPRESSION_DEFLATE;
+       else if (ibuf->foptions.flag & TIF_COMPRESS_LZW)
+               compress_mode = COMPRESSION_LZW;
+       else if (ibuf->foptions.flag & TIF_COMPRESS_PACKBITS)
+               compress_mode = COMPRESSION_PACKBITS;
+
        /* open TIFF file for writing */
        if (flags & IB_mem) {
                /* bork at the creation of a TIFF in memory */
@@ -805,26 +823,51 @@ int imb_savetiff(ImBuf *ibuf, const char *name, int flags)
        }
 
        /* copy pixel data.  While copying, we flip the image vertically. */
+       const int channels_in_float = ibuf->channels ? ibuf->channels : 4;
        for (x = 0; x < ibuf->x; x++) {
                for (y = 0; y < ibuf->y; y++) {
-                       from_i = 4 * (y * ibuf->x + x);
+                       from_i = ((size_t)channels_in_float) * (y * ibuf->x + x);
                        to_i   = samplesperpixel * ((ibuf->y - y - 1) * ibuf->x + x);
 
                        if (pixels16) {
                                /* convert from float source */
                                float rgb[4];
-                               
-                               if (ibuf->float_colorspace) {
-                                       /* float buffer was managed already, no need in color space conversion */
-                                       copy_v3_v3(rgb, &fromf[from_i]);
+
+                               if (channels_in_float == 3 || channels_in_float == 4) {
+                                       if (ibuf->float_colorspace ||
+                                           (ibuf->colormanage_flag & IMB_COLORMANAGE_IS_DATA))
+                                       {
+                                               /* Float buffer was managed already, no need in color
+                                                * space conversion.
+                                                */
+                                               copy_v3_v3(rgb, &fromf[from_i]);
+                                       }
+                                       else {
+                                               /* Standard linear-to-srgb conversion if float buffer
+                                                * wasn't managed.
+                                                */
+                                               linearrgb_to_srgb_v3_v3(rgb, &fromf[from_i]);
+                                       }
+                                       if (channels_in_float == 4) {
+                                               rgb[3] = fromf[from_i + 3];
+                                       }
+                                       else {
+                                               rgb[3] = 1.0f;
+                                       }
                                }
                                else {
-                                       /* standard linear-to-srgb conversion if float buffer wasn't managed */
-                                       linearrgb_to_srgb_v3_v3(rgb, &fromf[from_i]);
+                                       if (ibuf->float_colorspace ||
+                                           (ibuf->colormanage_flag & IMB_COLORMANAGE_IS_DATA))
+                                       {
+                                               rgb[0] = fromf[from_i];
+                                       }
+                                       else {
+                                               rgb[0] = linearrgb_to_srgb(fromf[from_i]);
+                                       }
+                                       rgb[1] = rgb[2] = rgb[0];
+                                       rgb[3] = 1.0f;
                                }
 
-                               rgb[3] = fromf[from_i + 3];
-
                                for (i = 0; i < samplesperpixel; i++, to_i++)
                                        to16[to_i] = FTOUSHORT(rgb[i]);
                        }
@@ -839,7 +882,7 @@ int imb_savetiff(ImBuf *ibuf, const char *name, int flags)
        TIFFSetField(image, TIFFTAG_IMAGEWIDTH,      ibuf->x);
        TIFFSetField(image, TIFFTAG_IMAGELENGTH,     ibuf->y);
        TIFFSetField(image, TIFFTAG_ROWSPERSTRIP,    ibuf->y);
-       TIFFSetField(image, TIFFTAG_COMPRESSION, COMPRESSION_DEFLATE);
+       TIFFSetField(image, TIFFTAG_COMPRESSION, compress_mode);
        TIFFSetField(image, TIFFTAG_FILLORDER, FILLORDER_MSB2LSB);
        TIFFSetField(image, TIFFTAG_PLANARCONFIG, PLANARCONFIG_CONTIG);
 
@@ -857,7 +900,7 @@ int imb_savetiff(ImBuf *ibuf, const char *name, int flags)
        TIFFSetField(image, TIFFTAG_RESOLUTIONUNIT,  RESUNIT_INCH);
        if (TIFFWriteEncodedStrip(image, 0,
                                  (bitspersample == 16) ? (unsigned char *)pixels16 : pixels,
-                                 ibuf->x * ibuf->y * samplesperpixel * bitspersample / 8) == -1)
+                                 (size_t)ibuf->x * ibuf->y * samplesperpixel * bitspersample / 8) == -1)
        {
                fprintf(stderr,
                        "imb_savetiff: Could not write encoded TIFF.\n");