Fix buffer overflows in TIFF, PNG, IRIS, DPX, HDR and AVI loading.
[blender.git] / source / blender / avi / intern / avi_mjpeg.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
19  * All rights reserved.
20  *
21  * The Original Code is: all of this file.
22  *
23  * Contributor(s): none yet.
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  *
27  */
28
29 /** \file blender/avi/intern/avi_mjpeg.c
30  *  \ingroup avi
31  *
32  * This is external code. Converts between avi and mpeg/jpeg.
33  */
34
35 #include <stdlib.h>
36 #include <string.h>
37
38 #include "AVI_avi.h"
39
40 #include "MEM_guardedalloc.h"
41
42 #include "IMB_imbuf.h"
43
44 #include "jpeglib.h"
45 #include "jerror.h"
46
47 #include "avi_mjpeg.h"
48
49 static void jpegmemdestmgr_build(j_compress_ptr cinfo, unsigned char *buffer, size_t bufsize);
50 static void jpegmemsrcmgr_build(j_decompress_ptr dinfo, unsigned char *buffer, size_t bufsize);
51
52 static size_t numbytes;
53
54 static void add_huff_table(j_decompress_ptr dinfo, JHUFF_TBL **htblptr, const UINT8 *bits, const UINT8 *val)
55 {
56         if (*htblptr == NULL)
57                 *htblptr = jpeg_alloc_huff_table((j_common_ptr) dinfo);
58
59         memcpy((*htblptr)->bits, bits, sizeof((*htblptr)->bits));
60         memcpy((*htblptr)->huffval, val, sizeof((*htblptr)->huffval));
61
62         /* Initialize sent_table false so table will be written to JPEG file. */
63         (*htblptr)->sent_table = false;
64 }
65
66 /* Set up the standard Huffman tables (cf. JPEG standard section K.3) */
67 /* IMPORTANT: these are only valid for 8-bit data precision! */
68
69 static void std_huff_tables(j_decompress_ptr dinfo)
70 {
71         static const UINT8 bits_dc_luminance[17] =
72         { /* 0-base */
73                 0, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0
74         };
75         static const UINT8 val_dc_luminance[] =
76         {
77                 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
78         };
79
80         static const UINT8 bits_dc_chrominance[17] =
81         { /* 0-base */
82                 0, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0
83         };
84         static const UINT8 val_dc_chrominance[] =
85         {
86                 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
87         };
88
89         static const UINT8 bits_ac_luminance[17] =
90         { /* 0-base */
91                 0, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7d
92         };
93         static const UINT8 val_ac_luminance[] =
94         {
95                 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12,
96                 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07,
97                 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08,
98                 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0,
99                 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16,
100                 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28,
101                 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39,
102                 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49,
103                 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59,
104                 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69,
105                 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79,
106                 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89,
107                 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98,
108                 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
109                 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6,
110                 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5,
111                 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4,
112                 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2,
113                 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea,
114                 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8,
115                 0xf9, 0xfa
116         };
117         static const UINT8 bits_ac_chrominance[17] =
118         { /* 0-base */
119                 0, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77
120         };
121         static const UINT8 val_ac_chrominance[] =
122         {
123                 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21,
124                 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71,
125                 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
126                 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0,
127                 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34,
128                 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26,
129                 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38,
130                 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48,
131                 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
132                 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68,
133                 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
134                 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
135                 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96,
136                 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5,
137                 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4,
138                 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3,
139                 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2,
140                 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda,
141                 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9,
142                 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8,
143                 0xf9, 0xfa
144         };
145
146         add_huff_table(dinfo, &dinfo->dc_huff_tbl_ptrs[0],
147                        bits_dc_luminance, val_dc_luminance);
148         add_huff_table(dinfo, &dinfo->ac_huff_tbl_ptrs[0],
149                        bits_ac_luminance, val_ac_luminance);
150         add_huff_table(dinfo, &dinfo->dc_huff_tbl_ptrs[1],
151                        bits_dc_chrominance, val_dc_chrominance);
152         add_huff_table(dinfo, &dinfo->ac_huff_tbl_ptrs[1],
153                        bits_ac_chrominance, val_ac_chrominance);
154 }
155
156 static int Decode_JPEG(unsigned char *inBuffer, unsigned char *outBuffer, unsigned int width, unsigned int height, size_t bufsize)
157 {
158         struct jpeg_decompress_struct dinfo;
159         struct jpeg_error_mgr jerr;
160         
161         (void)width; /* unused */
162
163         numbytes = 0;
164
165         dinfo.err = jpeg_std_error(&jerr);
166         jpeg_create_decompress(&dinfo);
167         jpegmemsrcmgr_build(&dinfo, inBuffer, bufsize);
168         jpeg_read_header(&dinfo, true);
169         if (dinfo.dc_huff_tbl_ptrs[0] == NULL) {
170                 std_huff_tables(&dinfo);
171         }
172         dinfo.out_color_space = JCS_RGB;
173         dinfo.dct_method = JDCT_IFAST;
174
175         jpeg_start_decompress(&dinfo);
176
177         size_t rowstride = dinfo.output_width * dinfo.output_components;
178         for (size_t y = 0; y < dinfo.output_height; y++) {
179                 jpeg_read_scanlines(&dinfo, (JSAMPARRAY) &outBuffer, 1);
180                 outBuffer += rowstride;
181         }
182         jpeg_finish_decompress(&dinfo);
183
184         if (dinfo.output_height >= height) return 0;
185         
186         inBuffer += numbytes;
187         jpegmemsrcmgr_build(&dinfo, inBuffer, bufsize - numbytes);
188
189         numbytes = 0;
190         jpeg_read_header(&dinfo, true);
191         if (dinfo.dc_huff_tbl_ptrs[0] == NULL) {
192                 std_huff_tables(&dinfo);
193         }
194
195         jpeg_start_decompress(&dinfo);
196         rowstride = dinfo.output_width * dinfo.output_components;
197         for (size_t y = 0; y < dinfo.output_height; y++) {
198                 jpeg_read_scanlines(&dinfo, (JSAMPARRAY) &outBuffer, 1);
199                 outBuffer += rowstride;
200         }
201         jpeg_finish_decompress(&dinfo);
202         jpeg_destroy_decompress(&dinfo);
203         
204         return 1;
205 }
206
207 static void Compress_JPEG(int quality, unsigned char *outbuffer, const unsigned char *inBuffer, int width, int height, size_t bufsize)
208 {
209         struct jpeg_compress_struct cinfo;
210         struct jpeg_error_mgr jerr;
211         unsigned char marker[60];
212
213         cinfo.err = jpeg_std_error(&jerr);
214         jpeg_create_compress(&cinfo);
215         jpegmemdestmgr_build(&cinfo, outbuffer, bufsize);
216
217         cinfo.image_width = width;
218         cinfo.image_height = height;
219         cinfo.input_components = 3;
220         cinfo.in_color_space = JCS_RGB;
221
222         jpeg_set_defaults(&cinfo);
223         jpeg_set_colorspace(&cinfo, JCS_YCbCr);
224                 
225         jpeg_set_quality(&cinfo, quality, true);
226
227         cinfo.dc_huff_tbl_ptrs[0]->sent_table = true;
228         cinfo.dc_huff_tbl_ptrs[1]->sent_table = true;
229         cinfo.ac_huff_tbl_ptrs[0]->sent_table = true;
230         cinfo.ac_huff_tbl_ptrs[1]->sent_table = true;
231
232         cinfo.comp_info[0].component_id = 0;
233         cinfo.comp_info[0].v_samp_factor = 1;
234         cinfo.comp_info[1].component_id = 1;
235         cinfo.comp_info[2].component_id = 2;
236
237         cinfo.write_JFIF_header = false;
238
239         jpeg_start_compress(&cinfo, false);
240
241         int i = 0;
242         marker[i++] = 'A';
243         marker[i++] = 'V';
244         marker[i++] = 'I';
245         marker[i++] = '1';
246         marker[i++] = 0;
247         while (i < 60)
248                 marker[i++] = 32;
249
250         jpeg_write_marker(&cinfo, JPEG_APP0, marker, 60);
251
252         i = 0;
253         while (i < 60)
254                 marker[i++] = 0;
255
256         jpeg_write_marker(&cinfo, JPEG_COM, marker, 60);
257
258         size_t rowstride = cinfo.image_width * cinfo.input_components;
259         for (size_t y = 0; y < cinfo.image_height; y++) {
260                 jpeg_write_scanlines(&cinfo, (JSAMPARRAY) &inBuffer, 1);
261                 inBuffer += rowstride;
262         }
263         jpeg_finish_compress(&cinfo);
264         jpeg_destroy_compress(&cinfo);
265 }
266
267 static void interlace(unsigned char *to, unsigned char *from, int width, int height)
268 {
269         size_t i, rowstride = width * 3;
270         
271         for (i = 0; i < height; i++) {
272                 if (i & 1)
273                         memcpy(&to[i * rowstride], &from[(i / 2 + height / 2) * rowstride], rowstride);
274                 else 
275                         memcpy(&to[i * rowstride], &from[(i / 2) * rowstride], rowstride);
276         }
277 }
278
279 static void deinterlace(int odd, unsigned char *to, unsigned char *from, int width, int height)
280 {
281         size_t i, rowstride = width * 3;
282         
283         for (i = 0; i < height; i++) {
284                 if ((i & 1) == odd)
285                         memcpy(&to[(i / 2 + height / 2) * rowstride], &from[i * rowstride], rowstride);
286                 else 
287                         memcpy(&to[(i / 2) * rowstride], &from[i * rowstride], rowstride);
288         }
289 }
290
291 void *avi_converter_from_mjpeg(AviMovie *movie, int stream, unsigned char *buffer, size_t *size)
292 {
293         int deint;
294         unsigned char *buf;
295
296         (void)stream; /* unused */
297
298         buf = imb_alloc_pixels(movie->header->Height, movie->header->Width, 3, sizeof(unsigned char), "avi.avi_converter_from_mjpeg 1");
299         if (!buf) {
300                 return NULL;
301         }
302
303         deint = Decode_JPEG(buffer, buf, movie->header->Width, movie->header->Height, *size);
304         
305         MEM_freeN(buffer);
306         
307         if (deint) {
308                 buffer = imb_alloc_pixels(movie->header->Height, movie->header->Width, 3, sizeof(unsigned char), "avi.avi_converter_from_mjpeg 2");
309                 if (buffer) {
310                         interlace(buffer, buf, movie->header->Width, movie->header->Height);
311                 }
312                 MEM_freeN(buf);
313         
314                 buf = buffer;
315         }
316                 
317         return buf;
318 }
319
320 void *avi_converter_to_mjpeg(AviMovie *movie, int stream, unsigned char *buffer, size_t *size)
321 {
322         unsigned char *buf;
323         size_t bufsize = *size;
324         
325         numbytes = 0;
326         *size = 0;
327
328         buf = imb_alloc_pixels(movie->header->Height, movie->header->Width, 3, sizeof(unsigned char), "avi.avi_converter_to_mjpeg 1");
329         if (!buf) {
330                 return NULL;
331         }
332
333         if (!movie->interlace) {
334                 Compress_JPEG(movie->streams[stream].sh.Quality / 100,
335                               buf, buffer,
336                               movie->header->Width,
337                               movie->header->Height,
338                               bufsize);
339                 *size += numbytes;
340         }
341         else {
342                 deinterlace(movie->odd_fields, buf, buffer, movie->header->Width, movie->header->Height);
343                 MEM_freeN(buffer);
344         
345                 buffer = buf;
346                 buf = imb_alloc_pixels(movie->header->Height, movie->header->Width, 3, sizeof(unsigned char), "avi.avi_converter_to_mjpeg 1");
347
348                 if (buf) {
349                         Compress_JPEG(movie->streams[stream].sh.Quality / 100,
350                                       buf, buffer,
351                                       movie->header->Width,
352                                       movie->header->Height / 2,
353                                       bufsize / 2);
354                         *size += numbytes;
355                         numbytes = 0;
356                         Compress_JPEG(movie->streams[stream].sh.Quality / 100,
357                                       buf + *size, buffer + (size_t)(movie->header->Height / 2) * (size_t)movie->header->Width * 3,
358                                       movie->header->Width,
359                                       movie->header->Height / 2,
360                                       bufsize / 2);
361                         *size += numbytes;
362                 }
363         }
364
365         MEM_freeN(buffer);
366         return buf;
367 }
368
369
370 /* Compression from memory */
371
372 static void jpegmemdestmgr_init_destination(j_compress_ptr cinfo)
373 {
374         (void)cinfo; /* unused */
375 }
376
377 static boolean jpegmemdestmgr_empty_output_buffer(j_compress_ptr cinfo)
378 {
379         (void)cinfo; /* unused */
380         return true;
381 }
382
383 static void jpegmemdestmgr_term_destination(j_compress_ptr cinfo)
384 {
385         numbytes -= cinfo->dest->free_in_buffer;
386
387         MEM_freeN(cinfo->dest);
388 }
389
390 static void jpegmemdestmgr_build(j_compress_ptr cinfo, unsigned char *buffer, size_t bufsize)
391 {
392         cinfo->dest = MEM_mallocN(sizeof(*(cinfo->dest)), "avi.jpegmemdestmgr_build");
393         
394         cinfo->dest->init_destination = jpegmemdestmgr_init_destination;
395         cinfo->dest->empty_output_buffer = jpegmemdestmgr_empty_output_buffer;
396         cinfo->dest->term_destination = jpegmemdestmgr_term_destination;
397
398         cinfo->dest->next_output_byte = buffer;
399         cinfo->dest->free_in_buffer = bufsize;
400         
401         numbytes = bufsize;
402 }
403
404 /* Decompression from memory */
405
406 static void jpegmemsrcmgr_init_source(j_decompress_ptr dinfo)
407 {
408         (void)dinfo;
409 }
410
411 static boolean jpegmemsrcmgr_fill_input_buffer(j_decompress_ptr dinfo)
412 {
413         unsigned char *buf = (unsigned char *) dinfo->src->next_input_byte - 2;
414         
415         /* if we get called, must have run out of data */
416         WARNMS(dinfo, JWRN_JPEG_EOF);
417         
418         buf[0] = (JOCTET) 0xFF;
419         buf[1] = (JOCTET) JPEG_EOI;
420         
421         dinfo->src->next_input_byte = buf;
422         dinfo->src->bytes_in_buffer = 2;
423         
424         return true;
425 }
426
427 static void jpegmemsrcmgr_skip_input_data(j_decompress_ptr dinfo, long skipcnt)
428 {
429         if (dinfo->src->bytes_in_buffer < skipcnt)
430                 skipcnt = dinfo->src->bytes_in_buffer;
431
432         dinfo->src->next_input_byte += skipcnt;
433         dinfo->src->bytes_in_buffer -= skipcnt;
434 }
435
436 static void jpegmemsrcmgr_term_source(j_decompress_ptr dinfo)
437 {
438         numbytes -= dinfo->src->bytes_in_buffer;
439         
440         MEM_freeN(dinfo->src);
441 }
442
443 static void jpegmemsrcmgr_build(j_decompress_ptr dinfo, unsigned char *buffer, size_t bufsize)
444 {
445         dinfo->src = MEM_mallocN(sizeof(*(dinfo->src)), "avi.jpegmemsrcmgr_build");
446         
447         dinfo->src->init_source = jpegmemsrcmgr_init_source;
448         dinfo->src->fill_input_buffer = jpegmemsrcmgr_fill_input_buffer;
449         dinfo->src->skip_input_data = jpegmemsrcmgr_skip_input_data;
450         dinfo->src->resync_to_restart = jpeg_resync_to_restart;
451         dinfo->src->term_source = jpegmemsrcmgr_term_source;
452         
453         dinfo->src->bytes_in_buffer = bufsize;
454         dinfo->src->next_input_byte = buffer;
455
456         numbytes = bufsize;
457 }