This is a fun one ;)
[blender.git] / source / blender / decrypt / intern / BLO_decrypt.c
1 /**
2  * $Id$
3  *
4  * ***** BEGIN GPL/BL DUAL LICENSE BLOCK *****
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version. The Blender
10  * Foundation also sells licenses for use in proprietary software under
11  * the Blender License.  See http://www.blender.org/BL/ for information
12  * about this.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software Foundation,
21  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
22  *
23  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
24  * All rights reserved.
25  *
26  * The Original Code is: all of this file.
27  *
28  * Contributor(s): none yet.
29  *
30  * ***** END GPL/BL DUAL LICENSE BLOCK *****
31  * openssl decryption wrapper library
32  */
33
34 #include <stdio.h>
35 #include <string.h> /* memcpy, strcpy */
36 #include <stdlib.h>
37
38 #include "openssl/rsa.h"
39 #include "openssl/rc4.h"
40 #include "openssl/err.h"
41 #include "zlib.h"
42
43 #include "GEN_messaging.h"
44
45 #include "BLO_getPubKey.h"      // real and stub implemented at writestream ...
46
47 #include "BLO_readStreamGlue.h"
48 #include "BLO_decrypt.h"
49 #include "BLO_en_de_cryptHeader.h"
50
51 #ifdef HAVE_CONFIG_H
52 #include <config.h>
53 #endif
54
55 struct decryptStructType {
56         struct readStreamGlueStruct *streamGlue;
57         unsigned int streamDone;
58         unsigned char *deCryptKey;
59         int deCryptKeyLen;                      // NOTE: signed int
60         unsigned char headerbuffer[EN_DE_CRYPTHEADERSTRUCTSIZE];
61         uint32_t datacrc;                       // crypted data checksum
62         struct BLO_en_de_cryptHeaderStruct *streamHeader;
63         RC4_KEY rc4_key;
64         void *endControl;
65 };
66
67         BLO_decryptStructHandle
68 BLO_decrypt_begin(
69         void *endControl)
70 {
71         struct decryptStructType *control;
72         control = malloc(sizeof(struct decryptStructType));
73         if (!control) return NULL;
74
75         control->streamGlue = NULL;
76         control->streamDone = 0;
77         control->deCryptKey = NULL;
78         control->deCryptKeyLen = 0;
79         strcpy(control->headerbuffer, "");
80         control->datacrc = 0;
81
82         control->streamHeader = malloc(EN_DE_CRYPTHEADERSTRUCTSIZE);
83         if (!control->streamHeader) {
84                 free(control);
85                 return NULL;
86         }
87
88         control->streamHeader->magic = 0;
89         control->streamHeader->length = 0;
90         strcpy(control->streamHeader->pubKey, "");
91         control->streamHeader->pubKeyLen = 0;
92         strcpy(control->streamHeader->cryptedKey, "");
93         control->streamHeader->cryptedKeyLen = 0;
94         control->streamHeader->datacrc = 0;
95         control->streamHeader->headercrc = 0;
96         control->endControl = endControl;
97
98         return((BLO_decryptStructHandle) control);
99 }
100
101         int
102 BLO_decrypt_process(
103         BLO_decryptStructHandle BLO_decryptHandle,
104         unsigned char *data,
105         unsigned int dataIn)
106 {
107         int err = 0;
108         struct decryptStructType *BLO_decrypt =
109                 (struct decryptStructType *) BLO_decryptHandle;
110
111         if (!BLO_decrypt) {
112                 err = BRS_SETFUNCTION(BRS_DECRYPT) |
113                           BRS_SETGENERR(BRS_NULL);
114                 return err;
115         }
116
117         /* First check if we have our header filled in yet */
118         if (BLO_decrypt->streamHeader->cryptedKeyLen == 0) {
119                 unsigned int processed;
120                 if (dataIn == 0) return err;    /* really need data to do anything */
121                 processed = ((dataIn + BLO_decrypt->streamDone) <=
122                                          EN_DE_CRYPTHEADERSTRUCTSIZE)
123                                          ? dataIn : EN_DE_CRYPTHEADERSTRUCTSIZE;
124                 memcpy(BLO_decrypt->headerbuffer + BLO_decrypt->streamDone,
125                            data, processed);
126                 BLO_decrypt->streamDone += processed;
127                 dataIn -= processed;
128                 data += processed;
129                 if (BLO_decrypt->streamDone == EN_DE_CRYPTHEADERSTRUCTSIZE) {
130                         /* we have the whole header, absorb it */
131                         struct BLO_en_de_cryptHeaderStruct *header;
132                         uint32_t crc;
133                         //static unsigned char rsa_e[] = "\x11";
134                         static unsigned char rsa_e[] = "\x01\x00\x01";
135                         RSA *rsa = NULL;
136                         unsigned char *publisherPubKey;
137                         int publisherPubKeyLen;
138
139                         header = (struct BLO_en_de_cryptHeaderStruct *)
140                                 BLO_decrypt->headerbuffer;
141                         crc = crc32(0L, (const Bytef *) header,
142                                                 EN_DE_CRYPTHEADERSTRUCTSIZE - 4);
143
144                         if (header->magic == 'A') {
145 #ifndef NDEBUG
146                                 fprintf(GEN_errorstream,
147                                                 "BLO_en_de_cryptHeaderStruct Magic confirmed\n");
148 #endif
149                         } else {
150 #ifndef NDEBUG
151                                 fprintf(GEN_errorstream,
152                                                 "ERROR BLO_en_de_cryptHeaderStruct Magic NOT confirmed\n");
153 #endif
154                                 err = BRS_SETFUNCTION(BRS_DECRYPT) |
155                                           BRS_SETGENERR(BRS_MAGIC);
156                                 if (BLO_decrypt->streamGlue) free(BLO_decrypt->streamGlue);
157                                 if (BLO_decrypt->streamHeader) free(BLO_decrypt->streamHeader);
158                                 if (BLO_decrypt->deCryptKey) free(BLO_decrypt->deCryptKey);
159                                 free(BLO_decrypt);
160                                 return err;
161                         }
162
163                         if (crc == ntohl(header->headercrc)) {
164 #ifndef NDEBUG
165                                 fprintf(GEN_errorstream,
166                                                 "BLO_en_de_cryptHeader CRC correct\n");
167 #endif
168                         } else {
169 #ifndef NDEBUG
170                                 fprintf(GEN_errorstream,
171                                                 "ERROR BLO_en_de_cryptHeader CRC NOT correct\n");
172 #endif
173                                 err = BRS_SETFUNCTION(BRS_DECRYPT) |
174                                           BRS_SETGENERR(BRS_CRCHEADER);
175                                 if (BLO_decrypt->streamGlue) free(BLO_decrypt->streamGlue);
176                                 if (BLO_decrypt->streamHeader) free(BLO_decrypt->streamHeader);
177                                 if (BLO_decrypt->deCryptKey) free(BLO_decrypt->deCryptKey);
178                                 free(BLO_decrypt);
179                                 return err;
180                         }
181                         BLO_decrypt->streamHeader->length = ntohl(header->length);
182                         BLO_decrypt->streamHeader->pubKeyLen = ntohl(header->pubKeyLen);
183                         memcpy(BLO_decrypt->streamHeader->pubKey, header->pubKey,
184                                    BLO_decrypt->streamHeader->pubKeyLen);
185
186                         // case Publisher: get the .BPkey public key
187                         // case Player/Plugin: simply use the data stream public key
188                         err = getPubKey(BLO_decrypt->streamHeader->pubKey,
189                                                         BLO_decrypt->streamHeader->pubKeyLen,
190                                                         &publisherPubKey,
191                                                         &publisherPubKeyLen);
192                         switch (err) {
193                                 case 0:
194                                         // everything OK
195                                         break;
196                                 case 1:
197                                         // publisher without a key
198                                 case 2:
199                                         // publishers keylen !=
200                                 case 3:
201                                         // publishers key !=
202                                 default:
203 #ifndef NDEBUG
204                                         fprintf(GEN_errorstream,
205                                                 "ALERT users-pubKey != datastream-pubKey, stop reading\n");
206 #endif
207                                         err = BRS_SETFUNCTION(BRS_DECRYPT) |
208                                                   BRS_SETSPECERR(BRS_NOTOURPUBKEY);
209                                         if (BLO_decrypt->streamGlue) free(BLO_decrypt->streamGlue);
210                                         if (BLO_decrypt->streamHeader) free(BLO_decrypt->streamHeader);
211                                         if (BLO_decrypt->deCryptKey) free(BLO_decrypt->deCryptKey);
212                                         free(BLO_decrypt);
213                                         return err;
214                                         break;
215                         }
216
217                         BLO_decrypt->streamHeader->cryptedKeyLen =
218                                 ntohl(header->cryptedKeyLen);
219                         memcpy(BLO_decrypt->streamHeader->cryptedKey,
220                                    header->cryptedKey,
221                                    BLO_decrypt->streamHeader->cryptedKeyLen);
222 #ifndef NDEBUG
223                         fprintf(GEN_errorstream,
224                                         "BLO_decrypt_process gets %u bytes\n",
225                                         (unsigned int) BLO_decrypt->streamHeader->length);
226 #endif
227                         BLO_decrypt->streamHeader->datacrc = ntohl(header->datacrc);
228
229                         // finished absorbing and testing the header, create rsa key from it
230                         rsa = RSA_new();
231                         if (rsa == NULL) {
232 #ifndef NDEBUG
233                                 fprintf(GEN_errorstream,
234                                                 "Error in RSA_new\n");
235 #endif
236                                 err = BRS_SETFUNCTION(BRS_DECRYPT) |
237                                           BRS_SETSPECERR(BRS_RSANEWERROR);
238                                 if (BLO_decrypt->streamGlue) free(BLO_decrypt->streamGlue);
239                                 if (BLO_decrypt->streamHeader) free(BLO_decrypt->streamHeader);
240                                 if (BLO_decrypt->deCryptKey) free(BLO_decrypt->deCryptKey);
241                                 free(BLO_decrypt);
242                                 return err;
243                         }
244                         // static exponent
245                         rsa->e = BN_bin2bn(rsa_e, sizeof(rsa_e)-1, rsa->e);
246
247                         // public part into rsa->n
248                         rsa->n = BN_bin2bn(publisherPubKey,
249                                                            publisherPubKeyLen,
250                                                            rsa->n);
251
252                         //DEBUG RSA_print_fp(stdout, rsa, 0);
253
254                         BLO_decrypt->deCryptKey = malloc(RSA_size(rsa) *
255                                                                                          sizeof(unsigned char));
256                         if (! BLO_decrypt->deCryptKey) {
257                                 err = BRS_SETFUNCTION(BRS_DECRYPT) |
258                                           BRS_SETGENERR(BRS_MALLOC);
259                                 if (BLO_decrypt->streamGlue) free(BLO_decrypt->streamGlue);
260                                 if (BLO_decrypt->streamHeader) free(BLO_decrypt->streamHeader);
261                                 if (BLO_decrypt->deCryptKey) free(BLO_decrypt->deCryptKey);
262                                 free(BLO_decrypt);
263                                 RSA_free(rsa);
264                                 return err;
265                         }
266
267                         // decrypt the cryptkey
268                         BLO_decrypt->deCryptKeyLen = RSA_public_decrypt(
269                                 BLO_decrypt->streamHeader->cryptedKeyLen,
270                                 BLO_decrypt->streamHeader->cryptedKey,
271                                 BLO_decrypt->deCryptKey,
272                                 rsa, RSA_PKCS1_PADDING);
273                         if (BLO_decrypt->deCryptKeyLen == -1) {
274 #ifndef NDEBUG
275                                 fprintf(GEN_errorstream,
276                                                 "Error in RSA_public_decrypt %s\n",
277                                                 ERR_error_string(ERR_get_error(),
278                                                                                  NULL));
279 #endif
280                                 err = BRS_SETFUNCTION(BRS_DECRYPT) |
281                                           BRS_SETSPECERR(BRS_DECRYPTERROR);
282                                 if (BLO_decrypt->streamGlue) free(BLO_decrypt->streamGlue);
283                                 if (BLO_decrypt->streamHeader) free(BLO_decrypt->streamHeader);
284                                 if (BLO_decrypt->deCryptKey) free(BLO_decrypt->deCryptKey);
285                                 free(BLO_decrypt);
286                                 RSA_free(rsa);
287                                 return err;
288                         }
289
290                         // Finally set the RC4 deCryptKey
291                         RC4_set_key(&(BLO_decrypt->rc4_key),
292                                                 BLO_decrypt->deCryptKeyLen,
293                                                 BLO_decrypt->deCryptKey);
294
295                         RSA_free(rsa);
296                 }
297         }
298
299         /* Is there really (still) new data available ? */
300         if (dataIn > 0) {
301                 unsigned char *deCryptBuf = malloc(dataIn);
302                 if (! deCryptBuf) {
303                         err = BRS_SETFUNCTION(BRS_DECRYPT) |
304                                   BRS_SETGENERR(BRS_MALLOC);
305                         if (BLO_decrypt->streamGlue) free(BLO_decrypt->streamGlue);
306                         if (BLO_decrypt->streamHeader) free(BLO_decrypt->streamHeader);
307                         if (BLO_decrypt->deCryptKey) free(BLO_decrypt->deCryptKey);
308                         free(BLO_decrypt);
309                         return err;
310                 }
311
312                 BLO_decrypt->streamDone += dataIn;
313
314                 // update datacrc
315                 BLO_decrypt->datacrc = crc32(
316                         BLO_decrypt->datacrc, (const Bytef *) data, dataIn);
317
318                 // TODO FIXME we might need to keylength-align the data !
319                 RC4(&(BLO_decrypt->rc4_key), dataIn, data, deCryptBuf);
320
321                 // give data to streamGlueRead, it will find out what to do next
322                 err = readStreamGlue(
323                         BLO_decrypt->endControl,
324                         &(BLO_decrypt->streamGlue),
325                         (unsigned char *) deCryptBuf,
326                         dataIn);
327
328                 free(deCryptBuf);
329         }
330         return err;
331 }
332
333 /**
334  * openssl decrypt final call and cleanup
335  * @param BLO_decrypt Pointer to decrypt control structure
336  * @retval streamGlueRead return value
337  */
338         int
339 BLO_decrypt_end(
340         BLO_decryptStructHandle BLO_decryptHandle)
341 {
342         int err = 0;
343         struct decryptStructType *BLO_decrypt =
344                 (struct decryptStructType *) BLO_decryptHandle;
345
346         if (!BLO_decrypt) {
347                 err = BRS_SETFUNCTION(BRS_DECRYPT) |
348                           BRS_SETGENERR(BRS_NULL);
349                 if (BLO_decrypt->streamGlue) free(BLO_decrypt->streamGlue);
350                 if (BLO_decrypt->streamHeader) free(BLO_decrypt->streamHeader);
351                 if (BLO_decrypt->deCryptKey) free(BLO_decrypt->deCryptKey);
352                 free(BLO_decrypt);
353                 return err;
354         }
355
356         if (BLO_decrypt->streamDone == BLO_decrypt->streamHeader->length +
357                 EN_DE_CRYPTHEADERSTRUCTSIZE) {
358 #ifndef NDEBUG
359                 fprintf(GEN_errorstream, "Crypted data length is correct\n");
360 #endif
361         } else {
362 #ifndef NDEBUG
363                 fprintf(GEN_errorstream, "Crypted data length is NOT correct\n");
364 #endif
365                 err = BRS_SETFUNCTION(BRS_DECRYPT) |
366                           BRS_SETGENERR(BRS_DATALEN);
367                 if (BLO_decrypt->streamGlue) free(BLO_decrypt->streamGlue);
368                 if (BLO_decrypt->streamHeader) free(BLO_decrypt->streamHeader);
369                 if (BLO_decrypt->deCryptKey) free(BLO_decrypt->deCryptKey);
370                 free(BLO_decrypt);
371                 return err;
372         }
373
374         if (BLO_decrypt->datacrc == BLO_decrypt->streamHeader->datacrc) {
375 #ifndef NDEBUG
376                 fprintf(GEN_errorstream, "Crypted data CRC is correct\n");
377 #endif
378         } else {
379 #ifndef NDEBUG
380                 fprintf(GEN_errorstream, "Crypted data CRC is NOT correct\n");
381 #endif
382                 err = BRS_SETFUNCTION(BRS_DECRYPT) |
383                           BRS_SETGENERR(BRS_CRCDATA);
384                 if (BLO_decrypt->streamGlue) free(BLO_decrypt->streamGlue);
385                 if (BLO_decrypt->streamHeader) free(BLO_decrypt->streamHeader);
386                 if (BLO_decrypt->deCryptKey) free(BLO_decrypt->deCryptKey);
387                 free(BLO_decrypt);
388                 return err;
389         }
390
391         free(BLO_decrypt->streamGlue);
392         free(BLO_decrypt->streamHeader);
393         free(BLO_decrypt->deCryptKey);
394         free(BLO_decrypt);
395
396         return err;
397 }
398