]>
Commit | Line | Data |
---|---|---|
ef416fc2 | 1 | //======================================================================== |
2 | // | |
3 | // Decrypt.cc | |
4 | // | |
5 | // Copyright 1996-2003 Glyph & Cog, LLC | |
6 | // | |
7 | //======================================================================== | |
8 | ||
9 | #include <config.h> | |
10 | ||
11 | #ifdef USE_GCC_PRAGMAS | |
12 | #pragma implementation | |
13 | #endif | |
14 | ||
15 | #include <string.h> | |
16 | #include "gmem.h" | |
17 | #include "Decrypt.h" | |
18 | ||
19 | static void rc4InitKey(Guchar *key, int keyLen, Guchar *state); | |
20 | static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c); | |
21 | static void md5(Guchar *msg, int msgLen, Guchar *digest); | |
22 | ||
23 | static Guchar passwordPad[32] = { | |
24 | 0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41, | |
25 | 0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08, | |
26 | 0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80, | |
27 | 0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a | |
28 | }; | |
29 | ||
30 | //------------------------------------------------------------------------ | |
31 | // Decrypt | |
32 | //------------------------------------------------------------------------ | |
33 | ||
34 | Decrypt::Decrypt(Guchar *fileKey, int keyLength, int objNum, int objGen) { | |
35 | int i; | |
36 | ||
37 | // construct object key | |
38 | for (i = 0; i < keyLength; ++i) { | |
39 | objKey[i] = fileKey[i]; | |
40 | } | |
41 | objKey[keyLength] = objNum & 0xff; | |
42 | objKey[keyLength + 1] = (objNum >> 8) & 0xff; | |
43 | objKey[keyLength + 2] = (objNum >> 16) & 0xff; | |
44 | objKey[keyLength + 3] = objGen & 0xff; | |
45 | objKey[keyLength + 4] = (objGen >> 8) & 0xff; | |
46 | md5(objKey, keyLength + 5, objKey); | |
47 | ||
48 | // set up for decryption | |
49 | x = y = 0; | |
50 | if ((objKeyLength = keyLength + 5) > 16) { | |
51 | objKeyLength = 16; | |
52 | } | |
53 | rc4InitKey(objKey, objKeyLength, state); | |
54 | } | |
55 | ||
56 | void Decrypt::reset() { | |
57 | x = y = 0; | |
58 | rc4InitKey(objKey, objKeyLength, state); | |
59 | } | |
60 | ||
61 | Guchar Decrypt::decryptByte(Guchar c) { | |
62 | return rc4DecryptByte(state, &x, &y, c); | |
63 | } | |
64 | ||
65 | GBool Decrypt::makeFileKey(int encVersion, int encRevision, int keyLength, | |
66 | GString *ownerKey, GString *userKey, | |
67 | int permissions, GString *fileID, | |
68 | GString *ownerPassword, GString *userPassword, | |
69 | Guchar *fileKey, GBool encryptMetadata, | |
70 | GBool *ownerPasswordOk) { | |
71 | Guchar test[32], test2[32]; | |
72 | GString *userPassword2; | |
73 | Guchar fState[256]; | |
74 | Guchar tmpKey[16]; | |
75 | Guchar fx, fy; | |
76 | int len, i, j; | |
77 | ||
78 | // try using the supplied owner password to generate the user password | |
79 | *ownerPasswordOk = gFalse; | |
80 | if (ownerPassword) { | |
81 | len = ownerPassword->getLength(); | |
82 | if (len < 32) { | |
83 | memcpy(test, ownerPassword->getCString(), len); | |
84 | memcpy(test + len, passwordPad, 32 - len); | |
85 | } else { | |
86 | memcpy(test, ownerPassword->getCString(), 32); | |
87 | } | |
88 | md5(test, 32, test); | |
89 | if (encRevision == 3) { | |
90 | for (i = 0; i < 50; ++i) { | |
91 | md5(test, 16, test); | |
92 | } | |
93 | } | |
94 | if (encRevision == 2) { | |
95 | rc4InitKey(test, keyLength, fState); | |
96 | fx = fy = 0; | |
97 | for (i = 0; i < 32; ++i) { | |
98 | test2[i] = rc4DecryptByte(fState, &fx, &fy, ownerKey->getChar(i)); | |
99 | } | |
100 | } else { | |
101 | memcpy(test2, ownerKey->getCString(), 32); | |
102 | for (i = 19; i >= 0; --i) { | |
103 | for (j = 0; j < keyLength; ++j) { | |
104 | tmpKey[j] = test[j] ^ i; | |
105 | } | |
106 | rc4InitKey(tmpKey, keyLength, fState); | |
107 | fx = fy = 0; | |
108 | for (j = 0; j < 32; ++j) { | |
109 | test2[j] = rc4DecryptByte(fState, &fx, &fy, test2[j]); | |
110 | } | |
111 | } | |
112 | } | |
113 | userPassword2 = new GString((char *)test2, 32); | |
114 | if (makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey, | |
115 | permissions, fileID, userPassword2, fileKey, | |
116 | encryptMetadata)) { | |
117 | *ownerPasswordOk = gTrue; | |
118 | delete userPassword2; | |
119 | return gTrue; | |
120 | } | |
121 | delete userPassword2; | |
122 | } | |
123 | ||
124 | // try using the supplied user password | |
125 | return makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey, | |
126 | permissions, fileID, userPassword, fileKey, | |
127 | encryptMetadata); | |
128 | } | |
129 | ||
130 | GBool Decrypt::makeFileKey2(int encVersion, int encRevision, int keyLength, | |
131 | GString *ownerKey, GString *userKey, | |
132 | int permissions, GString *fileID, | |
133 | GString *userPassword, Guchar *fileKey, | |
134 | GBool encryptMetadata) { | |
135 | Guchar *buf; | |
136 | Guchar test[32]; | |
137 | Guchar fState[256]; | |
138 | Guchar tmpKey[16]; | |
139 | Guchar fx, fy; | |
140 | int len, i, j; | |
141 | GBool ok; | |
142 | ||
a74454a7 | 143 | |
144 | // check whether we have non-zero keyLength | |
145 | if ( keyLength < 5 || keyLength > 16 ) { | |
146 | return gFalse; | |
147 | } | |
148 | ||
ef416fc2 | 149 | // generate file key |
150 | buf = (Guchar *)gmalloc(72 + fileID->getLength()); | |
151 | if (userPassword) { | |
152 | len = userPassword->getLength(); | |
153 | if (len < 32) { | |
154 | memcpy(buf, userPassword->getCString(), len); | |
155 | memcpy(buf + len, passwordPad, 32 - len); | |
156 | } else { | |
157 | memcpy(buf, userPassword->getCString(), 32); | |
158 | } | |
159 | } else { | |
160 | memcpy(buf, passwordPad, 32); | |
161 | } | |
162 | memcpy(buf + 32, ownerKey->getCString(), 32); | |
163 | buf[64] = permissions & 0xff; | |
164 | buf[65] = (permissions >> 8) & 0xff; | |
165 | buf[66] = (permissions >> 16) & 0xff; | |
166 | buf[67] = (permissions >> 24) & 0xff; | |
167 | memcpy(buf + 68, fileID->getCString(), fileID->getLength()); | |
168 | len = 68 + fileID->getLength(); | |
169 | if (!encryptMetadata) { | |
170 | buf[len++] = 0xff; | |
171 | buf[len++] = 0xff; | |
172 | buf[len++] = 0xff; | |
173 | buf[len++] = 0xff; | |
174 | } | |
175 | md5(buf, len, fileKey); | |
176 | if (encRevision == 3) { | |
177 | for (i = 0; i < 50; ++i) { | |
178 | md5(fileKey, keyLength, fileKey); | |
179 | } | |
180 | } | |
181 | ||
182 | // test user password | |
183 | if (encRevision == 2) { | |
184 | rc4InitKey(fileKey, keyLength, fState); | |
185 | fx = fy = 0; | |
186 | for (i = 0; i < 32; ++i) { | |
187 | test[i] = rc4DecryptByte(fState, &fx, &fy, userKey->getChar(i)); | |
188 | } | |
189 | ok = memcmp(test, passwordPad, 32) == 0; | |
190 | } else if (encRevision == 3) { | |
191 | memcpy(test, userKey->getCString(), 32); | |
192 | for (i = 19; i >= 0; --i) { | |
193 | for (j = 0; j < keyLength; ++j) { | |
194 | tmpKey[j] = fileKey[j] ^ i; | |
195 | } | |
196 | rc4InitKey(tmpKey, keyLength, fState); | |
197 | fx = fy = 0; | |
198 | for (j = 0; j < 32; ++j) { | |
199 | test[j] = rc4DecryptByte(fState, &fx, &fy, test[j]); | |
200 | } | |
201 | } | |
202 | memcpy(buf, passwordPad, 32); | |
203 | memcpy(buf + 32, fileID->getCString(), fileID->getLength()); | |
204 | md5(buf, 32 + fileID->getLength(), buf); | |
205 | ok = memcmp(test, buf, 16) == 0; | |
206 | } else { | |
207 | ok = gFalse; | |
208 | } | |
209 | ||
210 | gfree(buf); | |
211 | return ok; | |
212 | } | |
213 | ||
214 | //------------------------------------------------------------------------ | |
215 | // RC4-compatible decryption | |
216 | //------------------------------------------------------------------------ | |
217 | ||
218 | static void rc4InitKey(Guchar *key, int keyLen, Guchar *state) { | |
219 | Guchar index1, index2; | |
220 | Guchar t; | |
221 | int i; | |
222 | ||
223 | for (i = 0; i < 256; ++i) | |
224 | state[i] = i; | |
225 | index1 = index2 = 0; | |
226 | for (i = 0; i < 256; ++i) { | |
227 | index2 = (key[index1] + state[i] + index2) % 256; | |
228 | t = state[i]; | |
229 | state[i] = state[index2]; | |
230 | state[index2] = t; | |
231 | index1 = (index1 + 1) % keyLen; | |
232 | } | |
233 | } | |
234 | ||
235 | static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c) { | |
236 | Guchar x1, y1, tx, ty; | |
237 | ||
238 | x1 = *x = (*x + 1) % 256; | |
239 | y1 = *y = (state[*x] + *y) % 256; | |
240 | tx = state[x1]; | |
241 | ty = state[y1]; | |
242 | state[x1] = ty; | |
243 | state[y1] = tx; | |
244 | return c ^ state[(tx + ty) % 256]; | |
245 | } | |
246 | ||
247 | //------------------------------------------------------------------------ | |
248 | // MD5 message digest | |
249 | //------------------------------------------------------------------------ | |
250 | ||
251 | // this works around a bug in older Sun compilers | |
252 | static inline Gulong rotateLeft(Gulong x, int r) { | |
253 | x &= 0xffffffff; | |
254 | return ((x << r) | (x >> (32 - r))) & 0xffffffff; | |
255 | } | |
256 | ||
257 | static inline Gulong md5Round1(Gulong a, Gulong b, Gulong c, Gulong d, | |
258 | Gulong Xk, Gulong s, Gulong Ti) { | |
259 | return b + rotateLeft((a + ((b & c) | (~b & d)) + Xk + Ti), s); | |
260 | } | |
261 | ||
262 | static inline Gulong md5Round2(Gulong a, Gulong b, Gulong c, Gulong d, | |
263 | Gulong Xk, Gulong s, Gulong Ti) { | |
264 | return b + rotateLeft((a + ((b & d) | (c & ~d)) + Xk + Ti), s); | |
265 | } | |
266 | ||
267 | static inline Gulong md5Round3(Gulong a, Gulong b, Gulong c, Gulong d, | |
268 | Gulong Xk, Gulong s, Gulong Ti) { | |
269 | return b + rotateLeft((a + (b ^ c ^ d) + Xk + Ti), s); | |
270 | } | |
271 | ||
272 | static inline Gulong md5Round4(Gulong a, Gulong b, Gulong c, Gulong d, | |
273 | Gulong Xk, Gulong s, Gulong Ti) { | |
274 | return b + rotateLeft((a + (c ^ (b | ~d)) + Xk + Ti), s); | |
275 | } | |
276 | ||
277 | static void md5(Guchar *msg, int msgLen, Guchar *digest) { | |
278 | Gulong x[16]; | |
279 | Gulong a, b, c, d, aa, bb, cc, dd; | |
280 | int n64; | |
281 | int i, j, k; | |
282 | ||
283 | // compute number of 64-byte blocks | |
284 | // (length + pad byte (0x80) + 8 bytes for length) | |
285 | n64 = (msgLen + 1 + 8 + 63) / 64; | |
286 | ||
287 | // initialize a, b, c, d | |
288 | a = 0x67452301; | |
289 | b = 0xefcdab89; | |
290 | c = 0x98badcfe; | |
291 | d = 0x10325476; | |
292 | ||
293 | // loop through blocks | |
294 | k = 0; | |
295 | for (i = 0; i < n64; ++i) { | |
296 | ||
297 | // grab a 64-byte block | |
298 | for (j = 0; j < 16 && k < msgLen - 3; ++j, k += 4) | |
299 | x[j] = (((((msg[k+3] << 8) + msg[k+2]) << 8) + msg[k+1]) << 8) + msg[k]; | |
300 | if (i == n64 - 1) { | |
301 | if (k == msgLen - 3) | |
302 | x[j] = 0x80000000 + (((msg[k+2] << 8) + msg[k+1]) << 8) + msg[k]; | |
303 | else if (k == msgLen - 2) | |
304 | x[j] = 0x800000 + (msg[k+1] << 8) + msg[k]; | |
305 | else if (k == msgLen - 1) | |
306 | x[j] = 0x8000 + msg[k]; | |
307 | else | |
308 | x[j] = 0x80; | |
309 | ++j; | |
310 | while (j < 16) | |
311 | x[j++] = 0; | |
312 | x[14] = msgLen << 3; | |
313 | } | |
314 | ||
315 | // save a, b, c, d | |
316 | aa = a; | |
317 | bb = b; | |
318 | cc = c; | |
319 | dd = d; | |
320 | ||
321 | // round 1 | |
322 | a = md5Round1(a, b, c, d, x[0], 7, 0xd76aa478); | |
323 | d = md5Round1(d, a, b, c, x[1], 12, 0xe8c7b756); | |
324 | c = md5Round1(c, d, a, b, x[2], 17, 0x242070db); | |
325 | b = md5Round1(b, c, d, a, x[3], 22, 0xc1bdceee); | |
326 | a = md5Round1(a, b, c, d, x[4], 7, 0xf57c0faf); | |
327 | d = md5Round1(d, a, b, c, x[5], 12, 0x4787c62a); | |
328 | c = md5Round1(c, d, a, b, x[6], 17, 0xa8304613); | |
329 | b = md5Round1(b, c, d, a, x[7], 22, 0xfd469501); | |
330 | a = md5Round1(a, b, c, d, x[8], 7, 0x698098d8); | |
331 | d = md5Round1(d, a, b, c, x[9], 12, 0x8b44f7af); | |
332 | c = md5Round1(c, d, a, b, x[10], 17, 0xffff5bb1); | |
333 | b = md5Round1(b, c, d, a, x[11], 22, 0x895cd7be); | |
334 | a = md5Round1(a, b, c, d, x[12], 7, 0x6b901122); | |
335 | d = md5Round1(d, a, b, c, x[13], 12, 0xfd987193); | |
336 | c = md5Round1(c, d, a, b, x[14], 17, 0xa679438e); | |
337 | b = md5Round1(b, c, d, a, x[15], 22, 0x49b40821); | |
338 | ||
339 | // round 2 | |
340 | a = md5Round2(a, b, c, d, x[1], 5, 0xf61e2562); | |
341 | d = md5Round2(d, a, b, c, x[6], 9, 0xc040b340); | |
342 | c = md5Round2(c, d, a, b, x[11], 14, 0x265e5a51); | |
343 | b = md5Round2(b, c, d, a, x[0], 20, 0xe9b6c7aa); | |
344 | a = md5Round2(a, b, c, d, x[5], 5, 0xd62f105d); | |
345 | d = md5Round2(d, a, b, c, x[10], 9, 0x02441453); | |
346 | c = md5Round2(c, d, a, b, x[15], 14, 0xd8a1e681); | |
347 | b = md5Round2(b, c, d, a, x[4], 20, 0xe7d3fbc8); | |
348 | a = md5Round2(a, b, c, d, x[9], 5, 0x21e1cde6); | |
349 | d = md5Round2(d, a, b, c, x[14], 9, 0xc33707d6); | |
350 | c = md5Round2(c, d, a, b, x[3], 14, 0xf4d50d87); | |
351 | b = md5Round2(b, c, d, a, x[8], 20, 0x455a14ed); | |
352 | a = md5Round2(a, b, c, d, x[13], 5, 0xa9e3e905); | |
353 | d = md5Round2(d, a, b, c, x[2], 9, 0xfcefa3f8); | |
354 | c = md5Round2(c, d, a, b, x[7], 14, 0x676f02d9); | |
355 | b = md5Round2(b, c, d, a, x[12], 20, 0x8d2a4c8a); | |
356 | ||
357 | // round 3 | |
358 | a = md5Round3(a, b, c, d, x[5], 4, 0xfffa3942); | |
359 | d = md5Round3(d, a, b, c, x[8], 11, 0x8771f681); | |
360 | c = md5Round3(c, d, a, b, x[11], 16, 0x6d9d6122); | |
361 | b = md5Round3(b, c, d, a, x[14], 23, 0xfde5380c); | |
362 | a = md5Round3(a, b, c, d, x[1], 4, 0xa4beea44); | |
363 | d = md5Round3(d, a, b, c, x[4], 11, 0x4bdecfa9); | |
364 | c = md5Round3(c, d, a, b, x[7], 16, 0xf6bb4b60); | |
365 | b = md5Round3(b, c, d, a, x[10], 23, 0xbebfbc70); | |
366 | a = md5Round3(a, b, c, d, x[13], 4, 0x289b7ec6); | |
367 | d = md5Round3(d, a, b, c, x[0], 11, 0xeaa127fa); | |
368 | c = md5Round3(c, d, a, b, x[3], 16, 0xd4ef3085); | |
369 | b = md5Round3(b, c, d, a, x[6], 23, 0x04881d05); | |
370 | a = md5Round3(a, b, c, d, x[9], 4, 0xd9d4d039); | |
371 | d = md5Round3(d, a, b, c, x[12], 11, 0xe6db99e5); | |
372 | c = md5Round3(c, d, a, b, x[15], 16, 0x1fa27cf8); | |
373 | b = md5Round3(b, c, d, a, x[2], 23, 0xc4ac5665); | |
374 | ||
375 | // round 4 | |
376 | a = md5Round4(a, b, c, d, x[0], 6, 0xf4292244); | |
377 | d = md5Round4(d, a, b, c, x[7], 10, 0x432aff97); | |
378 | c = md5Round4(c, d, a, b, x[14], 15, 0xab9423a7); | |
379 | b = md5Round4(b, c, d, a, x[5], 21, 0xfc93a039); | |
380 | a = md5Round4(a, b, c, d, x[12], 6, 0x655b59c3); | |
381 | d = md5Round4(d, a, b, c, x[3], 10, 0x8f0ccc92); | |
382 | c = md5Round4(c, d, a, b, x[10], 15, 0xffeff47d); | |
383 | b = md5Round4(b, c, d, a, x[1], 21, 0x85845dd1); | |
384 | a = md5Round4(a, b, c, d, x[8], 6, 0x6fa87e4f); | |
385 | d = md5Round4(d, a, b, c, x[15], 10, 0xfe2ce6e0); | |
386 | c = md5Round4(c, d, a, b, x[6], 15, 0xa3014314); | |
387 | b = md5Round4(b, c, d, a, x[13], 21, 0x4e0811a1); | |
388 | a = md5Round4(a, b, c, d, x[4], 6, 0xf7537e82); | |
389 | d = md5Round4(d, a, b, c, x[11], 10, 0xbd3af235); | |
390 | c = md5Round4(c, d, a, b, x[2], 15, 0x2ad7d2bb); | |
391 | b = md5Round4(b, c, d, a, x[9], 21, 0xeb86d391); | |
392 | ||
393 | // increment a, b, c, d | |
394 | a += aa; | |
395 | b += bb; | |
396 | c += cc; | |
397 | d += dd; | |
398 | } | |
399 | ||
400 | // break digest into bytes | |
401 | digest[0] = (Guchar)(a & 0xff); | |
402 | digest[1] = (Guchar)((a >>= 8) & 0xff); | |
403 | digest[2] = (Guchar)((a >>= 8) & 0xff); | |
404 | digest[3] = (Guchar)((a >>= 8) & 0xff); | |
405 | digest[4] = (Guchar)(b & 0xff); | |
406 | digest[5] = (Guchar)((b >>= 8) & 0xff); | |
407 | digest[6] = (Guchar)((b >>= 8) & 0xff); | |
408 | digest[7] = (Guchar)((b >>= 8) & 0xff); | |
409 | digest[8] = (Guchar)(c & 0xff); | |
410 | digest[9] = (Guchar)((c >>= 8) & 0xff); | |
411 | digest[10] = (Guchar)((c >>= 8) & 0xff); | |
412 | digest[11] = (Guchar)((c >>= 8) & 0xff); | |
413 | digest[12] = (Guchar)(d & 0xff); | |
414 | digest[13] = (Guchar)((d >>= 8) & 0xff); | |
415 | digest[14] = (Guchar)((d >>= 8) & 0xff); | |
416 | digest[15] = (Guchar)((d >>= 8) & 0xff); | |
417 | } |