]>
Commit | Line | Data |
---|---|---|
ef416fc2 | 1 | //======================================================================== |
2 | // | |
3 | // Decrypt.cc | |
4 | // | |
5 | // Copyright 1996-2003 Glyph & Cog, LLC | |
6 | // | |
7 | //======================================================================== | |
8 | ||
9 | #include <config.h> | |
10 | ||
11 | #ifdef USE_GCC_PRAGMAS | |
12 | #pragma implementation | |
13 | #endif | |
14 | ||
15 | #include <string.h> | |
16 | #include "gmem.h" | |
17 | #include "Decrypt.h" | |
18 | ||
19 | static void rc4InitKey(Guchar *key, int keyLen, Guchar *state); | |
20 | static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c); | |
21 | static void md5(Guchar *msg, int msgLen, Guchar *digest); | |
22 | ||
23 | static Guchar passwordPad[32] = { | |
24 | 0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41, | |
25 | 0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08, | |
26 | 0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80, | |
27 | 0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a | |
28 | }; | |
29 | ||
30 | //------------------------------------------------------------------------ | |
31 | // Decrypt | |
32 | //------------------------------------------------------------------------ | |
33 | ||
34 | Decrypt::Decrypt(Guchar *fileKey, int keyLength, int objNum, int objGen) { | |
35 | int i; | |
36 | ||
37 | // construct object key | |
38 | for (i = 0; i < keyLength; ++i) { | |
39 | objKey[i] = fileKey[i]; | |
40 | } | |
41 | objKey[keyLength] = objNum & 0xff; | |
42 | objKey[keyLength + 1] = (objNum >> 8) & 0xff; | |
43 | objKey[keyLength + 2] = (objNum >> 16) & 0xff; | |
44 | objKey[keyLength + 3] = objGen & 0xff; | |
45 | objKey[keyLength + 4] = (objGen >> 8) & 0xff; | |
46 | md5(objKey, keyLength + 5, objKey); | |
47 | ||
48 | // set up for decryption | |
49 | x = y = 0; | |
50 | if ((objKeyLength = keyLength + 5) > 16) { | |
51 | objKeyLength = 16; | |
52 | } | |
53 | rc4InitKey(objKey, objKeyLength, state); | |
54 | } | |
55 | ||
56 | void Decrypt::reset() { | |
57 | x = y = 0; | |
58 | rc4InitKey(objKey, objKeyLength, state); | |
59 | } | |
60 | ||
61 | Guchar Decrypt::decryptByte(Guchar c) { | |
62 | return rc4DecryptByte(state, &x, &y, c); | |
63 | } | |
64 | ||
65 | GBool Decrypt::makeFileKey(int encVersion, int encRevision, int keyLength, | |
66 | GString *ownerKey, GString *userKey, | |
67 | int permissions, GString *fileID, | |
68 | GString *ownerPassword, GString *userPassword, | |
69 | Guchar *fileKey, GBool encryptMetadata, | |
70 | GBool *ownerPasswordOk) { | |
71 | Guchar test[32], test2[32]; | |
72 | GString *userPassword2; | |
73 | Guchar fState[256]; | |
74 | Guchar tmpKey[16]; | |
75 | Guchar fx, fy; | |
76 | int len, i, j; | |
77 | ||
78 | // try using the supplied owner password to generate the user password | |
79 | *ownerPasswordOk = gFalse; | |
80 | if (ownerPassword) { | |
81 | len = ownerPassword->getLength(); | |
82 | if (len < 32) { | |
83 | memcpy(test, ownerPassword->getCString(), len); | |
84 | memcpy(test + len, passwordPad, 32 - len); | |
85 | } else { | |
86 | memcpy(test, ownerPassword->getCString(), 32); | |
87 | } | |
88 | md5(test, 32, test); | |
89 | if (encRevision == 3) { | |
90 | for (i = 0; i < 50; ++i) { | |
91 | md5(test, 16, test); | |
92 | } | |
93 | } | |
94 | if (encRevision == 2) { | |
95 | rc4InitKey(test, keyLength, fState); | |
96 | fx = fy = 0; | |
97 | for (i = 0; i < 32; ++i) { | |
98 | test2[i] = rc4DecryptByte(fState, &fx, &fy, ownerKey->getChar(i)); | |
99 | } | |
100 | } else { | |
101 | memcpy(test2, ownerKey->getCString(), 32); | |
102 | for (i = 19; i >= 0; --i) { | |
103 | for (j = 0; j < keyLength; ++j) { | |
104 | tmpKey[j] = test[j] ^ i; | |
105 | } | |
106 | rc4InitKey(tmpKey, keyLength, fState); | |
107 | fx = fy = 0; | |
108 | for (j = 0; j < 32; ++j) { | |
109 | test2[j] = rc4DecryptByte(fState, &fx, &fy, test2[j]); | |
110 | } | |
111 | } | |
112 | } | |
113 | userPassword2 = new GString((char *)test2, 32); | |
114 | if (makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey, | |
115 | permissions, fileID, userPassword2, fileKey, | |
116 | encryptMetadata)) { | |
117 | *ownerPasswordOk = gTrue; | |
118 | delete userPassword2; | |
119 | return gTrue; | |
120 | } | |
121 | delete userPassword2; | |
122 | } | |
123 | ||
124 | // try using the supplied user password | |
125 | return makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey, | |
126 | permissions, fileID, userPassword, fileKey, | |
127 | encryptMetadata); | |
128 | } | |
129 | ||
130 | GBool Decrypt::makeFileKey2(int encVersion, int encRevision, int keyLength, | |
131 | GString *ownerKey, GString *userKey, | |
132 | int permissions, GString *fileID, | |
133 | GString *userPassword, Guchar *fileKey, | |
134 | GBool encryptMetadata) { | |
135 | Guchar *buf; | |
136 | Guchar test[32]; | |
137 | Guchar fState[256]; | |
138 | Guchar tmpKey[16]; | |
139 | Guchar fx, fy; | |
140 | int len, i, j; | |
141 | GBool ok; | |
142 | ||
143 | // generate file key | |
144 | buf = (Guchar *)gmalloc(72 + fileID->getLength()); | |
145 | if (userPassword) { | |
146 | len = userPassword->getLength(); | |
147 | if (len < 32) { | |
148 | memcpy(buf, userPassword->getCString(), len); | |
149 | memcpy(buf + len, passwordPad, 32 - len); | |
150 | } else { | |
151 | memcpy(buf, userPassword->getCString(), 32); | |
152 | } | |
153 | } else { | |
154 | memcpy(buf, passwordPad, 32); | |
155 | } | |
156 | memcpy(buf + 32, ownerKey->getCString(), 32); | |
157 | buf[64] = permissions & 0xff; | |
158 | buf[65] = (permissions >> 8) & 0xff; | |
159 | buf[66] = (permissions >> 16) & 0xff; | |
160 | buf[67] = (permissions >> 24) & 0xff; | |
161 | memcpy(buf + 68, fileID->getCString(), fileID->getLength()); | |
162 | len = 68 + fileID->getLength(); | |
163 | if (!encryptMetadata) { | |
164 | buf[len++] = 0xff; | |
165 | buf[len++] = 0xff; | |
166 | buf[len++] = 0xff; | |
167 | buf[len++] = 0xff; | |
168 | } | |
169 | md5(buf, len, fileKey); | |
170 | if (encRevision == 3) { | |
171 | for (i = 0; i < 50; ++i) { | |
172 | md5(fileKey, keyLength, fileKey); | |
173 | } | |
174 | } | |
175 | ||
176 | // test user password | |
177 | if (encRevision == 2) { | |
178 | rc4InitKey(fileKey, keyLength, fState); | |
179 | fx = fy = 0; | |
180 | for (i = 0; i < 32; ++i) { | |
181 | test[i] = rc4DecryptByte(fState, &fx, &fy, userKey->getChar(i)); | |
182 | } | |
183 | ok = memcmp(test, passwordPad, 32) == 0; | |
184 | } else if (encRevision == 3) { | |
185 | memcpy(test, userKey->getCString(), 32); | |
186 | for (i = 19; i >= 0; --i) { | |
187 | for (j = 0; j < keyLength; ++j) { | |
188 | tmpKey[j] = fileKey[j] ^ i; | |
189 | } | |
190 | rc4InitKey(tmpKey, keyLength, fState); | |
191 | fx = fy = 0; | |
192 | for (j = 0; j < 32; ++j) { | |
193 | test[j] = rc4DecryptByte(fState, &fx, &fy, test[j]); | |
194 | } | |
195 | } | |
196 | memcpy(buf, passwordPad, 32); | |
197 | memcpy(buf + 32, fileID->getCString(), fileID->getLength()); | |
198 | md5(buf, 32 + fileID->getLength(), buf); | |
199 | ok = memcmp(test, buf, 16) == 0; | |
200 | } else { | |
201 | ok = gFalse; | |
202 | } | |
203 | ||
204 | gfree(buf); | |
205 | return ok; | |
206 | } | |
207 | ||
208 | //------------------------------------------------------------------------ | |
209 | // RC4-compatible decryption | |
210 | //------------------------------------------------------------------------ | |
211 | ||
212 | static void rc4InitKey(Guchar *key, int keyLen, Guchar *state) { | |
213 | Guchar index1, index2; | |
214 | Guchar t; | |
215 | int i; | |
216 | ||
217 | for (i = 0; i < 256; ++i) | |
218 | state[i] = i; | |
219 | index1 = index2 = 0; | |
220 | for (i = 0; i < 256; ++i) { | |
221 | index2 = (key[index1] + state[i] + index2) % 256; | |
222 | t = state[i]; | |
223 | state[i] = state[index2]; | |
224 | state[index2] = t; | |
225 | index1 = (index1 + 1) % keyLen; | |
226 | } | |
227 | } | |
228 | ||
229 | static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c) { | |
230 | Guchar x1, y1, tx, ty; | |
231 | ||
232 | x1 = *x = (*x + 1) % 256; | |
233 | y1 = *y = (state[*x] + *y) % 256; | |
234 | tx = state[x1]; | |
235 | ty = state[y1]; | |
236 | state[x1] = ty; | |
237 | state[y1] = tx; | |
238 | return c ^ state[(tx + ty) % 256]; | |
239 | } | |
240 | ||
241 | //------------------------------------------------------------------------ | |
242 | // MD5 message digest | |
243 | //------------------------------------------------------------------------ | |
244 | ||
245 | // this works around a bug in older Sun compilers | |
246 | static inline Gulong rotateLeft(Gulong x, int r) { | |
247 | x &= 0xffffffff; | |
248 | return ((x << r) | (x >> (32 - r))) & 0xffffffff; | |
249 | } | |
250 | ||
251 | static inline Gulong md5Round1(Gulong a, Gulong b, Gulong c, Gulong d, | |
252 | Gulong Xk, Gulong s, Gulong Ti) { | |
253 | return b + rotateLeft((a + ((b & c) | (~b & d)) + Xk + Ti), s); | |
254 | } | |
255 | ||
256 | static inline Gulong md5Round2(Gulong a, Gulong b, Gulong c, Gulong d, | |
257 | Gulong Xk, Gulong s, Gulong Ti) { | |
258 | return b + rotateLeft((a + ((b & d) | (c & ~d)) + Xk + Ti), s); | |
259 | } | |
260 | ||
261 | static inline Gulong md5Round3(Gulong a, Gulong b, Gulong c, Gulong d, | |
262 | Gulong Xk, Gulong s, Gulong Ti) { | |
263 | return b + rotateLeft((a + (b ^ c ^ d) + Xk + Ti), s); | |
264 | } | |
265 | ||
266 | static inline Gulong md5Round4(Gulong a, Gulong b, Gulong c, Gulong d, | |
267 | Gulong Xk, Gulong s, Gulong Ti) { | |
268 | return b + rotateLeft((a + (c ^ (b | ~d)) + Xk + Ti), s); | |
269 | } | |
270 | ||
271 | static void md5(Guchar *msg, int msgLen, Guchar *digest) { | |
272 | Gulong x[16]; | |
273 | Gulong a, b, c, d, aa, bb, cc, dd; | |
274 | int n64; | |
275 | int i, j, k; | |
276 | ||
277 | // compute number of 64-byte blocks | |
278 | // (length + pad byte (0x80) + 8 bytes for length) | |
279 | n64 = (msgLen + 1 + 8 + 63) / 64; | |
280 | ||
281 | // initialize a, b, c, d | |
282 | a = 0x67452301; | |
283 | b = 0xefcdab89; | |
284 | c = 0x98badcfe; | |
285 | d = 0x10325476; | |
286 | ||
287 | // loop through blocks | |
288 | k = 0; | |
289 | for (i = 0; i < n64; ++i) { | |
290 | ||
291 | // grab a 64-byte block | |
292 | for (j = 0; j < 16 && k < msgLen - 3; ++j, k += 4) | |
293 | x[j] = (((((msg[k+3] << 8) + msg[k+2]) << 8) + msg[k+1]) << 8) + msg[k]; | |
294 | if (i == n64 - 1) { | |
295 | if (k == msgLen - 3) | |
296 | x[j] = 0x80000000 + (((msg[k+2] << 8) + msg[k+1]) << 8) + msg[k]; | |
297 | else if (k == msgLen - 2) | |
298 | x[j] = 0x800000 + (msg[k+1] << 8) + msg[k]; | |
299 | else if (k == msgLen - 1) | |
300 | x[j] = 0x8000 + msg[k]; | |
301 | else | |
302 | x[j] = 0x80; | |
303 | ++j; | |
304 | while (j < 16) | |
305 | x[j++] = 0; | |
306 | x[14] = msgLen << 3; | |
307 | } | |
308 | ||
309 | // save a, b, c, d | |
310 | aa = a; | |
311 | bb = b; | |
312 | cc = c; | |
313 | dd = d; | |
314 | ||
315 | // round 1 | |
316 | a = md5Round1(a, b, c, d, x[0], 7, 0xd76aa478); | |
317 | d = md5Round1(d, a, b, c, x[1], 12, 0xe8c7b756); | |
318 | c = md5Round1(c, d, a, b, x[2], 17, 0x242070db); | |
319 | b = md5Round1(b, c, d, a, x[3], 22, 0xc1bdceee); | |
320 | a = md5Round1(a, b, c, d, x[4], 7, 0xf57c0faf); | |
321 | d = md5Round1(d, a, b, c, x[5], 12, 0x4787c62a); | |
322 | c = md5Round1(c, d, a, b, x[6], 17, 0xa8304613); | |
323 | b = md5Round1(b, c, d, a, x[7], 22, 0xfd469501); | |
324 | a = md5Round1(a, b, c, d, x[8], 7, 0x698098d8); | |
325 | d = md5Round1(d, a, b, c, x[9], 12, 0x8b44f7af); | |
326 | c = md5Round1(c, d, a, b, x[10], 17, 0xffff5bb1); | |
327 | b = md5Round1(b, c, d, a, x[11], 22, 0x895cd7be); | |
328 | a = md5Round1(a, b, c, d, x[12], 7, 0x6b901122); | |
329 | d = md5Round1(d, a, b, c, x[13], 12, 0xfd987193); | |
330 | c = md5Round1(c, d, a, b, x[14], 17, 0xa679438e); | |
331 | b = md5Round1(b, c, d, a, x[15], 22, 0x49b40821); | |
332 | ||
333 | // round 2 | |
334 | a = md5Round2(a, b, c, d, x[1], 5, 0xf61e2562); | |
335 | d = md5Round2(d, a, b, c, x[6], 9, 0xc040b340); | |
336 | c = md5Round2(c, d, a, b, x[11], 14, 0x265e5a51); | |
337 | b = md5Round2(b, c, d, a, x[0], 20, 0xe9b6c7aa); | |
338 | a = md5Round2(a, b, c, d, x[5], 5, 0xd62f105d); | |
339 | d = md5Round2(d, a, b, c, x[10], 9, 0x02441453); | |
340 | c = md5Round2(c, d, a, b, x[15], 14, 0xd8a1e681); | |
341 | b = md5Round2(b, c, d, a, x[4], 20, 0xe7d3fbc8); | |
342 | a = md5Round2(a, b, c, d, x[9], 5, 0x21e1cde6); | |
343 | d = md5Round2(d, a, b, c, x[14], 9, 0xc33707d6); | |
344 | c = md5Round2(c, d, a, b, x[3], 14, 0xf4d50d87); | |
345 | b = md5Round2(b, c, d, a, x[8], 20, 0x455a14ed); | |
346 | a = md5Round2(a, b, c, d, x[13], 5, 0xa9e3e905); | |
347 | d = md5Round2(d, a, b, c, x[2], 9, 0xfcefa3f8); | |
348 | c = md5Round2(c, d, a, b, x[7], 14, 0x676f02d9); | |
349 | b = md5Round2(b, c, d, a, x[12], 20, 0x8d2a4c8a); | |
350 | ||
351 | // round 3 | |
352 | a = md5Round3(a, b, c, d, x[5], 4, 0xfffa3942); | |
353 | d = md5Round3(d, a, b, c, x[8], 11, 0x8771f681); | |
354 | c = md5Round3(c, d, a, b, x[11], 16, 0x6d9d6122); | |
355 | b = md5Round3(b, c, d, a, x[14], 23, 0xfde5380c); | |
356 | a = md5Round3(a, b, c, d, x[1], 4, 0xa4beea44); | |
357 | d = md5Round3(d, a, b, c, x[4], 11, 0x4bdecfa9); | |
358 | c = md5Round3(c, d, a, b, x[7], 16, 0xf6bb4b60); | |
359 | b = md5Round3(b, c, d, a, x[10], 23, 0xbebfbc70); | |
360 | a = md5Round3(a, b, c, d, x[13], 4, 0x289b7ec6); | |
361 | d = md5Round3(d, a, b, c, x[0], 11, 0xeaa127fa); | |
362 | c = md5Round3(c, d, a, b, x[3], 16, 0xd4ef3085); | |
363 | b = md5Round3(b, c, d, a, x[6], 23, 0x04881d05); | |
364 | a = md5Round3(a, b, c, d, x[9], 4, 0xd9d4d039); | |
365 | d = md5Round3(d, a, b, c, x[12], 11, 0xe6db99e5); | |
366 | c = md5Round3(c, d, a, b, x[15], 16, 0x1fa27cf8); | |
367 | b = md5Round3(b, c, d, a, x[2], 23, 0xc4ac5665); | |
368 | ||
369 | // round 4 | |
370 | a = md5Round4(a, b, c, d, x[0], 6, 0xf4292244); | |
371 | d = md5Round4(d, a, b, c, x[7], 10, 0x432aff97); | |
372 | c = md5Round4(c, d, a, b, x[14], 15, 0xab9423a7); | |
373 | b = md5Round4(b, c, d, a, x[5], 21, 0xfc93a039); | |
374 | a = md5Round4(a, b, c, d, x[12], 6, 0x655b59c3); | |
375 | d = md5Round4(d, a, b, c, x[3], 10, 0x8f0ccc92); | |
376 | c = md5Round4(c, d, a, b, x[10], 15, 0xffeff47d); | |
377 | b = md5Round4(b, c, d, a, x[1], 21, 0x85845dd1); | |
378 | a = md5Round4(a, b, c, d, x[8], 6, 0x6fa87e4f); | |
379 | d = md5Round4(d, a, b, c, x[15], 10, 0xfe2ce6e0); | |
380 | c = md5Round4(c, d, a, b, x[6], 15, 0xa3014314); | |
381 | b = md5Round4(b, c, d, a, x[13], 21, 0x4e0811a1); | |
382 | a = md5Round4(a, b, c, d, x[4], 6, 0xf7537e82); | |
383 | d = md5Round4(d, a, b, c, x[11], 10, 0xbd3af235); | |
384 | c = md5Round4(c, d, a, b, x[2], 15, 0x2ad7d2bb); | |
385 | b = md5Round4(b, c, d, a, x[9], 21, 0xeb86d391); | |
386 | ||
387 | // increment a, b, c, d | |
388 | a += aa; | |
389 | b += bb; | |
390 | c += cc; | |
391 | d += dd; | |
392 | } | |
393 | ||
394 | // break digest into bytes | |
395 | digest[0] = (Guchar)(a & 0xff); | |
396 | digest[1] = (Guchar)((a >>= 8) & 0xff); | |
397 | digest[2] = (Guchar)((a >>= 8) & 0xff); | |
398 | digest[3] = (Guchar)((a >>= 8) & 0xff); | |
399 | digest[4] = (Guchar)(b & 0xff); | |
400 | digest[5] = (Guchar)((b >>= 8) & 0xff); | |
401 | digest[6] = (Guchar)((b >>= 8) & 0xff); | |
402 | digest[7] = (Guchar)((b >>= 8) & 0xff); | |
403 | digest[8] = (Guchar)(c & 0xff); | |
404 | digest[9] = (Guchar)((c >>= 8) & 0xff); | |
405 | digest[10] = (Guchar)((c >>= 8) & 0xff); | |
406 | digest[11] = (Guchar)((c >>= 8) & 0xff); | |
407 | digest[12] = (Guchar)(d & 0xff); | |
408 | digest[13] = (Guchar)((d >>= 8) & 0xff); | |
409 | digest[14] = (Guchar)((d >>= 8) & 0xff); | |
410 | digest[15] = (Guchar)((d >>= 8) & 0xff); | |
411 | } |