]>
git.ipfire.org Git - thirdparty/cups.git/blob - pdftops/Decrypt.cxx
1 //========================================================================
5 // Copyright 1996-2003 Glyph & Cog, LLC
7 //========================================================================
11 #ifdef USE_GCC_PRAGMAS
12 #pragma implementation
19 static void rc4InitKey(Guchar
*key
, int keyLen
, Guchar
*state
);
20 static Guchar
rc4DecryptByte(Guchar
*state
, Guchar
*x
, Guchar
*y
, Guchar c
);
21 static void md5(Guchar
*msg
, int msgLen
, Guchar
*digest
);
23 static Guchar passwordPad
[32] = {
24 0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41,
25 0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08,
26 0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80,
27 0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a
30 //------------------------------------------------------------------------
32 //------------------------------------------------------------------------
34 Decrypt::Decrypt(Guchar
*fileKey
, int keyLength
, int objNum
, int objGen
) {
37 // construct object key
38 for (i
= 0; i
< keyLength
; ++i
) {
39 objKey
[i
] = fileKey
[i
];
41 objKey
[keyLength
] = objNum
& 0xff;
42 objKey
[keyLength
+ 1] = (objNum
>> 8) & 0xff;
43 objKey
[keyLength
+ 2] = (objNum
>> 16) & 0xff;
44 objKey
[keyLength
+ 3] = objGen
& 0xff;
45 objKey
[keyLength
+ 4] = (objGen
>> 8) & 0xff;
46 md5(objKey
, keyLength
+ 5, objKey
);
48 // set up for decryption
50 if ((objKeyLength
= keyLength
+ 5) > 16) {
53 rc4InitKey(objKey
, objKeyLength
, state
);
56 void Decrypt::reset() {
58 rc4InitKey(objKey
, objKeyLength
, state
);
61 Guchar
Decrypt::decryptByte(Guchar c
) {
62 return rc4DecryptByte(state
, &x
, &y
, c
);
65 GBool
Decrypt::makeFileKey(int encVersion
, int encRevision
, int keyLength
,
66 GString
*ownerKey
, GString
*userKey
,
67 int permissions
, GString
*fileID
,
68 GString
*ownerPassword
, GString
*userPassword
,
69 Guchar
*fileKey
, GBool encryptMetadata
,
70 GBool
*ownerPasswordOk
) {
71 Guchar test
[32], test2
[32];
72 GString
*userPassword2
;
78 // try using the supplied owner password to generate the user password
79 *ownerPasswordOk
= gFalse
;
81 len
= ownerPassword
->getLength();
83 memcpy(test
, ownerPassword
->getCString(), len
);
84 memcpy(test
+ len
, passwordPad
, 32 - len
);
86 memcpy(test
, ownerPassword
->getCString(), 32);
89 if (encRevision
== 3) {
90 for (i
= 0; i
< 50; ++i
) {
94 if (encRevision
== 2) {
95 rc4InitKey(test
, keyLength
, fState
);
97 for (i
= 0; i
< 32; ++i
) {
98 test2
[i
] = rc4DecryptByte(fState
, &fx
, &fy
, ownerKey
->getChar(i
));
101 memcpy(test2
, ownerKey
->getCString(), 32);
102 for (i
= 19; i
>= 0; --i
) {
103 for (j
= 0; j
< keyLength
; ++j
) {
104 tmpKey
[j
] = test
[j
] ^ i
;
106 rc4InitKey(tmpKey
, keyLength
, fState
);
108 for (j
= 0; j
< 32; ++j
) {
109 test2
[j
] = rc4DecryptByte(fState
, &fx
, &fy
, test2
[j
]);
113 userPassword2
= new GString((char *)test2
, 32);
114 if (makeFileKey2(encVersion
, encRevision
, keyLength
, ownerKey
, userKey
,
115 permissions
, fileID
, userPassword2
, fileKey
,
117 *ownerPasswordOk
= gTrue
;
118 delete userPassword2
;
121 delete userPassword2
;
124 // try using the supplied user password
125 return makeFileKey2(encVersion
, encRevision
, keyLength
, ownerKey
, userKey
,
126 permissions
, fileID
, userPassword
, fileKey
,
130 GBool
Decrypt::makeFileKey2(int encVersion
, int encRevision
, int keyLength
,
131 GString
*ownerKey
, GString
*userKey
,
132 int permissions
, GString
*fileID
,
133 GString
*userPassword
, Guchar
*fileKey
,
134 GBool encryptMetadata
) {
144 // check whether we have non-zero keyLength
145 if ( keyLength
< 5 || keyLength
> 16 ) {
150 buf
= (Guchar
*)gmalloc(72 + fileID
->getLength());
152 len
= userPassword
->getLength();
154 memcpy(buf
, userPassword
->getCString(), len
);
155 memcpy(buf
+ len
, passwordPad
, 32 - len
);
157 memcpy(buf
, userPassword
->getCString(), 32);
160 memcpy(buf
, passwordPad
, 32);
162 memcpy(buf
+ 32, ownerKey
->getCString(), 32);
163 buf
[64] = permissions
& 0xff;
164 buf
[65] = (permissions
>> 8) & 0xff;
165 buf
[66] = (permissions
>> 16) & 0xff;
166 buf
[67] = (permissions
>> 24) & 0xff;
167 memcpy(buf
+ 68, fileID
->getCString(), fileID
->getLength());
168 len
= 68 + fileID
->getLength();
169 if (!encryptMetadata
) {
175 md5(buf
, len
, fileKey
);
176 if (encRevision
== 3) {
177 for (i
= 0; i
< 50; ++i
) {
178 md5(fileKey
, keyLength
, fileKey
);
182 // test user password
183 if (encRevision
== 2) {
184 rc4InitKey(fileKey
, keyLength
, fState
);
186 for (i
= 0; i
< 32; ++i
) {
187 test
[i
] = rc4DecryptByte(fState
, &fx
, &fy
, userKey
->getChar(i
));
189 ok
= memcmp(test
, passwordPad
, 32) == 0;
190 } else if (encRevision
== 3) {
191 memcpy(test
, userKey
->getCString(), 32);
192 for (i
= 19; i
>= 0; --i
) {
193 for (j
= 0; j
< keyLength
; ++j
) {
194 tmpKey
[j
] = fileKey
[j
] ^ i
;
196 rc4InitKey(tmpKey
, keyLength
, fState
);
198 for (j
= 0; j
< 32; ++j
) {
199 test
[j
] = rc4DecryptByte(fState
, &fx
, &fy
, test
[j
]);
202 memcpy(buf
, passwordPad
, 32);
203 memcpy(buf
+ 32, fileID
->getCString(), fileID
->getLength());
204 md5(buf
, 32 + fileID
->getLength(), buf
);
205 ok
= memcmp(test
, buf
, 16) == 0;
214 //------------------------------------------------------------------------
215 // RC4-compatible decryption
216 //------------------------------------------------------------------------
218 static void rc4InitKey(Guchar
*key
, int keyLen
, Guchar
*state
) {
219 Guchar index1
, index2
;
223 for (i
= 0; i
< 256; ++i
)
226 for (i
= 0; i
< 256; ++i
) {
227 index2
= (key
[index1
] + state
[i
] + index2
) % 256;
229 state
[i
] = state
[index2
];
231 index1
= (index1
+ 1) % keyLen
;
235 static Guchar
rc4DecryptByte(Guchar
*state
, Guchar
*x
, Guchar
*y
, Guchar c
) {
236 Guchar x1
, y1
, tx
, ty
;
238 x1
= *x
= (*x
+ 1) % 256;
239 y1
= *y
= (state
[*x
] + *y
) % 256;
244 return c
^ state
[(tx
+ ty
) % 256];
247 //------------------------------------------------------------------------
248 // MD5 message digest
249 //------------------------------------------------------------------------
251 // this works around a bug in older Sun compilers
252 static inline Gulong
rotateLeft(Gulong x
, int r
) {
254 return ((x
<< r
) | (x
>> (32 - r
))) & 0xffffffff;
257 static inline Gulong
md5Round1(Gulong a
, Gulong b
, Gulong c
, Gulong d
,
258 Gulong Xk
, Gulong s
, Gulong Ti
) {
259 return b
+ rotateLeft((a
+ ((b
& c
) | (~b
& d
)) + Xk
+ Ti
), s
);
262 static inline Gulong
md5Round2(Gulong a
, Gulong b
, Gulong c
, Gulong d
,
263 Gulong Xk
, Gulong s
, Gulong Ti
) {
264 return b
+ rotateLeft((a
+ ((b
& d
) | (c
& ~d
)) + Xk
+ Ti
), s
);
267 static inline Gulong
md5Round3(Gulong a
, Gulong b
, Gulong c
, Gulong d
,
268 Gulong Xk
, Gulong s
, Gulong Ti
) {
269 return b
+ rotateLeft((a
+ (b
^ c
^ d
) + Xk
+ Ti
), s
);
272 static inline Gulong
md5Round4(Gulong a
, Gulong b
, Gulong c
, Gulong d
,
273 Gulong Xk
, Gulong s
, Gulong Ti
) {
274 return b
+ rotateLeft((a
+ (c
^ (b
| ~d
)) + Xk
+ Ti
), s
);
277 static void md5(Guchar
*msg
, int msgLen
, Guchar
*digest
) {
279 Gulong a
, b
, c
, d
, aa
, bb
, cc
, dd
;
283 // compute number of 64-byte blocks
284 // (length + pad byte (0x80) + 8 bytes for length)
285 n64
= (msgLen
+ 1 + 8 + 63) / 64;
287 // initialize a, b, c, d
293 // loop through blocks
295 for (i
= 0; i
< n64
; ++i
) {
297 // grab a 64-byte block
298 for (j
= 0; j
< 16 && k
< msgLen
- 3; ++j
, k
+= 4)
299 x
[j
] = (((((msg
[k
+3] << 8) + msg
[k
+2]) << 8) + msg
[k
+1]) << 8) + msg
[k
];
302 x
[j
] = 0x80000000 + (((msg
[k
+2] << 8) + msg
[k
+1]) << 8) + msg
[k
];
303 else if (k
== msgLen
- 2)
304 x
[j
] = 0x800000 + (msg
[k
+1] << 8) + msg
[k
];
305 else if (k
== msgLen
- 1)
306 x
[j
] = 0x8000 + msg
[k
];
322 a
= md5Round1(a
, b
, c
, d
, x
[0], 7, 0xd76aa478);
323 d
= md5Round1(d
, a
, b
, c
, x
[1], 12, 0xe8c7b756);
324 c
= md5Round1(c
, d
, a
, b
, x
[2], 17, 0x242070db);
325 b
= md5Round1(b
, c
, d
, a
, x
[3], 22, 0xc1bdceee);
326 a
= md5Round1(a
, b
, c
, d
, x
[4], 7, 0xf57c0faf);
327 d
= md5Round1(d
, a
, b
, c
, x
[5], 12, 0x4787c62a);
328 c
= md5Round1(c
, d
, a
, b
, x
[6], 17, 0xa8304613);
329 b
= md5Round1(b
, c
, d
, a
, x
[7], 22, 0xfd469501);
330 a
= md5Round1(a
, b
, c
, d
, x
[8], 7, 0x698098d8);
331 d
= md5Round1(d
, a
, b
, c
, x
[9], 12, 0x8b44f7af);
332 c
= md5Round1(c
, d
, a
, b
, x
[10], 17, 0xffff5bb1);
333 b
= md5Round1(b
, c
, d
, a
, x
[11], 22, 0x895cd7be);
334 a
= md5Round1(a
, b
, c
, d
, x
[12], 7, 0x6b901122);
335 d
= md5Round1(d
, a
, b
, c
, x
[13], 12, 0xfd987193);
336 c
= md5Round1(c
, d
, a
, b
, x
[14], 17, 0xa679438e);
337 b
= md5Round1(b
, c
, d
, a
, x
[15], 22, 0x49b40821);
340 a
= md5Round2(a
, b
, c
, d
, x
[1], 5, 0xf61e2562);
341 d
= md5Round2(d
, a
, b
, c
, x
[6], 9, 0xc040b340);
342 c
= md5Round2(c
, d
, a
, b
, x
[11], 14, 0x265e5a51);
343 b
= md5Round2(b
, c
, d
, a
, x
[0], 20, 0xe9b6c7aa);
344 a
= md5Round2(a
, b
, c
, d
, x
[5], 5, 0xd62f105d);
345 d
= md5Round2(d
, a
, b
, c
, x
[10], 9, 0x02441453);
346 c
= md5Round2(c
, d
, a
, b
, x
[15], 14, 0xd8a1e681);
347 b
= md5Round2(b
, c
, d
, a
, x
[4], 20, 0xe7d3fbc8);
348 a
= md5Round2(a
, b
, c
, d
, x
[9], 5, 0x21e1cde6);
349 d
= md5Round2(d
, a
, b
, c
, x
[14], 9, 0xc33707d6);
350 c
= md5Round2(c
, d
, a
, b
, x
[3], 14, 0xf4d50d87);
351 b
= md5Round2(b
, c
, d
, a
, x
[8], 20, 0x455a14ed);
352 a
= md5Round2(a
, b
, c
, d
, x
[13], 5, 0xa9e3e905);
353 d
= md5Round2(d
, a
, b
, c
, x
[2], 9, 0xfcefa3f8);
354 c
= md5Round2(c
, d
, a
, b
, x
[7], 14, 0x676f02d9);
355 b
= md5Round2(b
, c
, d
, a
, x
[12], 20, 0x8d2a4c8a);
358 a
= md5Round3(a
, b
, c
, d
, x
[5], 4, 0xfffa3942);
359 d
= md5Round3(d
, a
, b
, c
, x
[8], 11, 0x8771f681);
360 c
= md5Round3(c
, d
, a
, b
, x
[11], 16, 0x6d9d6122);
361 b
= md5Round3(b
, c
, d
, a
, x
[14], 23, 0xfde5380c);
362 a
= md5Round3(a
, b
, c
, d
, x
[1], 4, 0xa4beea44);
363 d
= md5Round3(d
, a
, b
, c
, x
[4], 11, 0x4bdecfa9);
364 c
= md5Round3(c
, d
, a
, b
, x
[7], 16, 0xf6bb4b60);
365 b
= md5Round3(b
, c
, d
, a
, x
[10], 23, 0xbebfbc70);
366 a
= md5Round3(a
, b
, c
, d
, x
[13], 4, 0x289b7ec6);
367 d
= md5Round3(d
, a
, b
, c
, x
[0], 11, 0xeaa127fa);
368 c
= md5Round3(c
, d
, a
, b
, x
[3], 16, 0xd4ef3085);
369 b
= md5Round3(b
, c
, d
, a
, x
[6], 23, 0x04881d05);
370 a
= md5Round3(a
, b
, c
, d
, x
[9], 4, 0xd9d4d039);
371 d
= md5Round3(d
, a
, b
, c
, x
[12], 11, 0xe6db99e5);
372 c
= md5Round3(c
, d
, a
, b
, x
[15], 16, 0x1fa27cf8);
373 b
= md5Round3(b
, c
, d
, a
, x
[2], 23, 0xc4ac5665);
376 a
= md5Round4(a
, b
, c
, d
, x
[0], 6, 0xf4292244);
377 d
= md5Round4(d
, a
, b
, c
, x
[7], 10, 0x432aff97);
378 c
= md5Round4(c
, d
, a
, b
, x
[14], 15, 0xab9423a7);
379 b
= md5Round4(b
, c
, d
, a
, x
[5], 21, 0xfc93a039);
380 a
= md5Round4(a
, b
, c
, d
, x
[12], 6, 0x655b59c3);
381 d
= md5Round4(d
, a
, b
, c
, x
[3], 10, 0x8f0ccc92);
382 c
= md5Round4(c
, d
, a
, b
, x
[10], 15, 0xffeff47d);
383 b
= md5Round4(b
, c
, d
, a
, x
[1], 21, 0x85845dd1);
384 a
= md5Round4(a
, b
, c
, d
, x
[8], 6, 0x6fa87e4f);
385 d
= md5Round4(d
, a
, b
, c
, x
[15], 10, 0xfe2ce6e0);
386 c
= md5Round4(c
, d
, a
, b
, x
[6], 15, 0xa3014314);
387 b
= md5Round4(b
, c
, d
, a
, x
[13], 21, 0x4e0811a1);
388 a
= md5Round4(a
, b
, c
, d
, x
[4], 6, 0xf7537e82);
389 d
= md5Round4(d
, a
, b
, c
, x
[11], 10, 0xbd3af235);
390 c
= md5Round4(c
, d
, a
, b
, x
[2], 15, 0x2ad7d2bb);
391 b
= md5Round4(b
, c
, d
, a
, x
[9], 21, 0xeb86d391);
393 // increment a, b, c, d
400 // break digest into bytes
401 digest
[0] = (Guchar
)(a
& 0xff);
402 digest
[1] = (Guchar
)((a
>>= 8) & 0xff);
403 digest
[2] = (Guchar
)((a
>>= 8) & 0xff);
404 digest
[3] = (Guchar
)((a
>>= 8) & 0xff);
405 digest
[4] = (Guchar
)(b
& 0xff);
406 digest
[5] = (Guchar
)((b
>>= 8) & 0xff);
407 digest
[6] = (Guchar
)((b
>>= 8) & 0xff);
408 digest
[7] = (Guchar
)((b
>>= 8) & 0xff);
409 digest
[8] = (Guchar
)(c
& 0xff);
410 digest
[9] = (Guchar
)((c
>>= 8) & 0xff);
411 digest
[10] = (Guchar
)((c
>>= 8) & 0xff);
412 digest
[11] = (Guchar
)((c
>>= 8) & 0xff);
413 digest
[12] = (Guchar
)(d
& 0xff);
414 digest
[13] = (Guchar
)((d
>>= 8) & 0xff);
415 digest
[14] = (Guchar
)((d
>>= 8) & 0xff);
416 digest
[15] = (Guchar
)((d
>>= 8) & 0xff);