blob: abceddfc9b8ac08a06ff816242738df8b06f1079 [file] [log] [blame]
William Lallemand82fe75c2012-10-23 10:25:10 +02001/*
2 * HTTP compression.
3 *
4 * Copyright 2012 Exceliance, David Du Colombier <dducolombier@exceliance.fr>
5 * William Lallemand <wlallemand@exceliance.fr>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 */
13
14#include <stdio.h>
Willy Tarreau34763642012-10-26 15:05:35 +020015
William Lallemand08289f12012-10-31 11:19:18 +010016#ifdef USE_ZLIB
Willy Tarreau34763642012-10-26 15:05:35 +020017/* Note: the crappy zlib and openssl libs both define the "free_func" type.
18 * That's a very clever idea to use such a generic name in general purpose
19 * libraries, really... The zlib one is easier to redefine than openssl's,
20 * so let's only fix this one.
21 */
22#define free_func zlib_free_func
William Lallemand82fe75c2012-10-23 10:25:10 +020023#include <zlib.h>
Willy Tarreau34763642012-10-26 15:05:35 +020024#undef free_func
William Lallemand08289f12012-10-31 11:19:18 +010025#endif /* USE_ZLIB */
William Lallemand82fe75c2012-10-23 10:25:10 +020026
27#include <common/compat.h>
William Lallemand2b502472012-10-30 14:30:39 +010028#include <common/memory.h>
William Lallemand82fe75c2012-10-23 10:25:10 +020029
30#include <types/global.h>
31#include <types/compression.h>
32
33#include <proto/compression.h>
William Lallemandd85f9172012-11-09 17:05:39 +010034#include <proto/freq_ctr.h>
William Lallemand82fe75c2012-10-23 10:25:10 +020035#include <proto/proto_http.h>
36
William Lallemand2b502472012-10-30 14:30:39 +010037
38#ifdef USE_ZLIB
39
40/* zlib allocation */
41static struct pool_head *zlib_pool_deflate_state = NULL;
42static struct pool_head *zlib_pool_window = NULL;
43static struct pool_head *zlib_pool_prev = NULL;
44static struct pool_head *zlib_pool_head = NULL;
45static struct pool_head *zlib_pool_pending_buf = NULL;
46
William Lallemand9d5f5482012-11-07 16:12:57 +010047static long long zlib_memory_available = -1;
48
49
William Lallemand2b502472012-10-30 14:30:39 +010050#endif
51
52
Cyril Bonté6162c432012-11-10 19:27:47 +010053const struct comp_algo comp_algos[] =
William Lallemand82fe75c2012-10-23 10:25:10 +020054{
55 { "identity", 8, identity_init, identity_add_data, identity_flush, identity_reset, identity_end },
56#ifdef USE_ZLIB
57 { "deflate", 7, deflate_init, deflate_add_data, deflate_flush, deflate_reset, deflate_end },
58 { "gzip", 4, gzip_init, deflate_add_data, deflate_flush, deflate_reset, deflate_end },
59#endif /* USE_ZLIB */
60 { NULL, 0, NULL , NULL, NULL, NULL, NULL }
61};
62
63/*
64 * Add a content-type in the configuration
65 */
66int comp_append_type(struct comp *comp, const char *type)
67{
68 struct comp_type *comp_type;
69
70 comp_type = calloc(1, sizeof(struct comp_type));
71 comp_type->name_len = strlen(type);
72 comp_type->name = strdup(type);
73 comp_type->next = comp->types;
74 comp->types = comp_type;
75 return 0;
76}
77
78/*
79 * Add an algorithm in the configuration
80 */
81int comp_append_algo(struct comp *comp, const char *algo)
82{
83 struct comp_algo *comp_algo;
84 int i;
85
86 for (i = 0; comp_algos[i].name; i++) {
87 if (!strcmp(algo, comp_algos[i].name)) {
88 comp_algo = calloc(1, sizeof(struct comp_algo));
89 memmove(comp_algo, &comp_algos[i], sizeof(struct comp_algo));
90 comp_algo->next = comp->algos;
91 comp->algos = comp_algo;
92 return 0;
93 }
94 }
95 return -1;
96}
97
98/* emit the chunksize followed by a CRLF on the output and return the number of
99 * bytes written. Appends <add_crlf> additional CRLF after the first one. Chunk
100 * sizes are truncated to 6 hex digits (16 MB) and padded left. The caller is
101 * responsible for ensuring there is enough room left in the output buffer for
102 * the string (8 bytes * add_crlf*2).
103 */
104int http_emit_chunk_size(char *out, unsigned int chksz, int add_crlf)
105{
106 int shift;
107 int pos = 0;
108
109 for (shift = 20; shift >= 0; shift -= 4)
110 out[pos++] = hextab[(chksz >> shift) & 0xF];
111
112 do {
113 out[pos++] = '\r';
114 out[pos++] = '\n';
115 } while (--add_crlf >= 0);
116
117 return pos;
118}
119
120/*
121 * Init HTTP compression
122 */
123int http_compression_buffer_init(struct session *s, struct buffer *in, struct buffer *out)
124{
125 struct http_msg *msg = &s->txn.rsp;
126 int left;
127
128 /* not enough space */
129 if (in->size - buffer_len(in) < 40)
130 return -1;
131
132 /*
133 * Skip data, we don't need them in the new buffer. They are results
134 * of CHUNK_CRLF and CHUNK_SIZE parsing.
135 */
136 b_adv(in, msg->next);
137 msg->next = 0;
138 msg->sov = 0;
139 msg->sol = 0;
140
141 out->size = global.tune.bufsize;
142 out->i = 0;
143 out->o = 0;
144 out->p = out->data;
145 /* copy output data */
146 if (in->o > 0) {
147 left = in->o - bo_contig_data(in);
148 memcpy(out->data, bo_ptr(in), bo_contig_data(in));
149 out->p += bo_contig_data(in);
150 if (left > 0) { /* second part of the buffer */
151 memcpy(out->p, in->data, left);
152 out->p += left;
153 }
154 out->o = in->o;
155 }
156 out->i += http_emit_chunk_size(out->p, 0, 0);
157
158 return 0;
159}
160
161/*
162 * Add data to compress
163 */
164int http_compression_buffer_add_data(struct session *s, struct buffer *in, struct buffer *out)
165{
166 struct http_msg *msg = &s->txn.rsp;
167 int data_process_len;
168 int left;
169 int ret;
170
171 /*
172 * Skip data, we don't need them in the new buffer. They are results
173 * of CHUNK_CRLF and CHUNK_SIZE parsing.
174 */
175 b_adv(in, msg->next);
176 msg->next = 0;
177 msg->sov = 0;
178 msg->sol = 0;
179
180 /*
181 * select the smallest size between the announced chunk size, the input
182 * data, and the available output buffer size
183 */
184 data_process_len = MIN(in->i, msg->chunk_len);
185 data_process_len = MIN(out->size - buffer_len(out), data_process_len);
186
187 left = data_process_len - bi_contig_data(in);
188 if (left <= 0) {
William Lallemand1c2d6222012-10-30 15:52:53 +0100189 ret = s->comp_algo->add_data(&s->comp_ctx, bi_ptr(in),
William Lallemand82fe75c2012-10-23 10:25:10 +0200190 data_process_len, bi_end(out),
191 out->size - buffer_len(out));
192 if (ret < 0)
193 return -1;
194 out->i += ret;
195
196 } else {
William Lallemand1c2d6222012-10-30 15:52:53 +0100197 ret = s->comp_algo->add_data(&s->comp_ctx, bi_ptr(in), bi_contig_data(in), bi_end(out), out->size - buffer_len(out));
William Lallemand82fe75c2012-10-23 10:25:10 +0200198 if (ret < 0)
199 return -1;
200 out->i += ret;
William Lallemand1c2d6222012-10-30 15:52:53 +0100201 ret = s->comp_algo->add_data(&s->comp_ctx, in->data, left, bi_end(out), out->size - buffer_len(out));
William Lallemand82fe75c2012-10-23 10:25:10 +0200202 if (ret < 0)
203 return -1;
204 out->i += ret;
205 }
206
207 b_adv(in, data_process_len);
208 msg->chunk_len -= data_process_len;
209
210 return 0;
211}
212
213/*
214 * Flush data in process, and write the header and footer of the chunk. Upon
215 * success, in and out buffers are swapped to avoid a copy.
216 */
217int http_compression_buffer_end(struct session *s, struct buffer **in, struct buffer **out, int end)
218{
219 int to_forward;
220 int left;
221 struct http_msg *msg = &s->txn.rsp;
222 struct buffer *ib = *in, *ob = *out;
William Lallemand08289f12012-10-31 11:19:18 +0100223
224#ifdef USE_ZLIB
William Lallemand82fe75c2012-10-23 10:25:10 +0200225 int ret;
226
227 /* flush data here */
228
229 if (end)
230 ret = s->comp_algo->flush(&s->comp_ctx, ob, Z_FINISH); /* end of data */
231 else
232 ret = s->comp_algo->flush(&s->comp_ctx, ob, Z_SYNC_FLUSH); /* end of buffer */
233
234 if (ret < 0)
235 return -1; /* flush failed */
236
William Lallemand08289f12012-10-31 11:19:18 +0100237#endif /* USE_ZLIB */
238
William Lallemand82fe75c2012-10-23 10:25:10 +0200239 if (ob->i > 8) {
240 /* more than a chunk size => some data were emitted */
241 char *tail = ob->p + ob->i;
242
243 /* write real size at the begining of the chunk, no need of wrapping */
244 http_emit_chunk_size(ob->p, ob->i - 8, 0);
245
246 /* chunked encoding requires CRLF after data */
247 *tail++ = '\r';
248 *tail++ = '\n';
249
250 if (!(msg->flags & HTTP_MSGF_TE_CHNK) && msg->chunk_len == 0) {
251 /* End of data, 0<CRLF><CRLF> is needed but we're not
252 * in chunked mode on input so we must add it ourselves.
253 */
254 memcpy(tail, "0\r\n\r\n", 5);
255 tail += 5;
256 }
257 ob->i = tail - ob->p;
258 } else {
259 /* no data were sent, cancel the chunk size */
260 ob->i = 0;
261 }
262
263 to_forward = ob->i;
264
William Lallemandd85f9172012-11-09 17:05:39 +0100265 /* update input rate */
266 if (s->comp_ctx.cur_lvl > 0)
267 update_freq_ctr(&global.comp_bps_in, ib->o - ob->o);
268
William Lallemand82fe75c2012-10-23 10:25:10 +0200269 /* copy the remaining data in the tmp buffer. */
270 if (ib->i > 0) {
271 left = ib->i - bi_contig_data(ib);
272 memcpy(bi_end(ob), bi_ptr(ib), bi_contig_data(ib));
273 ob->i += bi_contig_data(ib);
274 if (left > 0) {
275 memcpy(bi_end(ob), ib->data, left);
276 ob->i += left;
277 }
278 }
279
280 /* swap the buffers */
281 *in = ob;
282 *out = ib;
283
William Lallemandd85f9172012-11-09 17:05:39 +0100284 if (s->comp_ctx.cur_lvl > 0)
285 update_freq_ctr(&global.comp_bps_out, to_forward);
286
William Lallemand82fe75c2012-10-23 10:25:10 +0200287 /* forward the new chunk without remaining data */
288 b_adv(ob, to_forward);
289
290 /* if there are data between p and next, there are trailers, must forward them */
291 b_adv(ob, msg->next);
292 msg->next = 0;
293
294 return to_forward;
295}
296
297
298/****************************
299 **** Identity algorithm ****
300 ****************************/
301
302/*
303 * Init the identity algorithm
304 */
William Lallemand1c2d6222012-10-30 15:52:53 +0100305int identity_init(struct comp_ctx *comp_ctx, int level)
William Lallemand82fe75c2012-10-23 10:25:10 +0200306{
307 return 0;
308}
309
310/*
311 * Process data
312 * Return size of processed data or -1 on error
313 */
William Lallemand1c2d6222012-10-30 15:52:53 +0100314int identity_add_data(struct comp_ctx *comp_ctx, const char *in_data, int in_len, char *out_data, int out_len)
William Lallemand82fe75c2012-10-23 10:25:10 +0200315{
316 if (out_len < in_len)
317 return -1;
318
319 memcpy(out_data, in_data, in_len);
320
321 return in_len;
322}
323
William Lallemand1c2d6222012-10-30 15:52:53 +0100324int identity_flush(struct comp_ctx *comp_ctx, struct buffer *out, int flag)
William Lallemand82fe75c2012-10-23 10:25:10 +0200325{
326 return 0;
327}
328
329
William Lallemand1c2d6222012-10-30 15:52:53 +0100330int identity_reset(struct comp_ctx *comp_ctx)
William Lallemand82fe75c2012-10-23 10:25:10 +0200331{
332 return 0;
333}
334
335/*
336 * Deinit the algorithm
337 */
William Lallemand1c2d6222012-10-30 15:52:53 +0100338int identity_end(struct comp_ctx *comp_ctx)
William Lallemand82fe75c2012-10-23 10:25:10 +0200339{
340 return 0;
341}
342
343
344#ifdef USE_ZLIB
William Lallemand2b502472012-10-30 14:30:39 +0100345/*
346 * This is a tricky allocation function using the zlib.
347 * This is based on the allocation order in deflateInit2.
348 */
349static void *alloc_zlib(void *opaque, unsigned int items, unsigned int size)
350{
351 struct comp_ctx *ctx = opaque;
352 static char round = 0; /* order in deflateInit2 */
353 void *buf = NULL;
354
William Lallemand9d5f5482012-11-07 16:12:57 +0100355 if (global.maxzlibmem > 0 && zlib_memory_available < items * size){
356 buf = NULL;
357 goto end;
358 }
359
William Lallemand2b502472012-10-30 14:30:39 +0100360 switch (round) {
361 case 0:
362 if (zlib_pool_deflate_state == NULL)
363 zlib_pool_deflate_state = create_pool("zlib_state", size * items, MEM_F_SHARED);
364 ctx->zlib_deflate_state = buf = pool_alloc2(zlib_pool_deflate_state);
365 break;
366
367 case 1:
368 if (zlib_pool_window == NULL)
369 zlib_pool_window = create_pool("zlib_window", size * items, MEM_F_SHARED);
370 ctx->zlib_window = buf = pool_alloc2(zlib_pool_window);
371 break;
372
373 case 2:
374 if (zlib_pool_prev == NULL)
375 zlib_pool_prev = create_pool("zlib_prev", size * items, MEM_F_SHARED);
376 ctx->zlib_prev = buf = pool_alloc2(zlib_pool_prev);
377 break;
378
379 case 3:
380 if (zlib_pool_head == NULL)
381 zlib_pool_head = create_pool("zlib_head", size * items, MEM_F_SHARED);
382 ctx->zlib_head = buf = pool_alloc2(zlib_pool_head);
383 break;
384
385 case 4:
386 if (zlib_pool_pending_buf == NULL)
387 zlib_pool_pending_buf = create_pool("zlib_pending_buf", size * items, MEM_F_SHARED);
388 ctx->zlib_pending_buf = buf = pool_alloc2(zlib_pool_pending_buf);
389 break;
390 }
William Lallemand9d5f5482012-11-07 16:12:57 +0100391 if (buf != NULL && global.maxzlibmem > 0)
392 zlib_memory_available -= items * size;
393
394end:
William Lallemand2b502472012-10-30 14:30:39 +0100395
Willy Tarreau46909852012-11-15 14:57:56 +0100396 /* deflateInit2() first allocates and checks the deflate_state, then if
397 * it succeeds, it allocates all other 4 areas at ones and checks them
398 * at the end. So we want to correctly count the rounds depending on when
399 * zlib is supposed to abort.
400 */
401 if (buf || round)
402 round = (round + 1) % 5;
William Lallemand2b502472012-10-30 14:30:39 +0100403 return buf;
404}
405
406static void free_zlib(void *opaque, void *ptr)
407{
408 struct comp_ctx *ctx = opaque;
Willy Tarreaub1fbd052012-11-10 17:49:37 +0100409 struct pool_head *pool = NULL;
William Lallemand2b502472012-10-30 14:30:39 +0100410
411 if (ptr == ctx->zlib_window)
William Lallemand9d5f5482012-11-07 16:12:57 +0100412 pool = zlib_pool_window;
William Lallemand2b502472012-10-30 14:30:39 +0100413 else if (ptr == ctx->zlib_deflate_state)
William Lallemand9d5f5482012-11-07 16:12:57 +0100414 pool = zlib_pool_deflate_state;
William Lallemand2b502472012-10-30 14:30:39 +0100415 else if (ptr == ctx->zlib_prev)
William Lallemand9d5f5482012-11-07 16:12:57 +0100416 pool = zlib_pool_prev;
William Lallemand2b502472012-10-30 14:30:39 +0100417 else if (ptr == ctx->zlib_head)
William Lallemand9d5f5482012-11-07 16:12:57 +0100418 pool = zlib_pool_head;
William Lallemand2b502472012-10-30 14:30:39 +0100419 else if (ptr == ctx->zlib_pending_buf)
William Lallemand9d5f5482012-11-07 16:12:57 +0100420 pool = zlib_pool_pending_buf;
William Lallemand2b502472012-10-30 14:30:39 +0100421
William Lallemand9d5f5482012-11-07 16:12:57 +0100422 pool_free2(pool, ptr);
423 if (global.maxzlibmem > 0)
424 zlib_memory_available += pool->size;
William Lallemand2b502472012-10-30 14:30:39 +0100425}
426
William Lallemand82fe75c2012-10-23 10:25:10 +0200427
428/**************************
429**** gzip algorithm ****
430***************************/
William Lallemand1c2d6222012-10-30 15:52:53 +0100431int gzip_init(struct comp_ctx *comp_ctx, int level)
William Lallemand82fe75c2012-10-23 10:25:10 +0200432{
William Lallemand1c2d6222012-10-30 15:52:53 +0100433 z_stream *strm = &comp_ctx->strm;
William Lallemand82fe75c2012-10-23 10:25:10 +0200434
William Lallemand9d5f5482012-11-07 16:12:57 +0100435 if (global.maxzlibmem > 0 && zlib_memory_available < 0)
436 zlib_memory_available = global.maxzlibmem * 1024 * 1024; /* Megabytes to bytes */
437
William Lallemand2b502472012-10-30 14:30:39 +0100438 strm->zalloc = alloc_zlib;
439 strm->zfree = free_zlib;
440 strm->opaque = comp_ctx;
William Lallemand82fe75c2012-10-23 10:25:10 +0200441
William Lallemanda509e4c2012-11-07 16:54:34 +0100442 if (deflateInit2(&comp_ctx->strm, level, Z_DEFLATED, global.tune.zlibwindowsize + 16, global.tune.zlibmemlevel, Z_DEFAULT_STRATEGY) != Z_OK)
William Lallemand82fe75c2012-10-23 10:25:10 +0200443 return -1;
444
445 return 0;
446}
447/**************************
448**** Deflate algorithm ****
449***************************/
450
William Lallemand1c2d6222012-10-30 15:52:53 +0100451int deflate_init(struct comp_ctx *comp_ctx, int level)
William Lallemand82fe75c2012-10-23 10:25:10 +0200452{
William Lallemand1c2d6222012-10-30 15:52:53 +0100453 z_stream *strm = &comp_ctx->strm;
William Lallemand82fe75c2012-10-23 10:25:10 +0200454
William Lallemand2b502472012-10-30 14:30:39 +0100455 strm->zalloc = alloc_zlib;
456 strm->zfree = free_zlib;
457 strm->opaque = comp_ctx;
William Lallemand82fe75c2012-10-23 10:25:10 +0200458
William Lallemand1c2d6222012-10-30 15:52:53 +0100459 if (deflateInit(&comp_ctx->strm, level) != Z_OK)
William Lallemand82fe75c2012-10-23 10:25:10 +0200460 return -1;
461
462 return 0;
463}
464
William Lallemand1c2d6222012-10-30 15:52:53 +0100465int deflate_add_data(struct comp_ctx *comp_ctx, const char *in_data, int in_len, char *out_data, int out_len)
William Lallemand82fe75c2012-10-23 10:25:10 +0200466{
William Lallemand1c2d6222012-10-30 15:52:53 +0100467 z_stream *strm = &comp_ctx->strm;
William Lallemand82fe75c2012-10-23 10:25:10 +0200468 int ret;
469
470 if (in_len <= 0)
471 return 0;
472
473
474 if (out_len <= 0)
475 return -1;
476
William Lallemand82fe75c2012-10-23 10:25:10 +0200477 strm->next_in = (unsigned char *)in_data;
478 strm->avail_in = in_len;
479 strm->next_out = (unsigned char *)out_data;
480 strm->avail_out = out_len;
481
482 ret = deflate(strm, Z_NO_FLUSH);
483 if (ret != Z_OK)
484 return -1;
485
486 /* deflate update the available data out */
487
488 return out_len - strm->avail_out;
489}
490
William Lallemand1c2d6222012-10-30 15:52:53 +0100491int deflate_flush(struct comp_ctx *comp_ctx, struct buffer *out, int flag)
William Lallemand82fe75c2012-10-23 10:25:10 +0200492{
493 int ret;
William Lallemand82fe75c2012-10-23 10:25:10 +0200494 int out_len = 0;
William Lallemand1c2d6222012-10-30 15:52:53 +0100495 z_stream *strm = &comp_ctx->strm;
William Lallemand82fe75c2012-10-23 10:25:10 +0200496
William Lallemand82fe75c2012-10-23 10:25:10 +0200497 strm->next_out = (unsigned char *)bi_end(out);
498 strm->avail_out = out->size - buffer_len(out);
499
500 ret = deflate(strm, flag);
501 if (ret != Z_OK && ret != Z_STREAM_END)
502 return -1;
503
504 out_len = (out->size - buffer_len(out)) - strm->avail_out;
505 out->i += out_len;
506
William Lallemandd85f9172012-11-09 17:05:39 +0100507 /* compression rate limit */
508 if (global.comp_rate_lim > 0) {
509
510 if (read_freq_ctr(&global.comp_bps_out) > global.comp_rate_lim) {
511 /* decrease level */
512 if (comp_ctx->cur_lvl > 0) {
513 comp_ctx->cur_lvl--;
514 deflateParams(&comp_ctx->strm, comp_ctx->cur_lvl, Z_DEFAULT_STRATEGY);
515 }
516
517 } else if (comp_ctx->cur_lvl < global.comp_rate_lim) {
518 /* increase level */
519 comp_ctx->cur_lvl++ ;
520 deflateParams(&comp_ctx->strm, comp_ctx->cur_lvl, Z_DEFAULT_STRATEGY);
521 }
522 }
523
William Lallemand82fe75c2012-10-23 10:25:10 +0200524 return out_len;
525}
526
William Lallemand1c2d6222012-10-30 15:52:53 +0100527int deflate_reset(struct comp_ctx *comp_ctx)
William Lallemand82fe75c2012-10-23 10:25:10 +0200528{
William Lallemand1c2d6222012-10-30 15:52:53 +0100529 z_stream *strm = &comp_ctx->strm;
William Lallemand82fe75c2012-10-23 10:25:10 +0200530
William Lallemand82fe75c2012-10-23 10:25:10 +0200531 if (deflateReset(strm) == Z_OK)
532 return 0;
533 return -1;
534}
535
William Lallemand1c2d6222012-10-30 15:52:53 +0100536int deflate_end(struct comp_ctx *comp_ctx)
William Lallemand82fe75c2012-10-23 10:25:10 +0200537{
William Lallemand1c2d6222012-10-30 15:52:53 +0100538 z_stream *strm = &comp_ctx->strm;
William Lallemand82fe75c2012-10-23 10:25:10 +0200539
William Lallemand9d5f5482012-11-07 16:12:57 +0100540 if (deflateEnd(strm) != Z_OK)
541 return -1;
William Lallemand82fe75c2012-10-23 10:25:10 +0200542
William Lallemand9d5f5482012-11-07 16:12:57 +0100543 return 0;
William Lallemand82fe75c2012-10-23 10:25:10 +0200544}
545
546#endif /* USE_ZLIB */
547