blob: b07065d7a0d73b0220f467df29826ad9d7142160 [file] [log] [blame]
Christopher Faulet3d97c902015-12-09 14:59:38 +01001/*
2 * Stream filters related variables and functions.
3 *
4 * Copyright (C) 2015 Qualys Inc., Christopher Faulet <cfaulet@qualys.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
13#include <common/buffer.h>
14#include <common/cfgparse.h>
15#include <common/mini-clist.h>
16#include <common/standard.h>
17
18#include <types/compression.h>
19#include <types/filters.h>
20#include <types/proto_http.h>
21#include <types/proxy.h>
22#include <types/sample.h>
23
24#include <proto/compression.h>
Christopher Faulet92d36382015-11-05 13:35:03 +010025#include <proto/filters.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010026#include <proto/hdr_idx.h>
27#include <proto/proto_http.h>
28#include <proto/sample.h>
29#include <proto/stream.h>
30
Christopher Faulet92d36382015-11-05 13:35:03 +010031static const char *http_comp_flt_id = "compression filter";
32
33struct flt_ops comp_ops;
34
35static struct buffer *tmpbuf = &buf_empty;
36
Christopher Faulet92d36382015-11-05 13:35:03 +010037struct comp_state {
38 struct comp_ctx *comp_ctx; /* compression context */
39 struct comp_algo *comp_algo; /* compression algorithm if not NULL */
Christopher Faulet2fb28802015-12-01 10:40:57 +010040 int sov;
41 int consumed;
42 int initialized;
Christopher Faulet92d36382015-11-05 13:35:03 +010043};
44
Christopher Faulet92d36382015-11-05 13:35:03 +010045static int select_compression_request_header(struct comp_state *st,
46 struct stream *s,
47 struct http_msg *msg);
48static int select_compression_response_header(struct comp_state *st,
49 struct stream *s,
50 struct http_msg *msg);
51
52static int http_compression_buffer_init(struct buffer *in, struct buffer *out);
53static int http_compression_buffer_add_data(struct comp_state *st,
54 struct buffer *in,
55 struct buffer *out, int sz);
56static int http_compression_buffer_end(struct comp_state *st, struct stream *s,
57 struct buffer **in, struct buffer **out,
Christopher Faulet2fb28802015-12-01 10:40:57 +010058 int end);
Christopher Faulet92d36382015-11-05 13:35:03 +010059
60/***********************************************************************/
61static int
62comp_flt_init(struct proxy *px, struct filter *filter)
63{
64
65 /* We need a compression buffer in the DATA state to put the output of
66 * compressed data, and in CRLF state to let the TRAILERS state finish
67 * the job of removing the trailing CRLF.
68 */
69 if (!tmpbuf->size) {
70 if (b_alloc(&tmpbuf) == NULL)
71 return -1;
72 }
73 return 0;
74}
75
76static void
77comp_flt_deinit(struct proxy *px, struct filter *filter)
78{
79 if (tmpbuf->size)
80 b_free(&tmpbuf);
81}
82
83static int
84comp_start_analyze(struct stream *s, struct filter *filter, struct channel *chn)
85{
86 if (filter->ctx == NULL) {
87 struct comp_state *st;
88
89 if (!(st = malloc(sizeof(*st))))
90 return -1;
91
Christopher Faulet2fb28802015-12-01 10:40:57 +010092 st->comp_algo = NULL;
93 st->comp_ctx = NULL;
94 st->sov = 0;
95 st->consumed = 0;
96 st->initialized = 0;
97 filter->ctx = st;
Christopher Faulet92d36382015-11-05 13:35:03 +010098 }
99 return 1;
100}
101
102static int
103comp_analyze(struct stream *s, struct filter *filter, struct channel *chn,
104 unsigned int an_bit)
105{
106 struct comp_state *st = filter->ctx;
107
108 if (!strm_fe(s)->comp && !s->be->comp)
109 goto end;
110
Christopher Faulet309c6412015-12-02 09:57:32 +0100111 if (an_bit == AN_FLT_HTTP_HDRS) {
112 if (!(chn->flags & CF_ISRESP))
113 select_compression_request_header(st, s, &s->txn->req);
114 else {
Christopher Faulet92d36382015-11-05 13:35:03 +0100115 select_compression_response_header(st, s, &s->txn->rsp);
Christopher Faulet2fb28802015-12-01 10:40:57 +0100116 if (st->comp_algo)
117 st->sov = s->txn->rsp.sov;
Christopher Faulet309c6412015-12-02 09:57:32 +0100118 }
Christopher Faulet92d36382015-11-05 13:35:03 +0100119 }
Christopher Faulet309c6412015-12-02 09:57:32 +0100120
Christopher Faulet92d36382015-11-05 13:35:03 +0100121 end:
122 return 1;
123}
124
125static int
126comp_end_analyze(struct stream *s, struct filter *filter, struct channel *chn)
127{
128 struct comp_state *st = filter->ctx;
Christopher Faulet92d36382015-11-05 13:35:03 +0100129
130 if (!st || !(chn->flags & CF_ISRESP))
131 goto end;
132
Christopher Faulet92d36382015-11-05 13:35:03 +0100133 if (!st->comp_algo || !s->txn->status)
134 goto release_ctx;
135
136 if (strm_fe(s)->mode == PR_MODE_HTTP)
137 strm_fe(s)->fe_counters.p.http.comp_rsp++;
138 if ((s->flags & SF_BE_ASSIGNED) && (s->be->mode == PR_MODE_HTTP))
139 s->be->be_counters.p.http.comp_rsp++;
140
141 /* release any possible compression context */
142 st->comp_algo->end(&st->comp_ctx);
143
144 release_ctx:
145 free(st);
146 filter->ctx = NULL;
147 end:
148 return 1;
149}
150
151static int
Christopher Faulet2fb28802015-12-01 10:40:57 +0100152comp_http_data(struct stream *s, struct filter *filter, struct http_msg *msg)
Christopher Faulet92d36382015-11-05 13:35:03 +0100153{
154 struct comp_state *st = filter->ctx;
Christopher Faulet2fb28802015-12-01 10:40:57 +0100155 unsigned int len;
Christopher Faulet92d36382015-11-05 13:35:03 +0100156 int ret;
157
158 if (!(msg->chn->flags & CF_ISRESP) || !st->comp_algo) {
159 flt_set_forward_data(filter, msg->chn);
160 return 1;
161 }
162
Christopher Faulet2fb28802015-12-01 10:40:57 +0100163 len = MIN(msg->chunk_len + msg->next, msg->chn->buf->i) - FLT_NXT(filter, msg->chn);
164 if (!len)
165 return len;
166
167 if (!st->initialized) {
168 b_adv(msg->chn->buf, FLT_FWD(filter, msg->chn) + st->sov);
169 ret = http_compression_buffer_init(msg->chn->buf, tmpbuf);
170 b_rew(msg->chn->buf, FLT_FWD(filter, msg->chn) + st->sov);
171 if (ret < 0) {
172 msg->chn->flags |= CF_WAKE_WRITE;
173 return 0;
174 }
175 }
176 b_adv(msg->chn->buf, FLT_NXT(filter, msg->chn));
177 ret = http_compression_buffer_add_data(st, msg->chn->buf, tmpbuf, len);
178 b_rew(msg->chn->buf, FLT_NXT(filter, msg->chn));
179 if (ret < 0)
180 return ret;
Christopher Faulet92d36382015-11-05 13:35:03 +0100181
Christopher Faulet2fb28802015-12-01 10:40:57 +0100182 st->initialized = 1;
183 msg->next += ret;
184 msg->chunk_len -= ret;
185 FLT_NXT(filter, msg->chn) = msg->next;
186 return 0;
Christopher Faulet92d36382015-11-05 13:35:03 +0100187}
188
189static int
Christopher Faulet2fb28802015-12-01 10:40:57 +0100190comp_http_chunk_trailers(struct stream *s, struct filter *filter,
191 struct http_msg *msg)
Christopher Faulet92d36382015-11-05 13:35:03 +0100192{
193 struct comp_state *st = filter->ctx;
Christopher Faulet2fb28802015-12-01 10:40:57 +0100194 int ret;
Christopher Faulet92d36382015-11-05 13:35:03 +0100195
Christopher Faulet92d36382015-11-05 13:35:03 +0100196 if (!(msg->chn->flags & CF_ISRESP) || !st->comp_algo) {
197 flt_set_forward_data(filter, msg->chn);
Christopher Faulet2fb28802015-12-01 10:40:57 +0100198 return 1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100199 }
Christopher Faulet92d36382015-11-05 13:35:03 +0100200
Christopher Faulet2fb28802015-12-01 10:40:57 +0100201 if (!st->initialized)
202 return 1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100203
Christopher Faulet2fb28802015-12-01 10:40:57 +0100204 st->consumed = msg->next - st->sov;
205 b_adv(msg->chn->buf, FLT_FWD(filter, msg->chn) + st->sov);
206 ret = http_compression_buffer_end(st, s, &msg->chn->buf, &tmpbuf, 1);
207 if (ret < 0)
208 return ret;
209
210 st->initialized = 0;
211 st->sov = 0;
212 msg->next = ret;
213 FLT_NXT(filter, msg->chn) = ret;
214 FLT_FWD(filter, msg->chn) = 0;
215 return 1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100216}
217
Christopher Faulet2fb28802015-12-01 10:40:57 +0100218
Christopher Faulet92d36382015-11-05 13:35:03 +0100219static int
220comp_http_forward_data(struct stream *s, struct filter *filter,
221 struct http_msg *msg, unsigned int len)
222{
223 struct comp_state *st = filter->ctx;
Christopher Faulet2fb28802015-12-01 10:40:57 +0100224 int ret;
Christopher Faulet92d36382015-11-05 13:35:03 +0100225
226 if (!(msg->chn->flags & CF_ISRESP) || !st->comp_algo) {
227 flt_set_forward_data(filter, msg->chn);
Christopher Faulet2fb28802015-12-01 10:40:57 +0100228 ret = len;
229 return ret;
Christopher Faulet92d36382015-11-05 13:35:03 +0100230 }
231
Christopher Faulet2fb28802015-12-01 10:40:57 +0100232 /* To work, previous filters MUST forward all data */
233 if (FLT_FWD(filter, msg->chn) + len != FLT_NXT(filter, msg->chn)) {
234 Warning("HTTP compression failed: unexpected behavior of previous filters\n");
235 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100236 }
237
Christopher Faulet2fb28802015-12-01 10:40:57 +0100238 if (!st->initialized) {
239 ret = len;
240 st->sov = ((st->sov > ret) ? (st->sov-ret) : 0);
241 return ret;
Christopher Faulet92d36382015-11-05 13:35:03 +0100242 }
243
Christopher Faulet2fb28802015-12-01 10:40:57 +0100244 st->consumed = len - st->sov;
245 b_adv(msg->chn->buf, FLT_FWD(filter, msg->chn) + st->sov);
246 ret = http_compression_buffer_end(st, s, &msg->chn->buf, &tmpbuf,
247 msg->msg_state == HTTP_MSG_ENDING);
248 if (ret < 0)
249 return ret;
Christopher Faulet92d36382015-11-05 13:35:03 +0100250
Christopher Faulet2fb28802015-12-01 10:40:57 +0100251 st->initialized = 0;
252 st->sov = 0;
253 msg->next = ret;
254 FLT_NXT(filter, msg->chn) = ret;
255 FLT_FWD(filter, msg->chn) = 0;
Christopher Faulet92d36382015-11-05 13:35:03 +0100256 return ret;
257}
Christopher Faulet3d97c902015-12-09 14:59:38 +0100258
259/***********************************************************************/
260/*
261 * Selects a compression algorithm depending on the client request.
262 */
263int
Christopher Faulet92d36382015-11-05 13:35:03 +0100264select_compression_request_header(struct comp_state *st, struct stream *s,
265 struct http_msg *msg)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100266{
267 struct http_txn *txn = s->txn;
Christopher Faulet92d36382015-11-05 13:35:03 +0100268 struct buffer *req = msg->chn->buf;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100269 struct hdr_ctx ctx;
270 struct comp_algo *comp_algo = NULL;
271 struct comp_algo *comp_algo_back = NULL;
272
273 /* Disable compression for older user agents announcing themselves as "Mozilla/4"
274 * unless they are known good (MSIE 6 with XP SP2, or MSIE 7 and later).
275 * See http://zoompf.com/2012/02/lose-the-wait-http-compression for more details.
276 */
277 ctx.idx = 0;
278 if (http_find_header2("User-Agent", 10, req->p, &txn->hdr_idx, &ctx) &&
279 ctx.vlen >= 9 &&
280 memcmp(ctx.line + ctx.val, "Mozilla/4", 9) == 0 &&
281 (ctx.vlen < 31 ||
282 memcmp(ctx.line + ctx.val + 25, "MSIE ", 5) != 0 ||
283 ctx.line[ctx.val + 30] < '6' ||
284 (ctx.line[ctx.val + 30] == '6' &&
285 (ctx.vlen < 54 || memcmp(ctx.line + 51, "SV1", 3) != 0)))) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100286 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100287 return 0;
288 }
289
290 /* search for the algo in the backend in priority or the frontend */
Christopher Faulet92d36382015-11-05 13:35:03 +0100291 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
292 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100293 int best_q = 0;
294
295 ctx.idx = 0;
296 while (http_find_header2("Accept-Encoding", 15, req->p, &txn->hdr_idx, &ctx)) {
297 const char *qval;
298 int q;
299 int toklen;
300
301 /* try to isolate the token from the optional q-value */
302 toklen = 0;
303 while (toklen < ctx.vlen && http_is_token[(unsigned char)*(ctx.line + ctx.val + toklen)])
304 toklen++;
305
306 qval = ctx.line + ctx.val + toklen;
307 while (1) {
308 while (qval < ctx.line + ctx.val + ctx.vlen && http_is_lws[(unsigned char)*qval])
309 qval++;
310
311 if (qval >= ctx.line + ctx.val + ctx.vlen || *qval != ';') {
312 qval = NULL;
313 break;
314 }
315 qval++;
316
317 while (qval < ctx.line + ctx.val + ctx.vlen && http_is_lws[(unsigned char)*qval])
318 qval++;
319
320 if (qval >= ctx.line + ctx.val + ctx.vlen) {
321 qval = NULL;
322 break;
323 }
324 if (strncmp(qval, "q=", MIN(ctx.line + ctx.val + ctx.vlen - qval, 2)) == 0)
325 break;
326
327 while (qval < ctx.line + ctx.val + ctx.vlen && *qval != ';')
328 qval++;
329 }
330
331 /* here we have qval pointing to the first "q=" attribute or NULL if not found */
332 q = qval ? parse_qvalue(qval + 2, NULL) : 1000;
333
334 if (q <= best_q)
335 continue;
336
337 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
338 if (*(ctx.line + ctx.val) == '*' ||
339 word_match(ctx.line + ctx.val, toklen, comp_algo->ua_name, comp_algo->ua_name_len)) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100340 st->comp_algo = comp_algo;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100341 best_q = q;
342 break;
343 }
344 }
345 }
346 }
347
348 /* remove all occurrences of the header when "compression offload" is set */
Christopher Faulet92d36382015-11-05 13:35:03 +0100349 if (st->comp_algo) {
350 if ((s->be->comp && s->be->comp->offload) ||
351 (strm_fe(s)->comp && strm_fe(s)->comp->offload)) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100352 http_remove_header2(msg, &txn->hdr_idx, &ctx);
353 ctx.idx = 0;
354 while (http_find_header2("Accept-Encoding", 15, req->p, &txn->hdr_idx, &ctx)) {
355 http_remove_header2(msg, &txn->hdr_idx, &ctx);
356 }
357 }
358 return 1;
359 }
360
361 /* identity is implicit does not require headers */
Christopher Faulet92d36382015-11-05 13:35:03 +0100362 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
363 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100364 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
365 if (comp_algo->cfg_name_len == 8 && memcmp(comp_algo->cfg_name, "identity", 8) == 0) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100366 st->comp_algo = comp_algo;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100367 return 1;
368 }
369 }
370 }
371
Christopher Faulet92d36382015-11-05 13:35:03 +0100372 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100373 return 0;
374}
375
Christopher Faulet92d36382015-11-05 13:35:03 +0100376
Christopher Faulet3d97c902015-12-09 14:59:38 +0100377/*
378 * Selects a comression algorithm depending of the server response.
379 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100380static int
381select_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100382{
383 struct http_txn *txn = s->txn;
Christopher Faulet92d36382015-11-05 13:35:03 +0100384 struct buffer *res = msg->chn->buf;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100385 struct hdr_ctx ctx;
386 struct comp_type *comp_type;
387
388 /* no common compression algorithm was found in request header */
Christopher Faulet92d36382015-11-05 13:35:03 +0100389 if (st->comp_algo == NULL)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100390 goto fail;
391
392 /* HTTP < 1.1 should not be compressed */
393 if (!(msg->flags & HTTP_MSGF_VER_11) || !(txn->req.flags & HTTP_MSGF_VER_11))
394 goto fail;
395
Christopher Faulet92d36382015-11-05 13:35:03 +0100396 if (txn->meth == HTTP_METH_HEAD)
397 goto fail;
398
Christopher Faulet3d97c902015-12-09 14:59:38 +0100399 /* compress 200,201,202,203 responses only */
400 if ((txn->status != 200) &&
401 (txn->status != 201) &&
402 (txn->status != 202) &&
403 (txn->status != 203))
404 goto fail;
405
406
407 /* Content-Length is null */
408 if (!(msg->flags & HTTP_MSGF_TE_CHNK) && msg->body_len == 0)
409 goto fail;
410
411 /* content is already compressed */
412 ctx.idx = 0;
413 if (http_find_header2("Content-Encoding", 16, res->p, &txn->hdr_idx, &ctx))
414 goto fail;
415
416 /* no compression when Cache-Control: no-transform is present in the message */
417 ctx.idx = 0;
418 while (http_find_header2("Cache-Control", 13, res->p, &txn->hdr_idx, &ctx)) {
419 if (word_match(ctx.line + ctx.val, ctx.vlen, "no-transform", 12))
420 goto fail;
421 }
422
423 comp_type = NULL;
424
425 /* we don't want to compress multipart content-types, nor content-types that are
426 * not listed in the "compression type" directive if any. If no content-type was
427 * found but configuration requires one, we don't compress either. Backend has
428 * the priority.
429 */
430 ctx.idx = 0;
431 if (http_find_header2("Content-Type", 12, res->p, &txn->hdr_idx, &ctx)) {
432 if (ctx.vlen >= 9 && strncasecmp("multipart", ctx.line+ctx.val, 9) == 0)
433 goto fail;
434
435 if ((s->be->comp && (comp_type = s->be->comp->types)) ||
436 (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types))) {
437 for (; comp_type; comp_type = comp_type->next) {
438 if (ctx.vlen >= comp_type->name_len &&
439 strncasecmp(ctx.line+ctx.val, comp_type->name, comp_type->name_len) == 0)
440 /* this Content-Type should be compressed */
441 break;
442 }
443 /* this Content-Type should not be compressed */
444 if (comp_type == NULL)
445 goto fail;
446 }
447 }
448 else { /* no content-type header */
Christopher Faulet92d36382015-11-05 13:35:03 +0100449 if ((s->be->comp && s->be->comp->types) ||
450 (strm_fe(s)->comp && strm_fe(s)->comp->types))
Christopher Faulet3d97c902015-12-09 14:59:38 +0100451 goto fail; /* a content-type was required */
452 }
453
454 /* limit compression rate */
455 if (global.comp_rate_lim > 0)
456 if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim)
457 goto fail;
458
459 /* limit cpu usage */
460 if (idle_pct < compress_min_idle)
461 goto fail;
462
463 /* initialize compression */
Christopher Faulet92d36382015-11-05 13:35:03 +0100464 if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100465 goto fail;
466
Christopher Faulet3d97c902015-12-09 14:59:38 +0100467 /* remove Content-Length header */
468 ctx.idx = 0;
469 if ((msg->flags & HTTP_MSGF_CNT_LEN) && http_find_header2("Content-Length", 14, res->p, &txn->hdr_idx, &ctx))
470 http_remove_header2(msg, &txn->hdr_idx, &ctx);
471
472 /* add Transfer-Encoding header */
473 if (!(msg->flags & HTTP_MSGF_TE_CHNK))
474 http_header_add_tail2(&txn->rsp, &txn->hdr_idx, "Transfer-Encoding: chunked", 26);
475
476 /*
477 * Add Content-Encoding header when it's not identity encoding.
478 * RFC 2616 : Identity encoding: This content-coding is used only in the
479 * Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding
480 * header.
481 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100482 if (st->comp_algo->cfg_name_len != 8 || memcmp(st->comp_algo->cfg_name, "identity", 8) != 0) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100483 trash.len = 18;
484 memcpy(trash.str, "Content-Encoding: ", trash.len);
Christopher Faulet92d36382015-11-05 13:35:03 +0100485 memcpy(trash.str + trash.len, st->comp_algo->ua_name, st->comp_algo->ua_name_len);
486 trash.len += st->comp_algo->ua_name_len;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100487 trash.str[trash.len] = '\0';
488 http_header_add_tail2(&txn->rsp, &txn->hdr_idx, trash.str, trash.len);
489 }
Christopher Faulet92d36382015-11-05 13:35:03 +0100490 msg->flags |= HTTP_MSGF_COMPRESSING;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100491 return 1;
492
493fail:
Christopher Faulet92d36382015-11-05 13:35:03 +0100494 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100495 return 0;
496}
497
498/***********************************************************************/
499/* emit the chunksize followed by a CRLF on the output and return the number of
500 * bytes written. It goes backwards and starts with the byte before <end>. It
501 * returns the number of bytes written which will not exceed 10 (8 digits, CR,
502 * and LF). The caller is responsible for ensuring there is enough room left in
503 * the output buffer for the string.
504 */
505static int
506http_emit_chunk_size(char *end, unsigned int chksz)
507{
508 char *beg = end;
509
510 *--beg = '\n';
511 *--beg = '\r';
512 do {
513 *--beg = hextab[chksz & 0xF];
514 } while (chksz >>= 4);
515 return end - beg;
516}
517
518/*
519 * Init HTTP compression
520 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100521static int
522http_compression_buffer_init(struct buffer *in, struct buffer *out)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100523{
524 /* output stream requires at least 10 bytes for the gzip header, plus
525 * at least 8 bytes for the gzip trailer (crc+len), plus a possible
526 * plus at most 5 bytes per 32kB block and 2 bytes to close the stream.
527 */
528 if (in->size - buffer_len(in) < 20 + 5 * ((in->i + 32767) >> 15))
529 return -1;
530
531 /* prepare an empty output buffer in which we reserve enough room for
532 * copying the output bytes from <in>, plus 10 extra bytes to write
533 * the chunk size. We don't copy the bytes yet so that if we have to
534 * cancel the operation later, it's cheap.
535 */
536 b_reset(out);
537 out->o = in->o;
538 out->p += out->o;
539 out->i = 10;
540 return 0;
541}
542
543/*
544 * Add data to compress
545 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100546static int
547http_compression_buffer_add_data(struct comp_state *st, struct buffer *in,
548 struct buffer *out, int sz)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100549{
Christopher Faulet3d97c902015-12-09 14:59:38 +0100550 int consumed_data = 0;
551 int data_process_len;
552 int block1, block2;
553
Christopher Faulet92d36382015-11-05 13:35:03 +0100554 if (!sz)
555 return 0;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100556
Christopher Faulet92d36382015-11-05 13:35:03 +0100557 /* select the smallest size between the announced chunk size, the input
Christopher Faulet3d97c902015-12-09 14:59:38 +0100558 * data, and the available output buffer size. The compressors are
Christopher Faulet92d36382015-11-05 13:35:03 +0100559 * assumed to be able to process all the bytes we pass to them at
560 * once. */
561 data_process_len = sz;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100562 data_process_len = MIN(out->size - buffer_len(out), data_process_len);
563
Christopher Faulet92d36382015-11-05 13:35:03 +0100564
Christopher Faulet3d97c902015-12-09 14:59:38 +0100565 block1 = data_process_len;
566 if (block1 > bi_contig_data(in))
567 block1 = bi_contig_data(in);
568 block2 = data_process_len - block1;
569
570 /* compressors return < 0 upon error or the amount of bytes read */
Christopher Faulet92d36382015-11-05 13:35:03 +0100571 consumed_data = st->comp_algo->add_data(st->comp_ctx, bi_ptr(in), block1, out);
Christopher Faulet3d97c902015-12-09 14:59:38 +0100572 if (consumed_data >= 0 && block2 > 0) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100573 consumed_data = st->comp_algo->add_data(st->comp_ctx, in->data, block2, out);
Christopher Faulet3d97c902015-12-09 14:59:38 +0100574 if (consumed_data >= 0)
575 consumed_data += block1;
576 }
Christopher Faulet3d97c902015-12-09 14:59:38 +0100577 return consumed_data;
578}
579
580/*
581 * Flush data in process, and write the header and footer of the chunk. Upon
582 * success, in and out buffers are swapped to avoid a copy.
583 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100584static int
585http_compression_buffer_end(struct comp_state *st, struct stream *s,
586 struct buffer **in, struct buffer **out,
Christopher Faulet2fb28802015-12-01 10:40:57 +0100587 int end)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100588{
Christopher Faulet3d97c902015-12-09 14:59:38 +0100589 struct buffer *ib = *in, *ob = *out;
590 char *tail;
Christopher Faulet92d36382015-11-05 13:35:03 +0100591 int to_forward, left;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100592
593#if defined(USE_SLZ) || defined(USE_ZLIB)
594 int ret;
595
596 /* flush data here */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100597 if (end)
Christopher Faulet92d36382015-11-05 13:35:03 +0100598 ret = st->comp_algo->finish(st->comp_ctx, ob); /* end of data */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100599 else
Christopher Faulet92d36382015-11-05 13:35:03 +0100600 ret = st->comp_algo->flush(st->comp_ctx, ob); /* end of buffer */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100601
602 if (ret < 0)
603 return -1; /* flush failed */
604
605#endif /* USE_ZLIB */
606
607 if (ob->i == 10) {
608 /* No data were appended, let's drop the output buffer and
609 * keep the input buffer unchanged.
610 */
611 return 0;
612 }
613
614 /* OK so at this stage, we have an output buffer <ob> looking like this :
615 *
616 * <-- o --> <------ i ----->
617 * +---------+---+------------+-----------+
618 * | out | c | comp_in | empty |
619 * +---------+---+------------+-----------+
620 * data p size
621 *
622 * <out> is the room reserved to copy ib->o. It starts at ob->data and
623 * has not yet been filled. <c> is the room reserved to write the chunk
624 * size (10 bytes). <comp_in> is the compressed equivalent of the data
625 * part of ib->i. <empty> is the amount of empty bytes at the end of
626 * the buffer, into which we may have to copy the remaining bytes from
627 * ib->i after the data (chunk size, trailers, ...).
628 */
629
630 /* Write real size at the begining of the chunk, no need of wrapping.
631 * We write the chunk using a dynamic length and adjust ob->p and ob->i
632 * accordingly afterwards. That will move <out> away from <data>.
633 */
634 left = 10 - http_emit_chunk_size(ob->p + 10, ob->i - 10);
635 ob->p += left;
636 ob->i -= left;
637
638 /* Copy previous data from ib->o into ob->o */
639 if (ib->o > 0) {
640 left = bo_contig_data(ib);
641 memcpy(ob->p - ob->o, bo_ptr(ib), left);
642 if (ib->o - left) /* second part of the buffer */
643 memcpy(ob->p - ob->o + left, ib->data, ib->o - left);
644 }
645
646 /* chunked encoding requires CRLF after data */
647 tail = ob->p + ob->i;
648 *tail++ = '\r';
649 *tail++ = '\n';
650
Christopher Faulet2fb28802015-12-01 10:40:57 +0100651 /* At the end of data, we must write the empty chunk 0<CRLF>,
652 * and terminate the trailers section with a last <CRLF>. If
653 * we're forwarding a chunked-encoded response, we'll have a
654 * trailers section after the empty chunk which needs to be
655 * forwarded and which will provide the last CRLF. Otherwise
656 * we write it ourselves.
657 */
658 if (end) {
659 struct http_msg *msg = &s->txn->rsp;
660
661 memcpy(tail, "0\r\n", 3);
662 tail += 3;
663 if (msg->msg_state == HTTP_MSG_ENDING) {
664 memcpy(tail, "\r\n", 2);
665 tail += 2;
666 }
667 }
668
Christopher Faulet3d97c902015-12-09 14:59:38 +0100669 ob->i = tail - ob->p;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100670 to_forward = ob->i;
671
672 /* update input rate */
Christopher Faulet92d36382015-11-05 13:35:03 +0100673 if (st->comp_ctx && st->comp_ctx->cur_lvl > 0) {
Christopher Faulet2fb28802015-12-01 10:40:57 +0100674 update_freq_ctr(&global.comp_bps_in, st->consumed);
675 strm_fe(s)->fe_counters.comp_in += st->consumed;
676 s->be->be_counters.comp_in += st->consumed;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100677 } else {
Christopher Faulet2fb28802015-12-01 10:40:57 +0100678 strm_fe(s)->fe_counters.comp_byp += st->consumed;
679 s->be->be_counters.comp_byp += st->consumed;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100680 }
681
682 /* copy the remaining data in the tmp buffer. */
Christopher Faulet2fb28802015-12-01 10:40:57 +0100683 b_adv(ib, st->consumed);
Christopher Faulet3d97c902015-12-09 14:59:38 +0100684 if (ib->i > 0) {
685 left = bi_contig_data(ib);
686 memcpy(ob->p + ob->i, bi_ptr(ib), left);
687 ob->i += left;
688 if (ib->i - left) {
689 memcpy(ob->p + ob->i, ib->data, ib->i - left);
690 ob->i += ib->i - left;
691 }
692 }
693
694 /* swap the buffers */
695 *in = ob;
696 *out = ib;
697
Christopher Faulet92d36382015-11-05 13:35:03 +0100698
699 if (st->comp_ctx && st->comp_ctx->cur_lvl > 0) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100700 update_freq_ctr(&global.comp_bps_out, to_forward);
701 strm_fe(s)->fe_counters.comp_out += to_forward;
702 s->be->be_counters.comp_out += to_forward;
703 }
704
Christopher Faulet3d97c902015-12-09 14:59:38 +0100705 return to_forward;
706}
707
708
709/***********************************************************************/
Christopher Faulet92d36382015-11-05 13:35:03 +0100710struct flt_ops comp_ops = {
711 .init = comp_flt_init,
712 .deinit = comp_flt_deinit,
713
714 .channel_start_analyze = comp_start_analyze,
715 .channel_analyze = comp_analyze,
716 .channel_end_analyze = comp_end_analyze,
717
Christopher Faulet309c6412015-12-02 09:57:32 +0100718 .http_data = comp_http_data,
719 .http_chunk_trailers = comp_http_chunk_trailers,
720 .http_forward_data = comp_http_forward_data,
Christopher Faulet92d36382015-11-05 13:35:03 +0100721};
722
Christopher Faulet3d97c902015-12-09 14:59:38 +0100723static int
724parse_compression_options(char **args, int section, struct proxy *proxy,
725 struct proxy *defpx, const char *file, int line,
726 char **err)
727{
Christopher Faulet92d36382015-11-05 13:35:03 +0100728 struct comp *comp;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100729
730 if (proxy->comp == NULL) {
731 comp = calloc(1, sizeof(struct comp));
732 proxy->comp = comp;
733 }
734 else
735 comp = proxy->comp;
736
737 if (!strcmp(args[1], "algo")) {
738 struct comp_ctx *ctx;
739 int cur_arg = 2;
740
741 if (!*args[cur_arg]) {
742 memprintf(err, "parsing [%s:%d] : '%s' expects <algorithm>\n",
743 file, line, args[0]);
744 return -1;
745 }
746 while (*(args[cur_arg])) {
747 if (comp_append_algo(comp, args[cur_arg]) < 0) {
748 memprintf(err, "'%s' : '%s' is not a supported algorithm.\n",
749 args[0], args[cur_arg]);
750 return -1;
751 }
752 if (proxy->comp->algos->init(&ctx, 9) == 0)
753 proxy->comp->algos->end(&ctx);
754 else {
755 memprintf(err, "'%s' : Can't init '%s' algorithm.\n",
756 args[0], args[cur_arg]);
757 return -1;
758 }
759 cur_arg++;
760 continue;
761 }
762 }
763 else if (!strcmp(args[1], "offload"))
764 comp->offload = 1;
765 else if (!strcmp(args[1], "type")) {
766 int cur_arg = 2;
767
768 if (!*args[cur_arg]) {
769 memprintf(err, "'%s' expects <type>\n", args[0]);
770 return -1;
771 }
772 while (*(args[cur_arg])) {
773 comp_append_type(comp, args[cur_arg]);
774 cur_arg++;
775 continue;
776 }
777 }
778 else {
779 memprintf(err, "'%s' expects 'algo', 'type' or 'offload'\n",
780 args[0]);
781 return -1;
782 }
783
784 return 0;
785}
786
Christopher Faulet92d36382015-11-05 13:35:03 +0100787static int
788parse_http_comp_flt(char **args, int *cur_arg, struct proxy *px,
789 struct filter *filter, char **err)
790{
791 struct filter *flt, *back;
792
793 list_for_each_entry_safe(flt, back, &px->filters, list) {
794 if (flt->id == http_comp_flt_id) {
795 memprintf(err, "%s: Proxy supports only one compression filter\n", px->id);
796 return -1;
797 }
798 }
799
800 filter->id = http_comp_flt_id;
801 filter->conf = NULL;
802 filter->ops = &comp_ops;
803 (*cur_arg)++;
804
805 return 0;
806}
807
808
809int
810check_legacy_http_comp_flt(struct proxy *proxy)
811{
812 struct filter *filter;
813 int err = 0;
814
815 if (proxy->comp == NULL)
816 goto end;
817 if (!LIST_ISEMPTY(&proxy->filters)) {
818 list_for_each_entry(filter, &proxy->filters, list) {
819 if (filter->id == http_comp_flt_id)
820 goto end;
821 }
822 Alert("config: %s '%s': require an explicit filter declaration to use HTTP compression\n",
823 proxy_type_str(proxy), proxy->id);
824 err++;
825 goto end;
826 }
827
828 filter = pool_alloc2(pool2_filter);
829 if (!filter) {
830 Alert("config: %s '%s': out of memory\n",
831 proxy_type_str(proxy), proxy->id);
832 err++;
833 goto end;
834 }
835 memset(filter, 0, sizeof(*filter));
836 filter->id = http_comp_flt_id;
837 filter->conf = NULL;
838 filter->ops = &comp_ops;
839 LIST_ADDQ(&proxy->filters, &filter->list);
840
841 end:
842 return err;
843}
844
845/*
846 * boolean, returns true if compression is used (either gzip or deflate) in the
847 * response.
848 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100849static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100850smp_fetch_res_comp(const struct arg *args, struct sample *smp, const char *kw,
851 void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100852{
Christopher Faulet92d36382015-11-05 13:35:03 +0100853 struct http_txn *txn = smp->strm->txn;
854
Christopher Faulet3d97c902015-12-09 14:59:38 +0100855 smp->data.type = SMP_T_BOOL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100856 smp->data.u.sint = (txn && (txn->rsp.flags & HTTP_MSGF_COMPRESSING));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100857 return 1;
858}
859
Christopher Faulet92d36382015-11-05 13:35:03 +0100860/*
861 * string, returns algo
862 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100863static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100864smp_fetch_res_comp_algo(const struct arg *args, struct sample *smp,
865 const char *kw, void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100866{
Christopher Faulet92d36382015-11-05 13:35:03 +0100867 struct http_txn *txn = smp->strm->txn;
868 struct filter *filter;
869 struct comp_state *st;
870
871 if (!(txn || !(txn->rsp.flags & HTTP_MSGF_COMPRESSING)))
Christopher Faulet3d97c902015-12-09 14:59:38 +0100872 return 0;
873
Christopher Faulet92d36382015-11-05 13:35:03 +0100874 list_for_each_entry(filter, &smp->strm->strm_flt.filters, list) {
875 if (filter->id != http_comp_flt_id)
876 continue;
877
878 if (!(st = filter->ctx))
879 break;
880
881 smp->data.type = SMP_T_STR;
882 smp->flags = SMP_F_CONST;
883 smp->data.u.str.str = st->comp_algo->cfg_name;
884 smp->data.u.str.len = st->comp_algo->cfg_name_len;
885 return 1;
886 }
887 return 0;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100888}
889
890/* Declare the config parser for "compression" keyword */
891static struct cfg_kw_list cfg_kws = {ILH, {
892 { CFG_LISTEN, "compression", parse_compression_options },
893 { 0, NULL, NULL },
894 }
895};
896
Christopher Faulet92d36382015-11-05 13:35:03 +0100897/* Declare the filter parser for "compression" keyword */
898static struct flt_kw_list filter_kws = { "COMP", { }, {
899 { "compression", parse_http_comp_flt },
900 { NULL, NULL },
901 }
902};
903
Christopher Faulet3d97c902015-12-09 14:59:38 +0100904/* Note: must not be declared <const> as its list will be overwritten */
905static struct sample_fetch_kw_list sample_fetch_keywords = {ILH, {
Christopher Faulet92d36382015-11-05 13:35:03 +0100906 { "res.comp", smp_fetch_res_comp, 0, NULL, SMP_T_BOOL, SMP_USE_HRSHP },
907 { "res.comp_algo", smp_fetch_res_comp_algo, 0, NULL, SMP_T_STR, SMP_USE_HRSHP },
908 { /* END */ },
909 }
910};
Christopher Faulet3d97c902015-12-09 14:59:38 +0100911
912__attribute__((constructor))
Christopher Faulet92d36382015-11-05 13:35:03 +0100913static void
914__flt_http_comp_init(void)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100915{
916 cfg_register_keywords(&cfg_kws);
Christopher Faulet92d36382015-11-05 13:35:03 +0100917 flt_register_keywords(&filter_kws);
Christopher Faulet3d97c902015-12-09 14:59:38 +0100918 sample_register_fetches(&sample_fetch_keywords);
919}