blob: 6bbf6a1a14bdf295cc80115150f8bc104e28f6e0 [file] [log] [blame]
Christopher Faulet3d97c902015-12-09 14:59:38 +01001/*
2 * Stream filters related variables and functions.
3 *
4 * Copyright (C) 2015 Qualys Inc., Christopher Faulet <cfaulet@qualys.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
13#include <common/buffer.h>
14#include <common/cfgparse.h>
15#include <common/mini-clist.h>
16#include <common/standard.h>
17
18#include <types/compression.h>
19#include <types/filters.h>
20#include <types/proto_http.h>
21#include <types/proxy.h>
22#include <types/sample.h>
23
24#include <proto/compression.h>
Christopher Faulet92d36382015-11-05 13:35:03 +010025#include <proto/filters.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010026#include <proto/hdr_idx.h>
27#include <proto/proto_http.h>
28#include <proto/sample.h>
29#include <proto/stream.h>
30
Christopher Faulet92d36382015-11-05 13:35:03 +010031static const char *http_comp_flt_id = "compression filter";
32
33struct flt_ops comp_ops;
34
35static struct buffer *tmpbuf = &buf_empty;
Christopher Fauletb77c5c22015-12-07 16:48:42 +010036static struct buffer *zbuf = &buf_empty;
Christopher Faulet92d36382015-11-05 13:35:03 +010037
Christopher Faulet92d36382015-11-05 13:35:03 +010038struct comp_state {
39 struct comp_ctx *comp_ctx; /* compression context */
40 struct comp_algo *comp_algo; /* compression algorithm if not NULL */
Christopher Fauletb77c5c22015-12-07 16:48:42 +010041 int hdrs_len;
42 int tlrs_len;
Christopher Faulet2fb28802015-12-01 10:40:57 +010043 int consumed;
44 int initialized;
Christopher Fauletb77c5c22015-12-07 16:48:42 +010045 int finished;
Christopher Faulet92d36382015-11-05 13:35:03 +010046};
47
Christopher Faulet92d36382015-11-05 13:35:03 +010048static int select_compression_request_header(struct comp_state *st,
49 struct stream *s,
50 struct http_msg *msg);
51static int select_compression_response_header(struct comp_state *st,
52 struct stream *s,
53 struct http_msg *msg);
54
55static int http_compression_buffer_init(struct buffer *in, struct buffer *out);
56static int http_compression_buffer_add_data(struct comp_state *st,
57 struct buffer *in,
58 struct buffer *out, int sz);
59static int http_compression_buffer_end(struct comp_state *st, struct stream *s,
60 struct buffer **in, struct buffer **out,
Christopher Faulet2fb28802015-12-01 10:40:57 +010061 int end);
Christopher Faulet92d36382015-11-05 13:35:03 +010062
63/***********************************************************************/
64static int
Christopher Faulet443ea1a2016-02-04 13:40:26 +010065comp_flt_init(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010066{
67
Christopher Fauletb77c5c22015-12-07 16:48:42 +010068 if (!tmpbuf->size && b_alloc(&tmpbuf) == NULL)
69 return -1;
70 if (!zbuf->size && b_alloc(&zbuf) == NULL)
71 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +010072 return 0;
73}
74
75static void
Christopher Faulet443ea1a2016-02-04 13:40:26 +010076comp_flt_deinit(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010077{
78 if (tmpbuf->size)
79 b_free(&tmpbuf);
Christopher Fauletb77c5c22015-12-07 16:48:42 +010080 if (zbuf->size)
81 b_free(&zbuf);
Christopher Faulet92d36382015-11-05 13:35:03 +010082}
83
84static int
85comp_start_analyze(struct stream *s, struct filter *filter, struct channel *chn)
86{
87 if (filter->ctx == NULL) {
88 struct comp_state *st;
89
90 if (!(st = malloc(sizeof(*st))))
91 return -1;
92
Christopher Faulet2fb28802015-12-01 10:40:57 +010093 st->comp_algo = NULL;
94 st->comp_ctx = NULL;
Christopher Fauletb77c5c22015-12-07 16:48:42 +010095 st->hdrs_len = 0;
96 st->tlrs_len = 0;
Christopher Faulet2fb28802015-12-01 10:40:57 +010097 st->consumed = 0;
98 st->initialized = 0;
Christopher Fauletb77c5c22015-12-07 16:48:42 +010099 st->finished = 0;
Christopher Faulet2fb28802015-12-01 10:40:57 +0100100 filter->ctx = st;
Christopher Faulet92d36382015-11-05 13:35:03 +0100101 }
102 return 1;
103}
104
105static int
106comp_analyze(struct stream *s, struct filter *filter, struct channel *chn,
107 unsigned int an_bit)
108{
109 struct comp_state *st = filter->ctx;
110
111 if (!strm_fe(s)->comp && !s->be->comp)
112 goto end;
113
Christopher Faulet309c6412015-12-02 09:57:32 +0100114 if (an_bit == AN_FLT_HTTP_HDRS) {
115 if (!(chn->flags & CF_ISRESP))
116 select_compression_request_header(st, s, &s->txn->req);
117 else {
Christopher Faulet92d36382015-11-05 13:35:03 +0100118 select_compression_response_header(st, s, &s->txn->rsp);
Christopher Fauletda02e172015-12-04 09:25:05 +0100119 if (st->comp_algo) {
Christopher Fauletda02e172015-12-04 09:25:05 +0100120 register_data_filter(s, chn, filter);
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100121 st->hdrs_len = s->txn->rsp.sov;
Christopher Fauletda02e172015-12-04 09:25:05 +0100122 }
Christopher Faulet309c6412015-12-02 09:57:32 +0100123 }
Christopher Faulet92d36382015-11-05 13:35:03 +0100124 }
Christopher Faulet309c6412015-12-02 09:57:32 +0100125
Christopher Faulet92d36382015-11-05 13:35:03 +0100126 end:
127 return 1;
128}
129
130static int
131comp_end_analyze(struct stream *s, struct filter *filter, struct channel *chn)
132{
133 struct comp_state *st = filter->ctx;
Christopher Faulet92d36382015-11-05 13:35:03 +0100134
135 if (!st || !(chn->flags & CF_ISRESP))
136 goto end;
137
Christopher Faulet92d36382015-11-05 13:35:03 +0100138 if (!st->comp_algo || !s->txn->status)
139 goto release_ctx;
140
141 if (strm_fe(s)->mode == PR_MODE_HTTP)
142 strm_fe(s)->fe_counters.p.http.comp_rsp++;
143 if ((s->flags & SF_BE_ASSIGNED) && (s->be->mode == PR_MODE_HTTP))
144 s->be->be_counters.p.http.comp_rsp++;
145
146 /* release any possible compression context */
147 st->comp_algo->end(&st->comp_ctx);
148
149 release_ctx:
150 free(st);
151 filter->ctx = NULL;
152 end:
153 return 1;
154}
155
156static int
Christopher Faulet2fb28802015-12-01 10:40:57 +0100157comp_http_data(struct stream *s, struct filter *filter, struct http_msg *msg)
Christopher Faulet92d36382015-11-05 13:35:03 +0100158{
159 struct comp_state *st = filter->ctx;
Christopher Faulet3e7bc672015-12-07 13:39:08 +0100160 struct buffer *buf = msg->chn->buf;
161 unsigned int *nxt = &flt_rsp_nxt(filter);
Christopher Faulet2fb28802015-12-01 10:40:57 +0100162 unsigned int len;
Christopher Faulet92d36382015-11-05 13:35:03 +0100163 int ret;
164
Christopher Faulet3e7bc672015-12-07 13:39:08 +0100165 len = MIN(msg->chunk_len + msg->next, buf->i) - *nxt;
Christopher Faulet2fb28802015-12-01 10:40:57 +0100166 if (!len)
167 return len;
168
169 if (!st->initialized) {
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100170 unsigned int fwd = flt_rsp_fwd(filter) + st->hdrs_len;
Christopher Faulet3e7bc672015-12-07 13:39:08 +0100171
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100172 b_reset(tmpbuf);
Christopher Faulet3e7bc672015-12-07 13:39:08 +0100173 b_adv(buf, fwd);
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100174 ret = http_compression_buffer_init(buf, zbuf);
Christopher Faulet3e7bc672015-12-07 13:39:08 +0100175 b_rew(buf, fwd);
Christopher Faulet2fb28802015-12-01 10:40:57 +0100176 if (ret < 0) {
177 msg->chn->flags |= CF_WAKE_WRITE;
178 return 0;
179 }
180 }
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100181
182 if (msg->flags & HTTP_MSGF_TE_CHNK) {
183 int block = bi_contig_data(buf);
184
185 len = MIN(tmpbuf->size - buffer_len(tmpbuf), len);
186 if (len > block) {
187 memcpy(bi_end(tmpbuf), b_ptr(buf, *nxt), block);
188 memcpy(bi_end(tmpbuf)+block, buf->data, len - block);
189 }
190 else
191 memcpy(bi_end(tmpbuf), b_ptr(buf, *nxt), len);
192 tmpbuf->i += len;
193 ret = len;
194 }
195 else {
196 b_adv(buf, *nxt);
197 ret = http_compression_buffer_add_data(st, buf, zbuf, len);
198 b_rew(buf, *nxt);
199 if (ret < 0)
200 return ret;
201 }
Christopher Faulet92d36382015-11-05 13:35:03 +0100202
Christopher Faulet2fb28802015-12-01 10:40:57 +0100203 st->initialized = 1;
204 msg->next += ret;
205 msg->chunk_len -= ret;
Christopher Faulet3e7bc672015-12-07 13:39:08 +0100206 *nxt = msg->next;
Christopher Faulet2fb28802015-12-01 10:40:57 +0100207 return 0;
Christopher Faulet92d36382015-11-05 13:35:03 +0100208}
209
210static int
Christopher Faulet2fb28802015-12-01 10:40:57 +0100211comp_http_chunk_trailers(struct stream *s, struct filter *filter,
212 struct http_msg *msg)
Christopher Faulet92d36382015-11-05 13:35:03 +0100213{
214 struct comp_state *st = filter->ctx;
Christopher Faulet92d36382015-11-05 13:35:03 +0100215
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100216 if (!st->initialized) {
217 if (!st->finished) {
218 struct buffer *buf = msg->chn->buf;
219 unsigned int fwd = flt_rsp_fwd(filter) + st->hdrs_len;
Christopher Faulet2fb28802015-12-01 10:40:57 +0100220
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100221 b_reset(tmpbuf);
222 b_adv(buf, fwd);
223 http_compression_buffer_init(buf, zbuf);
224 b_rew(buf, fwd);
225 st->initialized = 1;
226 }
227 }
228 st->tlrs_len = msg->sol;
Christopher Faulet2fb28802015-12-01 10:40:57 +0100229 return 1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100230}
231
Christopher Faulet2fb28802015-12-01 10:40:57 +0100232
Christopher Faulet92d36382015-11-05 13:35:03 +0100233static int
234comp_http_forward_data(struct stream *s, struct filter *filter,
235 struct http_msg *msg, unsigned int len)
236{
237 struct comp_state *st = filter->ctx;
Christopher Faulet2fb28802015-12-01 10:40:57 +0100238 int ret;
Christopher Faulet92d36382015-11-05 13:35:03 +0100239
Christopher Faulet2fb28802015-12-01 10:40:57 +0100240 /* To work, previous filters MUST forward all data */
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100241 if (flt_rsp_fwd(filter) + len != flt_rsp_nxt(filter)) {
Christopher Faulet2fb28802015-12-01 10:40:57 +0100242 Warning("HTTP compression failed: unexpected behavior of previous filters\n");
243 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100244 }
245
Christopher Faulet2fb28802015-12-01 10:40:57 +0100246 if (!st->initialized) {
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100247 if (!len) {
248 /* Nothing to foward */
249 ret = len;
250 }
251 else if (st->hdrs_len > len) {
252 /* Forward part of headers */
253 ret = len;
254 st->hdrs_len -= len;
255 }
256 else if (st->hdrs_len > 0) {
257 /* Forward remaining headers */
258 ret = st->hdrs_len;
259 st->hdrs_len = 0;
260 }
261 else if (msg->msg_state < HTTP_MSG_TRAILERS) {
262 /* Do not forward anything for now. This only happens
263 * with chunk-encoded responses. Waiting data are part
264 * of the chunk envelope (the chunk size or the chunk
265 * CRLF). These data will be skipped during the
266 * compression. */
267 ret = 0;
268 }
269 else {
270 /* Forward trailers data */
271 ret = len;
272 }
Christopher Faulet2fb28802015-12-01 10:40:57 +0100273 return ret;
Christopher Faulet92d36382015-11-05 13:35:03 +0100274 }
275
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100276 if (msg->flags & HTTP_MSGF_TE_CHNK) {
277 ret = http_compression_buffer_add_data(st, tmpbuf, zbuf, tmpbuf->i);
278 if (ret != tmpbuf->i) {
279 Warning("HTTP compression failed: Must consume %d bytes but only %d bytes consumed\n",
280 tmpbuf->i, ret);
281 return -1;
282 }
283 }
284
285 st->consumed = len - st->hdrs_len - st->tlrs_len;
286 b_adv(msg->chn->buf, flt_rsp_fwd(filter) + st->hdrs_len);
287 ret = http_compression_buffer_end(st, s, &msg->chn->buf, &zbuf, msg->msg_state >= HTTP_MSG_TRAILERS);
288 b_rew(msg->chn->buf, flt_rsp_fwd(filter) + st->hdrs_len);
Christopher Faulet2fb28802015-12-01 10:40:57 +0100289 if (ret < 0)
290 return ret;
Christopher Faulet92d36382015-11-05 13:35:03 +0100291
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100292 flt_change_forward_size(filter, msg->chn, ret - st->consumed);
293 msg->next += (ret - st->consumed);
294 ret += st->hdrs_len + st->tlrs_len;
295
Christopher Faulet2fb28802015-12-01 10:40:57 +0100296 st->initialized = 0;
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100297 st->finished = (msg->msg_state >= HTTP_MSG_TRAILERS);
298 st->hdrs_len = 0;
299 st->tlrs_len = 0;
Christopher Faulet92d36382015-11-05 13:35:03 +0100300 return ret;
301}
Christopher Faulet3d97c902015-12-09 14:59:38 +0100302
303/***********************************************************************/
304/*
305 * Selects a compression algorithm depending on the client request.
306 */
307int
Christopher Faulet92d36382015-11-05 13:35:03 +0100308select_compression_request_header(struct comp_state *st, struct stream *s,
309 struct http_msg *msg)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100310{
311 struct http_txn *txn = s->txn;
Christopher Faulet92d36382015-11-05 13:35:03 +0100312 struct buffer *req = msg->chn->buf;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100313 struct hdr_ctx ctx;
314 struct comp_algo *comp_algo = NULL;
315 struct comp_algo *comp_algo_back = NULL;
316
317 /* Disable compression for older user agents announcing themselves as "Mozilla/4"
318 * unless they are known good (MSIE 6 with XP SP2, or MSIE 7 and later).
319 * See http://zoompf.com/2012/02/lose-the-wait-http-compression for more details.
320 */
321 ctx.idx = 0;
322 if (http_find_header2("User-Agent", 10, req->p, &txn->hdr_idx, &ctx) &&
323 ctx.vlen >= 9 &&
324 memcmp(ctx.line + ctx.val, "Mozilla/4", 9) == 0 &&
325 (ctx.vlen < 31 ||
326 memcmp(ctx.line + ctx.val + 25, "MSIE ", 5) != 0 ||
327 ctx.line[ctx.val + 30] < '6' ||
328 (ctx.line[ctx.val + 30] == '6' &&
329 (ctx.vlen < 54 || memcmp(ctx.line + 51, "SV1", 3) != 0)))) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100330 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100331 return 0;
332 }
333
334 /* search for the algo in the backend in priority or the frontend */
Christopher Faulet92d36382015-11-05 13:35:03 +0100335 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
336 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100337 int best_q = 0;
338
339 ctx.idx = 0;
340 while (http_find_header2("Accept-Encoding", 15, req->p, &txn->hdr_idx, &ctx)) {
341 const char *qval;
342 int q;
343 int toklen;
344
345 /* try to isolate the token from the optional q-value */
346 toklen = 0;
347 while (toklen < ctx.vlen && http_is_token[(unsigned char)*(ctx.line + ctx.val + toklen)])
348 toklen++;
349
350 qval = ctx.line + ctx.val + toklen;
351 while (1) {
352 while (qval < ctx.line + ctx.val + ctx.vlen && http_is_lws[(unsigned char)*qval])
353 qval++;
354
355 if (qval >= ctx.line + ctx.val + ctx.vlen || *qval != ';') {
356 qval = NULL;
357 break;
358 }
359 qval++;
360
361 while (qval < ctx.line + ctx.val + ctx.vlen && http_is_lws[(unsigned char)*qval])
362 qval++;
363
364 if (qval >= ctx.line + ctx.val + ctx.vlen) {
365 qval = NULL;
366 break;
367 }
368 if (strncmp(qval, "q=", MIN(ctx.line + ctx.val + ctx.vlen - qval, 2)) == 0)
369 break;
370
371 while (qval < ctx.line + ctx.val + ctx.vlen && *qval != ';')
372 qval++;
373 }
374
375 /* here we have qval pointing to the first "q=" attribute or NULL if not found */
376 q = qval ? parse_qvalue(qval + 2, NULL) : 1000;
377
378 if (q <= best_q)
379 continue;
380
381 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
382 if (*(ctx.line + ctx.val) == '*' ||
383 word_match(ctx.line + ctx.val, toklen, comp_algo->ua_name, comp_algo->ua_name_len)) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100384 st->comp_algo = comp_algo;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100385 best_q = q;
386 break;
387 }
388 }
389 }
390 }
391
392 /* remove all occurrences of the header when "compression offload" is set */
Christopher Faulet92d36382015-11-05 13:35:03 +0100393 if (st->comp_algo) {
394 if ((s->be->comp && s->be->comp->offload) ||
395 (strm_fe(s)->comp && strm_fe(s)->comp->offload)) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100396 http_remove_header2(msg, &txn->hdr_idx, &ctx);
397 ctx.idx = 0;
398 while (http_find_header2("Accept-Encoding", 15, req->p, &txn->hdr_idx, &ctx)) {
399 http_remove_header2(msg, &txn->hdr_idx, &ctx);
400 }
401 }
402 return 1;
403 }
404
405 /* identity is implicit does not require headers */
Christopher Faulet92d36382015-11-05 13:35:03 +0100406 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
407 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100408 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
409 if (comp_algo->cfg_name_len == 8 && memcmp(comp_algo->cfg_name, "identity", 8) == 0) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100410 st->comp_algo = comp_algo;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100411 return 1;
412 }
413 }
414 }
415
Christopher Faulet92d36382015-11-05 13:35:03 +0100416 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100417 return 0;
418}
419
Christopher Faulet92d36382015-11-05 13:35:03 +0100420
Christopher Faulet3d97c902015-12-09 14:59:38 +0100421/*
422 * Selects a comression algorithm depending of the server response.
423 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100424static int
425select_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100426{
427 struct http_txn *txn = s->txn;
Christopher Faulet92d36382015-11-05 13:35:03 +0100428 struct buffer *res = msg->chn->buf;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100429 struct hdr_ctx ctx;
430 struct comp_type *comp_type;
431
432 /* no common compression algorithm was found in request header */
Christopher Faulet92d36382015-11-05 13:35:03 +0100433 if (st->comp_algo == NULL)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100434 goto fail;
435
436 /* HTTP < 1.1 should not be compressed */
437 if (!(msg->flags & HTTP_MSGF_VER_11) || !(txn->req.flags & HTTP_MSGF_VER_11))
438 goto fail;
439
Christopher Faulet92d36382015-11-05 13:35:03 +0100440 if (txn->meth == HTTP_METH_HEAD)
441 goto fail;
442
Christopher Faulet3d97c902015-12-09 14:59:38 +0100443 /* compress 200,201,202,203 responses only */
444 if ((txn->status != 200) &&
445 (txn->status != 201) &&
446 (txn->status != 202) &&
447 (txn->status != 203))
448 goto fail;
449
450
451 /* Content-Length is null */
452 if (!(msg->flags & HTTP_MSGF_TE_CHNK) && msg->body_len == 0)
453 goto fail;
454
455 /* content is already compressed */
456 ctx.idx = 0;
457 if (http_find_header2("Content-Encoding", 16, res->p, &txn->hdr_idx, &ctx))
458 goto fail;
459
460 /* no compression when Cache-Control: no-transform is present in the message */
461 ctx.idx = 0;
462 while (http_find_header2("Cache-Control", 13, res->p, &txn->hdr_idx, &ctx)) {
463 if (word_match(ctx.line + ctx.val, ctx.vlen, "no-transform", 12))
464 goto fail;
465 }
466
467 comp_type = NULL;
468
469 /* we don't want to compress multipart content-types, nor content-types that are
470 * not listed in the "compression type" directive if any. If no content-type was
471 * found but configuration requires one, we don't compress either. Backend has
472 * the priority.
473 */
474 ctx.idx = 0;
475 if (http_find_header2("Content-Type", 12, res->p, &txn->hdr_idx, &ctx)) {
476 if (ctx.vlen >= 9 && strncasecmp("multipart", ctx.line+ctx.val, 9) == 0)
477 goto fail;
478
479 if ((s->be->comp && (comp_type = s->be->comp->types)) ||
480 (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types))) {
481 for (; comp_type; comp_type = comp_type->next) {
482 if (ctx.vlen >= comp_type->name_len &&
483 strncasecmp(ctx.line+ctx.val, comp_type->name, comp_type->name_len) == 0)
484 /* this Content-Type should be compressed */
485 break;
486 }
487 /* this Content-Type should not be compressed */
488 if (comp_type == NULL)
489 goto fail;
490 }
491 }
492 else { /* no content-type header */
Christopher Faulet92d36382015-11-05 13:35:03 +0100493 if ((s->be->comp && s->be->comp->types) ||
494 (strm_fe(s)->comp && strm_fe(s)->comp->types))
Christopher Faulet3d97c902015-12-09 14:59:38 +0100495 goto fail; /* a content-type was required */
496 }
497
498 /* limit compression rate */
499 if (global.comp_rate_lim > 0)
500 if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim)
501 goto fail;
502
503 /* limit cpu usage */
504 if (idle_pct < compress_min_idle)
505 goto fail;
506
507 /* initialize compression */
Christopher Faulet92d36382015-11-05 13:35:03 +0100508 if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100509 goto fail;
510
Christopher Faulet3d97c902015-12-09 14:59:38 +0100511 /* remove Content-Length header */
512 ctx.idx = 0;
513 if ((msg->flags & HTTP_MSGF_CNT_LEN) && http_find_header2("Content-Length", 14, res->p, &txn->hdr_idx, &ctx))
514 http_remove_header2(msg, &txn->hdr_idx, &ctx);
515
516 /* add Transfer-Encoding header */
517 if (!(msg->flags & HTTP_MSGF_TE_CHNK))
518 http_header_add_tail2(&txn->rsp, &txn->hdr_idx, "Transfer-Encoding: chunked", 26);
519
520 /*
521 * Add Content-Encoding header when it's not identity encoding.
522 * RFC 2616 : Identity encoding: This content-coding is used only in the
523 * Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding
524 * header.
525 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100526 if (st->comp_algo->cfg_name_len != 8 || memcmp(st->comp_algo->cfg_name, "identity", 8) != 0) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100527 trash.len = 18;
528 memcpy(trash.str, "Content-Encoding: ", trash.len);
Christopher Faulet92d36382015-11-05 13:35:03 +0100529 memcpy(trash.str + trash.len, st->comp_algo->ua_name, st->comp_algo->ua_name_len);
530 trash.len += st->comp_algo->ua_name_len;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100531 trash.str[trash.len] = '\0';
532 http_header_add_tail2(&txn->rsp, &txn->hdr_idx, trash.str, trash.len);
533 }
Christopher Faulet92d36382015-11-05 13:35:03 +0100534 msg->flags |= HTTP_MSGF_COMPRESSING;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100535 return 1;
536
537fail:
Christopher Faulet92d36382015-11-05 13:35:03 +0100538 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100539 return 0;
540}
541
542/***********************************************************************/
543/* emit the chunksize followed by a CRLF on the output and return the number of
544 * bytes written. It goes backwards and starts with the byte before <end>. It
545 * returns the number of bytes written which will not exceed 10 (8 digits, CR,
546 * and LF). The caller is responsible for ensuring there is enough room left in
547 * the output buffer for the string.
548 */
549static int
550http_emit_chunk_size(char *end, unsigned int chksz)
551{
552 char *beg = end;
553
554 *--beg = '\n';
555 *--beg = '\r';
556 do {
557 *--beg = hextab[chksz & 0xF];
558 } while (chksz >>= 4);
559 return end - beg;
560}
561
562/*
563 * Init HTTP compression
564 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100565static int
566http_compression_buffer_init(struct buffer *in, struct buffer *out)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100567{
568 /* output stream requires at least 10 bytes for the gzip header, plus
569 * at least 8 bytes for the gzip trailer (crc+len), plus a possible
570 * plus at most 5 bytes per 32kB block and 2 bytes to close the stream.
571 */
572 if (in->size - buffer_len(in) < 20 + 5 * ((in->i + 32767) >> 15))
573 return -1;
574
575 /* prepare an empty output buffer in which we reserve enough room for
576 * copying the output bytes from <in>, plus 10 extra bytes to write
577 * the chunk size. We don't copy the bytes yet so that if we have to
578 * cancel the operation later, it's cheap.
579 */
580 b_reset(out);
581 out->o = in->o;
582 out->p += out->o;
583 out->i = 10;
584 return 0;
585}
586
587/*
588 * Add data to compress
589 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100590static int
591http_compression_buffer_add_data(struct comp_state *st, struct buffer *in,
592 struct buffer *out, int sz)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100593{
Christopher Faulet3d97c902015-12-09 14:59:38 +0100594 int consumed_data = 0;
595 int data_process_len;
596 int block1, block2;
597
Christopher Faulet92d36382015-11-05 13:35:03 +0100598 if (!sz)
Christopher Faulet3e7bc672015-12-07 13:39:08 +0100599 goto end;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100600
Christopher Faulet92d36382015-11-05 13:35:03 +0100601 /* select the smallest size between the announced chunk size, the input
Christopher Faulet3d97c902015-12-09 14:59:38 +0100602 * data, and the available output buffer size. The compressors are
Christopher Faulet92d36382015-11-05 13:35:03 +0100603 * assumed to be able to process all the bytes we pass to them at
604 * once. */
Christopher Faulet3e7bc672015-12-07 13:39:08 +0100605 data_process_len = MIN(out->size - buffer_len(out), sz);
Christopher Faulet92d36382015-11-05 13:35:03 +0100606
Christopher Faulet3d97c902015-12-09 14:59:38 +0100607 block1 = data_process_len;
608 if (block1 > bi_contig_data(in))
609 block1 = bi_contig_data(in);
610 block2 = data_process_len - block1;
611
612 /* compressors return < 0 upon error or the amount of bytes read */
Christopher Faulet92d36382015-11-05 13:35:03 +0100613 consumed_data = st->comp_algo->add_data(st->comp_ctx, bi_ptr(in), block1, out);
Christopher Faulet3e7bc672015-12-07 13:39:08 +0100614 if (consumed_data != block1 || !block2)
615 goto end;
616 consumed_data = st->comp_algo->add_data(st->comp_ctx, in->data, block2, out);
617 if (consumed_data < 0)
618 goto end;
619 consumed_data += block1;
620
621 end:
Christopher Faulet3d97c902015-12-09 14:59:38 +0100622 return consumed_data;
623}
624
625/*
626 * Flush data in process, and write the header and footer of the chunk. Upon
627 * success, in and out buffers are swapped to avoid a copy.
628 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100629static int
630http_compression_buffer_end(struct comp_state *st, struct stream *s,
631 struct buffer **in, struct buffer **out,
Christopher Faulet2fb28802015-12-01 10:40:57 +0100632 int end)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100633{
Christopher Faulet3d97c902015-12-09 14:59:38 +0100634 struct buffer *ib = *in, *ob = *out;
635 char *tail;
Christopher Faulet92d36382015-11-05 13:35:03 +0100636 int to_forward, left;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100637
638#if defined(USE_SLZ) || defined(USE_ZLIB)
639 int ret;
640
641 /* flush data here */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100642 if (end)
Christopher Faulet92d36382015-11-05 13:35:03 +0100643 ret = st->comp_algo->finish(st->comp_ctx, ob); /* end of data */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100644 else
Christopher Faulet92d36382015-11-05 13:35:03 +0100645 ret = st->comp_algo->flush(st->comp_ctx, ob); /* end of buffer */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100646
647 if (ret < 0)
648 return -1; /* flush failed */
649
650#endif /* USE_ZLIB */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100651 if (ob->i == 10) {
652 /* No data were appended, let's drop the output buffer and
653 * keep the input buffer unchanged.
654 */
655 return 0;
656 }
657
658 /* OK so at this stage, we have an output buffer <ob> looking like this :
659 *
660 * <-- o --> <------ i ----->
661 * +---------+---+------------+-----------+
662 * | out | c | comp_in | empty |
663 * +---------+---+------------+-----------+
664 * data p size
665 *
666 * <out> is the room reserved to copy ib->o. It starts at ob->data and
667 * has not yet been filled. <c> is the room reserved to write the chunk
668 * size (10 bytes). <comp_in> is the compressed equivalent of the data
669 * part of ib->i. <empty> is the amount of empty bytes at the end of
670 * the buffer, into which we may have to copy the remaining bytes from
671 * ib->i after the data (chunk size, trailers, ...).
672 */
673
674 /* Write real size at the begining of the chunk, no need of wrapping.
675 * We write the chunk using a dynamic length and adjust ob->p and ob->i
676 * accordingly afterwards. That will move <out> away from <data>.
677 */
678 left = 10 - http_emit_chunk_size(ob->p + 10, ob->i - 10);
679 ob->p += left;
680 ob->i -= left;
681
682 /* Copy previous data from ib->o into ob->o */
683 if (ib->o > 0) {
684 left = bo_contig_data(ib);
685 memcpy(ob->p - ob->o, bo_ptr(ib), left);
686 if (ib->o - left) /* second part of the buffer */
687 memcpy(ob->p - ob->o + left, ib->data, ib->o - left);
688 }
689
690 /* chunked encoding requires CRLF after data */
691 tail = ob->p + ob->i;
692 *tail++ = '\r';
693 *tail++ = '\n';
694
Christopher Faulet2fb28802015-12-01 10:40:57 +0100695 /* At the end of data, we must write the empty chunk 0<CRLF>,
696 * and terminate the trailers section with a last <CRLF>. If
697 * we're forwarding a chunked-encoded response, we'll have a
698 * trailers section after the empty chunk which needs to be
699 * forwarded and which will provide the last CRLF. Otherwise
700 * we write it ourselves.
701 */
702 if (end) {
703 struct http_msg *msg = &s->txn->rsp;
704
705 memcpy(tail, "0\r\n", 3);
706 tail += 3;
Christopher Fauletb77c5c22015-12-07 16:48:42 +0100707 if (!(msg->flags & HTTP_MSGF_TE_CHNK)) {
Christopher Faulet2fb28802015-12-01 10:40:57 +0100708 memcpy(tail, "\r\n", 2);
709 tail += 2;
710 }
711 }
712
Christopher Faulet3d97c902015-12-09 14:59:38 +0100713 ob->i = tail - ob->p;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100714 to_forward = ob->i;
715
716 /* update input rate */
Christopher Faulet92d36382015-11-05 13:35:03 +0100717 if (st->comp_ctx && st->comp_ctx->cur_lvl > 0) {
Christopher Faulet2fb28802015-12-01 10:40:57 +0100718 update_freq_ctr(&global.comp_bps_in, st->consumed);
719 strm_fe(s)->fe_counters.comp_in += st->consumed;
720 s->be->be_counters.comp_in += st->consumed;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100721 } else {
Christopher Faulet2fb28802015-12-01 10:40:57 +0100722 strm_fe(s)->fe_counters.comp_byp += st->consumed;
723 s->be->be_counters.comp_byp += st->consumed;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100724 }
725
726 /* copy the remaining data in the tmp buffer. */
Christopher Faulet2fb28802015-12-01 10:40:57 +0100727 b_adv(ib, st->consumed);
Christopher Faulet3d97c902015-12-09 14:59:38 +0100728 if (ib->i > 0) {
729 left = bi_contig_data(ib);
730 memcpy(ob->p + ob->i, bi_ptr(ib), left);
731 ob->i += left;
732 if (ib->i - left) {
733 memcpy(ob->p + ob->i, ib->data, ib->i - left);
734 ob->i += ib->i - left;
735 }
736 }
737
738 /* swap the buffers */
739 *in = ob;
740 *out = ib;
741
Christopher Faulet92d36382015-11-05 13:35:03 +0100742
743 if (st->comp_ctx && st->comp_ctx->cur_lvl > 0) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100744 update_freq_ctr(&global.comp_bps_out, to_forward);
745 strm_fe(s)->fe_counters.comp_out += to_forward;
746 s->be->be_counters.comp_out += to_forward;
747 }
748
Christopher Faulet3d97c902015-12-09 14:59:38 +0100749 return to_forward;
750}
751
752
753/***********************************************************************/
Christopher Faulet92d36382015-11-05 13:35:03 +0100754struct flt_ops comp_ops = {
755 .init = comp_flt_init,
756 .deinit = comp_flt_deinit,
757
758 .channel_start_analyze = comp_start_analyze,
759 .channel_analyze = comp_analyze,
760 .channel_end_analyze = comp_end_analyze,
761
Christopher Faulet309c6412015-12-02 09:57:32 +0100762 .http_data = comp_http_data,
763 .http_chunk_trailers = comp_http_chunk_trailers,
764 .http_forward_data = comp_http_forward_data,
Christopher Faulet92d36382015-11-05 13:35:03 +0100765};
766
Christopher Faulet3d97c902015-12-09 14:59:38 +0100767static int
768parse_compression_options(char **args, int section, struct proxy *proxy,
769 struct proxy *defpx, const char *file, int line,
770 char **err)
771{
Christopher Faulet92d36382015-11-05 13:35:03 +0100772 struct comp *comp;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100773
774 if (proxy->comp == NULL) {
775 comp = calloc(1, sizeof(struct comp));
776 proxy->comp = comp;
777 }
778 else
779 comp = proxy->comp;
780
781 if (!strcmp(args[1], "algo")) {
782 struct comp_ctx *ctx;
783 int cur_arg = 2;
784
785 if (!*args[cur_arg]) {
786 memprintf(err, "parsing [%s:%d] : '%s' expects <algorithm>\n",
787 file, line, args[0]);
788 return -1;
789 }
790 while (*(args[cur_arg])) {
791 if (comp_append_algo(comp, args[cur_arg]) < 0) {
792 memprintf(err, "'%s' : '%s' is not a supported algorithm.\n",
793 args[0], args[cur_arg]);
794 return -1;
795 }
796 if (proxy->comp->algos->init(&ctx, 9) == 0)
797 proxy->comp->algos->end(&ctx);
798 else {
799 memprintf(err, "'%s' : Can't init '%s' algorithm.\n",
800 args[0], args[cur_arg]);
801 return -1;
802 }
803 cur_arg++;
804 continue;
805 }
806 }
807 else if (!strcmp(args[1], "offload"))
808 comp->offload = 1;
809 else if (!strcmp(args[1], "type")) {
810 int cur_arg = 2;
811
812 if (!*args[cur_arg]) {
813 memprintf(err, "'%s' expects <type>\n", args[0]);
814 return -1;
815 }
816 while (*(args[cur_arg])) {
817 comp_append_type(comp, args[cur_arg]);
818 cur_arg++;
819 continue;
820 }
821 }
822 else {
823 memprintf(err, "'%s' expects 'algo', 'type' or 'offload'\n",
824 args[0]);
825 return -1;
826 }
827
828 return 0;
829}
830
Christopher Faulet92d36382015-11-05 13:35:03 +0100831static int
832parse_http_comp_flt(char **args, int *cur_arg, struct proxy *px,
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100833 struct flt_conf *fconf, char **err)
Christopher Faulet92d36382015-11-05 13:35:03 +0100834{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100835 struct flt_conf *fc, *back;
Christopher Faulet92d36382015-11-05 13:35:03 +0100836
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100837 list_for_each_entry_safe(fc, back, &px->filter_configs, list) {
838 if (fc->id == http_comp_flt_id) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100839 memprintf(err, "%s: Proxy supports only one compression filter\n", px->id);
840 return -1;
841 }
842 }
843
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100844 fconf->id = http_comp_flt_id;
845 fconf->conf = NULL;
846 fconf->ops = &comp_ops;
Christopher Faulet92d36382015-11-05 13:35:03 +0100847 (*cur_arg)++;
848
849 return 0;
850}
851
852
853int
854check_legacy_http_comp_flt(struct proxy *proxy)
855{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100856 struct flt_conf *fconf;
Christopher Faulet92d36382015-11-05 13:35:03 +0100857 int err = 0;
858
859 if (proxy->comp == NULL)
860 goto end;
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100861 if (!LIST_ISEMPTY(&proxy->filter_configs)) {
862 list_for_each_entry(fconf, &proxy->filter_configs, list) {
863 if (fconf->id == http_comp_flt_id)
Christopher Faulet92d36382015-11-05 13:35:03 +0100864 goto end;
865 }
866 Alert("config: %s '%s': require an explicit filter declaration to use HTTP compression\n",
867 proxy_type_str(proxy), proxy->id);
868 err++;
869 goto end;
870 }
871
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100872 fconf = calloc(1, sizeof(*fconf));
873 if (!fconf) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100874 Alert("config: %s '%s': out of memory\n",
875 proxy_type_str(proxy), proxy->id);
876 err++;
877 goto end;
878 }
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100879 fconf->id = http_comp_flt_id;
880 fconf->conf = NULL;
881 fconf->ops = &comp_ops;
882 LIST_ADDQ(&proxy->filter_configs, &fconf->list);
Christopher Faulet92d36382015-11-05 13:35:03 +0100883
884 end:
885 return err;
886}
887
888/*
889 * boolean, returns true if compression is used (either gzip or deflate) in the
890 * response.
891 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100892static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100893smp_fetch_res_comp(const struct arg *args, struct sample *smp, const char *kw,
894 void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100895{
Willy Tarreaube508f12016-03-10 11:47:01 +0100896 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100897
Christopher Faulet3d97c902015-12-09 14:59:38 +0100898 smp->data.type = SMP_T_BOOL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100899 smp->data.u.sint = (txn && (txn->rsp.flags & HTTP_MSGF_COMPRESSING));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100900 return 1;
901}
902
Christopher Faulet92d36382015-11-05 13:35:03 +0100903/*
904 * string, returns algo
905 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100906static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100907smp_fetch_res_comp_algo(const struct arg *args, struct sample *smp,
908 const char *kw, void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100909{
Willy Tarreaube508f12016-03-10 11:47:01 +0100910 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100911 struct filter *filter;
912 struct comp_state *st;
913
914 if (!(txn || !(txn->rsp.flags & HTTP_MSGF_COMPRESSING)))
Christopher Faulet3d97c902015-12-09 14:59:38 +0100915 return 0;
916
Christopher Fauletfcf035c2015-12-03 11:48:03 +0100917 list_for_each_entry(filter, &strm_flt(smp->strm)->filters, list) {
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100918 if (FLT_ID(filter) != http_comp_flt_id)
Christopher Faulet92d36382015-11-05 13:35:03 +0100919 continue;
920
921 if (!(st = filter->ctx))
922 break;
923
924 smp->data.type = SMP_T_STR;
925 smp->flags = SMP_F_CONST;
926 smp->data.u.str.str = st->comp_algo->cfg_name;
927 smp->data.u.str.len = st->comp_algo->cfg_name_len;
928 return 1;
929 }
930 return 0;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100931}
932
933/* Declare the config parser for "compression" keyword */
934static struct cfg_kw_list cfg_kws = {ILH, {
935 { CFG_LISTEN, "compression", parse_compression_options },
936 { 0, NULL, NULL },
937 }
938};
939
Christopher Faulet92d36382015-11-05 13:35:03 +0100940/* Declare the filter parser for "compression" keyword */
941static struct flt_kw_list filter_kws = { "COMP", { }, {
942 { "compression", parse_http_comp_flt },
943 { NULL, NULL },
944 }
945};
946
Christopher Faulet3d97c902015-12-09 14:59:38 +0100947/* Note: must not be declared <const> as its list will be overwritten */
948static struct sample_fetch_kw_list sample_fetch_keywords = {ILH, {
Christopher Faulet92d36382015-11-05 13:35:03 +0100949 { "res.comp", smp_fetch_res_comp, 0, NULL, SMP_T_BOOL, SMP_USE_HRSHP },
950 { "res.comp_algo", smp_fetch_res_comp_algo, 0, NULL, SMP_T_STR, SMP_USE_HRSHP },
951 { /* END */ },
952 }
953};
Christopher Faulet3d97c902015-12-09 14:59:38 +0100954
955__attribute__((constructor))
Christopher Faulet92d36382015-11-05 13:35:03 +0100956static void
957__flt_http_comp_init(void)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100958{
959 cfg_register_keywords(&cfg_kws);
Christopher Faulet92d36382015-11-05 13:35:03 +0100960 flt_register_keywords(&filter_kws);
Christopher Faulet3d97c902015-12-09 14:59:38 +0100961 sample_register_fetches(&sample_fetch_keywords);
962}