blob: e812b80032be15358e55b24d72f21b7b11d92952 [file] [log] [blame]
Christopher Faulet3d97c902015-12-09 14:59:38 +01001/*
2 * Stream filters related variables and functions.
3 *
4 * Copyright (C) 2015 Qualys Inc., Christopher Faulet <cfaulet@qualys.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
13#include <common/buffer.h>
14#include <common/cfgparse.h>
Willy Tarreaub96b77e2018-12-11 10:22:41 +010015#include <common/htx.h>
Willy Tarreau0108d902018-11-25 19:14:37 +010016#include <common/initcall.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010017#include <common/mini-clist.h>
18#include <common/standard.h>
19
20#include <types/compression.h>
21#include <types/filters.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010022#include <types/proxy.h>
23#include <types/sample.h>
24
25#include <proto/compression.h>
Christopher Faulet92d36382015-11-05 13:35:03 +010026#include <proto/filters.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010027#include <proto/hdr_idx.h>
Christopher Faulete6902cd2018-11-30 22:29:48 +010028#include <proto/http_htx.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010029#include <proto/proto_http.h>
30#include <proto/sample.h>
31#include <proto/stream.h>
32
Christopher Fauletf4a4ef72018-12-07 17:39:53 +010033const char *http_comp_flt_id = "compression filter";
Christopher Faulet92d36382015-11-05 13:35:03 +010034
35struct flt_ops comp_ops;
36
Christopher Faulet92d36382015-11-05 13:35:03 +010037struct comp_state {
38 struct comp_ctx *comp_ctx; /* compression context */
39 struct comp_algo *comp_algo; /* compression algorithm if not NULL */
Christopher Faulet92d36382015-11-05 13:35:03 +010040};
41
Willy Tarreau8ceae722018-11-26 11:58:30 +010042/* Pools used to allocate comp_state structs */
43DECLARE_STATIC_POOL(pool_head_comp_state, "comp_state", sizeof(struct comp_state));
44
45static THREAD_LOCAL struct buffer tmpbuf;
46static THREAD_LOCAL struct buffer zbuf;
Willy Tarreau8ceae722018-11-26 11:58:30 +010047
Christopher Faulet92d36382015-11-05 13:35:03 +010048static int select_compression_request_header(struct comp_state *st,
49 struct stream *s,
50 struct http_msg *msg);
51static int select_compression_response_header(struct comp_state *st,
52 struct stream *s,
53 struct http_msg *msg);
Christopher Faulet27d93c32018-12-15 22:32:02 +010054static int set_compression_response_header(struct comp_state *st,
55 struct stream *s,
56 struct http_msg *msg);
Christopher Faulet92d36382015-11-05 13:35:03 +010057
Christopher Faulete6902cd2018-11-30 22:29:48 +010058static int htx_compression_buffer_init(struct htx *htx, struct buffer *out);
59static int htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
60 struct buffer *out);
61static int htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end);
62
Christopher Faulet92d36382015-11-05 13:35:03 +010063/***********************************************************************/
64static int
Christopher Faulete6902cd2018-11-30 22:29:48 +010065comp_flt_init(struct proxy *px, struct flt_conf *fconf)
66{
Christopher Faulet6e540952018-12-03 22:43:41 +010067 fconf->flags |= FLT_CFG_FL_HTX;
Christopher Faulete6902cd2018-11-30 22:29:48 +010068 return 0;
69}
70
71static int
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020072comp_flt_init_per_thread(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010073{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020074 if (!tmpbuf.size && b_alloc(&tmpbuf) == NULL)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010075 return -1;
Willy Tarreauc9fa0482018-07-10 17:43:27 +020076 if (!zbuf.size && b_alloc(&zbuf) == NULL)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010077 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +010078 return 0;
79}
80
81static void
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020082comp_flt_deinit_per_thread(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010083{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020084 if (tmpbuf.size)
Christopher Faulet92d36382015-11-05 13:35:03 +010085 b_free(&tmpbuf);
Willy Tarreauc9fa0482018-07-10 17:43:27 +020086 if (zbuf.size)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010087 b_free(&zbuf);
Christopher Faulet92d36382015-11-05 13:35:03 +010088}
89
90static int
91comp_start_analyze(struct stream *s, struct filter *filter, struct channel *chn)
92{
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020093
Christopher Faulet92d36382015-11-05 13:35:03 +010094 if (filter->ctx == NULL) {
95 struct comp_state *st;
96
Willy Tarreaubafbe012017-11-24 17:34:44 +010097 st = pool_alloc_dirty(pool_head_comp_state);
Christopher Fauleta03d4ad2017-06-26 16:53:33 +020098 if (st == NULL)
Christopher Faulet92d36382015-11-05 13:35:03 +010099 return -1;
100
Christopher Faulet89f2b162019-07-15 21:16:04 +0200101 st->comp_algo = NULL;
102 st->comp_ctx = NULL;
103 filter->ctx = st;
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200104
105 /* Register post-analyzer on AN_RES_WAIT_HTTP because we need to
106 * analyze response headers before http-response rules execution
107 * to be sure we can use res.comp and res.comp_algo sample
108 * fetches */
109 filter->post_analyzers |= AN_RES_WAIT_HTTP;
Christopher Faulet92d36382015-11-05 13:35:03 +0100110 }
111 return 1;
112}
113
114static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100115comp_end_analyze(struct stream *s, struct filter *filter, struct channel *chn)
116{
117 struct comp_state *st = filter->ctx;
Christopher Faulet92d36382015-11-05 13:35:03 +0100118
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200119 if (!st)
Christopher Faulet92d36382015-11-05 13:35:03 +0100120 goto end;
121
Christopher Faulet92d36382015-11-05 13:35:03 +0100122 /* release any possible compression context */
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200123 if (st->comp_algo)
124 st->comp_algo->end(&st->comp_ctx);
Willy Tarreaubafbe012017-11-24 17:34:44 +0100125 pool_free(pool_head_comp_state, st);
Christopher Faulet92d36382015-11-05 13:35:03 +0100126 filter->ctx = NULL;
127 end:
128 return 1;
129}
130
131static int
Christopher Faulet1339d742016-05-11 16:48:33 +0200132comp_http_headers(struct stream *s, struct filter *filter, struct http_msg *msg)
133{
134 struct comp_state *st = filter->ctx;
135
136 if (!strm_fe(s)->comp && !s->be->comp)
137 goto end;
138
139 if (!(msg->chn->flags & CF_ISRESP))
140 select_compression_request_header(st, s, msg);
141 else {
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200142 /* Response headers have already been checked in
143 * comp_http_post_analyze callback. */
Christopher Faulet1339d742016-05-11 16:48:33 +0200144 if (st->comp_algo) {
Christopher Faulet27d93c32018-12-15 22:32:02 +0100145 if (!set_compression_response_header(st, s, msg))
146 goto end;
Christopher Faulet1339d742016-05-11 16:48:33 +0200147 register_data_filter(s, msg->chn, filter);
Christopher Faulet1339d742016-05-11 16:48:33 +0200148 }
149 }
150
151 end:
152 return 1;
153}
154
155static int
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200156comp_http_post_analyze(struct stream *s, struct filter *filter,
157 struct channel *chn, unsigned an_bit)
158{
159 struct http_txn *txn = s->txn;
160 struct http_msg *msg = &txn->rsp;
161 struct comp_state *st = filter->ctx;
162
163 if (an_bit != AN_RES_WAIT_HTTP)
164 goto end;
165
166 if (!strm_fe(s)->comp && !s->be->comp)
167 goto end;
168
169 select_compression_response_header(st, s, msg);
170
171 end:
172 return 1;
173}
174
175static int
Christopher Faulete6902cd2018-11-30 22:29:48 +0100176comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg,
177 unsigned int offset, unsigned int len)
178{
179 struct comp_state *st = filter->ctx;
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100180 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100181 struct htx_blk *blk;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100182 int ret, consumed = 0, to_forward = 0;
183
Christopher Fauletee847d42019-05-23 11:55:33 +0200184 for (blk = htx_get_first_blk(htx); blk && len; blk = htx_get_next_blk(htx, blk)) {
Christopher Faulete6902cd2018-11-30 22:29:48 +0100185 enum htx_blk_type type = htx_get_blk_type(blk);
186 uint32_t sz = htx_get_blksz(blk);
187 struct ist v;
188
Christopher Fauletee847d42019-05-23 11:55:33 +0200189 if (offset >= sz) {
190 offset -= sz;
191 continue;
192 }
193
Christopher Faulete6902cd2018-11-30 22:29:48 +0100194 switch (type) {
195 case HTX_BLK_UNUSED:
196 break;
197
198 case HTX_BLK_DATA:
199 v = htx_get_blk_value(htx, blk);
200 v.ptr += offset;
201 v.len -= offset;
202 if (v.len > len)
203 v.len = len;
204 if (htx_compression_buffer_init(htx, &trash) < 0) {
205 msg->chn->flags |= CF_WAKE_WRITE;
206 goto end;
207 }
208 ret = htx_compression_buffer_add_data(st, v.ptr, v.len, &trash);
209 if (ret < 0)
210 goto error;
211 if (htx_compression_buffer_end(st, &trash, 0) < 0)
212 goto error;
213 len -= ret;
214 consumed += ret;
215 to_forward += b_data(&trash);
216 if (ret == sz && !b_data(&trash)) {
217 offset = 0;
218 blk = htx_remove_blk(htx, blk);
219 continue;
220 }
221 v.len = ret;
222 blk = htx_replace_blk_value(htx, blk, v, ist2(b_head(&trash), b_data(&trash)));
223 break;
224
Christopher Faulete6902cd2018-11-30 22:29:48 +0100225 case HTX_BLK_TLR:
Christopher Faulet2d7c5392019-06-03 10:41:26 +0200226 case HTX_BLK_EOT:
Christopher Faulete6902cd2018-11-30 22:29:48 +0100227 case HTX_BLK_EOM:
228 if (msg->flags & HTTP_MSGF_COMPRESSING) {
229 if (htx_compression_buffer_init(htx, &trash) < 0) {
230 msg->chn->flags |= CF_WAKE_WRITE;
231 goto end;
232 }
233 if (htx_compression_buffer_end(st, &trash, 1) < 0)
234 goto error;
Christopher Fauletd238ae32018-12-21 15:10:25 +0100235 if (b_data(&trash)) {
Christopher Faulet86bc8df2019-06-11 10:38:38 +0200236 struct htx_blk *last = htx_add_last_data(htx, ist2(b_head(&trash), b_data(&trash)));
237 if (!last)
238 goto error;
239 blk = htx_get_next_blk(htx, last);
Christopher Fauletd238ae32018-12-21 15:10:25 +0100240 if (!blk)
241 goto error;
242 to_forward += b_data(&trash);
243 }
Christopher Faulete6902cd2018-11-30 22:29:48 +0100244 msg->flags &= ~HTTP_MSGF_COMPRESSING;
245 /* We let the mux add last empty chunk and empty trailers */
246 }
247 /* fall through */
248
249 default:
250 sz -= offset;
251 if (sz > len)
252 sz = len;
253 consumed += sz;
254 to_forward += sz;
255 len -= sz;
256 break;
257 }
258
259 offset = 0;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100260 }
261
262 end:
263 if (to_forward != consumed)
264 flt_update_offsets(filter, msg->chn, to_forward - consumed);
265
266 if (st->comp_ctx && st->comp_ctx->cur_lvl > 0) {
Willy Tarreauef6fd852019-02-04 11:48:03 +0100267 update_freq_ctr(&global.comp_bps_in, consumed);
Olivier Houchard43da3432019-03-08 18:50:27 +0100268 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_in, consumed);
269 _HA_ATOMIC_ADD(&s->be->be_counters.comp_in, consumed);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100270 update_freq_ctr(&global.comp_bps_out, to_forward);
Olivier Houchard43da3432019-03-08 18:50:27 +0100271 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_out, to_forward);
272 _HA_ATOMIC_ADD(&s->be->be_counters.comp_out, to_forward);
Willy Tarreauef6fd852019-02-04 11:48:03 +0100273 } else {
Olivier Houchard43da3432019-03-08 18:50:27 +0100274 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_byp, consumed);
275 _HA_ATOMIC_ADD(&s->be->be_counters.comp_byp, consumed);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100276 }
277 return to_forward;
278
279 error:
280 return -1;
281}
282
Christopher Faulet2fb28802015-12-01 10:40:57 +0100283
Christopher Faulet92d36382015-11-05 13:35:03 +0100284static int
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200285comp_http_end(struct stream *s, struct filter *filter,
286 struct http_msg *msg)
287{
288 struct comp_state *st = filter->ctx;
289
290 if (!(msg->chn->flags & CF_ISRESP) || !st || !st->comp_algo)
291 goto end;
292
293 if (strm_fe(s)->mode == PR_MODE_HTTP)
Olivier Houchard43da3432019-03-08 18:50:27 +0100294 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.p.http.comp_rsp, 1);
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200295 if ((s->flags & SF_BE_ASSIGNED) && (s->be->mode == PR_MODE_HTTP))
Olivier Houchard43da3432019-03-08 18:50:27 +0100296 _HA_ATOMIC_ADD(&s->be->be_counters.p.http.comp_rsp, 1);
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200297 end:
298 return 1;
299}
Christopher Faulet27d93c32018-12-15 22:32:02 +0100300
Christopher Faulet89f2b162019-07-15 21:16:04 +0200301/***********************************************************************/
Christopher Faulet27d93c32018-12-15 22:32:02 +0100302static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200303set_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulet27d93c32018-12-15 22:32:02 +0100304{
305 struct htx *htx = htxbuf(&msg->chn->buf);
Tim Duesterhusb229f012019-01-29 16:38:56 +0100306 struct http_hdr_ctx ctx;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100307
308 /*
309 * Add Content-Encoding header when it's not identity encoding.
310 * RFC 2616 : Identity encoding: This content-coding is used only in the
311 * Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding
312 * header.
313 */
314 if (st->comp_algo->cfg_name_len != 8 || memcmp(st->comp_algo->cfg_name, "identity", 8) != 0) {
315 struct ist v = ist2(st->comp_algo->ua_name, st->comp_algo->ua_name_len);
316
317 if (!http_add_header(htx, ist("Content-Encoding"), v))
318 goto error;
319 }
320
321 /* remove Content-Length header */
322 if (msg->flags & HTTP_MSGF_CNT_LEN) {
Christopher Faulet27d93c32018-12-15 22:32:02 +0100323 ctx.blk = NULL;
324 while (http_find_header(htx, ist("Content-Length"), &ctx, 1))
325 http_remove_header(htx, &ctx);
326 }
327
328 /* add "Transfer-Encoding: chunked" header */
329 if (!(msg->flags & HTTP_MSGF_TE_CHNK)) {
330 if (!http_add_header(htx, ist("Transfer-Encoding"), ist("chunked")))
331 goto error;
332 }
333
Tim Duesterhusb229f012019-01-29 16:38:56 +0100334 /* convert "ETag" header to a weak ETag */
335 ctx.blk = NULL;
336 if (http_find_header(htx, ist("ETag"), &ctx, 1)) {
337 if (ctx.value.ptr[0] == '"') {
338 /* This a strong ETag. Convert it to a weak one. */
339 struct ist v = ist2(trash.area, 0);
340 if (istcat(&v, ist("W/"), trash.size) == -1 || istcat(&v, ctx.value, trash.size) == -1)
341 goto error;
342
343 if (!http_replace_header_value(htx, &ctx, v))
344 goto error;
345 }
346 }
347
Tim Duesterhus721d6862019-06-17 16:10:07 +0200348 if (!http_add_header(htx, ist("Vary"), ist("Accept-Encoding")))
349 goto error;
350
Christopher Faulet27d93c32018-12-15 22:32:02 +0100351 return 1;
352
353 error:
354 st->comp_algo->end(&st->comp_ctx);
355 st->comp_algo = NULL;
356 return 0;
357}
358
Christopher Faulet3d97c902015-12-09 14:59:38 +0100359/*
360 * Selects a compression algorithm depending on the client request.
361 */
Christopher Faulete6902cd2018-11-30 22:29:48 +0100362static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200363select_compression_request_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100364{
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100365 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100366 struct http_hdr_ctx ctx;
367 struct comp_algo *comp_algo = NULL;
368 struct comp_algo *comp_algo_back = NULL;
369
370 /* Disable compression for older user agents announcing themselves as "Mozilla/4"
371 * unless they are known good (MSIE 6 with XP SP2, or MSIE 7 and later).
372 * See http://zoompf.com/2012/02/lose-the-wait-http-compression for more details.
373 */
374 ctx.blk = NULL;
375 if (http_find_header(htx, ist("User-Agent"), &ctx, 1) &&
376 ctx.value.len >= 9 &&
377 memcmp(ctx.value.ptr, "Mozilla/4", 9) == 0 &&
378 (ctx.value.len < 31 ||
379 memcmp(ctx.value.ptr + 25, "MSIE ", 5) != 0 ||
380 *(ctx.value.ptr + 30) < '6' ||
381 (*(ctx.value.ptr + 30) == '6' &&
382 (ctx.value.len < 54 || memcmp(ctx.value.ptr + 51, "SV1", 3) != 0)))) {
383 st->comp_algo = NULL;
384 return 0;
385 }
386
387 /* search for the algo in the backend in priority or the frontend */
388 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
389 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
390 int best_q = 0;
391
392 ctx.blk = NULL;
393 while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 0)) {
394 const char *qval;
395 int q;
396 int toklen;
397
398 /* try to isolate the token from the optional q-value */
399 toklen = 0;
400 while (toklen < ctx.value.len && HTTP_IS_TOKEN(*(ctx.value.ptr + toklen)))
401 toklen++;
402
403 qval = ctx.value.ptr + toklen;
404 while (1) {
405 while (qval < ctx.value.ptr + ctx.value.len && HTTP_IS_LWS(*qval))
406 qval++;
407
408 if (qval >= ctx.value.ptr + ctx.value.len || *qval != ';') {
409 qval = NULL;
410 break;
411 }
412 qval++;
413
414 while (qval < ctx.value.ptr + ctx.value.len && HTTP_IS_LWS(*qval))
415 qval++;
416
417 if (qval >= ctx.value.ptr + ctx.value.len) {
418 qval = NULL;
419 break;
420 }
421 if (strncmp(qval, "q=", MIN(ctx.value.ptr + ctx.value.len - qval, 2)) == 0)
422 break;
423
424 while (qval < ctx.value.ptr + ctx.value.len && *qval != ';')
425 qval++;
426 }
427
428 /* here we have qval pointing to the first "q=" attribute or NULL if not found */
429 q = qval ? http_parse_qvalue(qval + 2, NULL) : 1000;
430
431 if (q <= best_q)
432 continue;
433
434 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
435 if (*(ctx.value.ptr) == '*' ||
436 word_match(ctx.value.ptr, toklen, comp_algo->ua_name, comp_algo->ua_name_len)) {
437 st->comp_algo = comp_algo;
438 best_q = q;
439 break;
440 }
441 }
442 }
443 }
444
445 /* remove all occurrences of the header when "compression offload" is set */
446 if (st->comp_algo) {
447 if ((s->be->comp && s->be->comp->offload) ||
448 (strm_fe(s)->comp && strm_fe(s)->comp->offload)) {
449 http_remove_header(htx, &ctx);
450 ctx.blk = NULL;
451 while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 1))
452 http_remove_header(htx, &ctx);
453 }
Christopher Faulet3d97c902015-12-09 14:59:38 +0100454 return 1;
455 }
456
457 /* identity is implicit does not require headers */
Christopher Faulet92d36382015-11-05 13:35:03 +0100458 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
459 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100460 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
461 if (comp_algo->cfg_name_len == 8 && memcmp(comp_algo->cfg_name, "identity", 8) == 0) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100462 st->comp_algo = comp_algo;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100463 return 1;
464 }
465 }
466 }
467
Christopher Faulet92d36382015-11-05 13:35:03 +0100468 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100469 return 0;
470}
471
472/*
473 * Selects a comression algorithm depending of the server response.
474 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100475static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200476select_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100477{
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100478 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100479 struct http_txn *txn = s->txn;
480 struct http_hdr_ctx ctx;
481 struct comp_type *comp_type;
482
483 /* no common compression algorithm was found in request header */
484 if (st->comp_algo == NULL)
485 goto fail;
486
Christopher Faulet1d3613a2019-01-07 14:41:59 +0100487 /* compression already in progress */
488 if (msg->flags & HTTP_MSGF_COMPRESSING)
489 goto fail;
490
Christopher Faulete6902cd2018-11-30 22:29:48 +0100491 /* HTTP < 1.1 should not be compressed */
492 if (!(msg->flags & HTTP_MSGF_VER_11) || !(txn->req.flags & HTTP_MSGF_VER_11))
493 goto fail;
494
495 if (txn->meth == HTTP_METH_HEAD)
496 goto fail;
497
498 /* compress 200,201,202,203 responses only */
499 if ((txn->status != 200) &&
500 (txn->status != 201) &&
501 (txn->status != 202) &&
502 (txn->status != 203))
503 goto fail;
504
Christopher Fauletc963eb22018-12-21 14:53:54 +0100505 if (!(msg->flags & HTTP_MSGF_XFER_LEN) || msg->flags & HTTP_MSGF_BODYLESS)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100506 goto fail;
507
508 /* content is already compressed */
509 ctx.blk = NULL;
510 if (http_find_header(htx, ist("Content-Encoding"), &ctx, 1))
511 goto fail;
512
513 /* no compression when Cache-Control: no-transform is present in the message */
514 ctx.blk = NULL;
515 while (http_find_header(htx, ist("Cache-Control"), &ctx, 0)) {
516 if (word_match(ctx.value.ptr, ctx.value.len, "no-transform", 12))
517 goto fail;
518 }
519
Tim Duesterhusb229f012019-01-29 16:38:56 +0100520 /* no compression when ETag is malformed */
521 ctx.blk = NULL;
522 if (http_find_header(htx, ist("ETag"), &ctx, 1)) {
523 if (!(((ctx.value.len >= 4 && memcmp(ctx.value.ptr, "W/\"", 3) == 0) || /* Either a weak ETag */
524 (ctx.value.len >= 2 && ctx.value.ptr[0] == '"')) && /* or strong ETag */
525 ctx.value.ptr[ctx.value.len - 1] == '"')) {
526 goto fail;
527 }
528 }
529 /* no compression when multiple ETags are present
530 * Note: Do not reset ctx.blk!
531 */
532 if (http_find_header(htx, ist("ETag"), &ctx, 1))
533 goto fail;
534
Christopher Faulete6902cd2018-11-30 22:29:48 +0100535 comp_type = NULL;
536
537 /* we don't want to compress multipart content-types, nor content-types that are
538 * not listed in the "compression type" directive if any. If no content-type was
539 * found but configuration requires one, we don't compress either. Backend has
540 * the priority.
541 */
542 ctx.blk = NULL;
543 if (http_find_header(htx, ist("Content-Type"), &ctx, 1)) {
544 if (ctx.value.len >= 9 && strncasecmp("multipart", ctx.value.ptr, 9) == 0)
545 goto fail;
546
547 if ((s->be->comp && (comp_type = s->be->comp->types)) ||
548 (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types))) {
549 for (; comp_type; comp_type = comp_type->next) {
550 if (ctx.value.len >= comp_type->name_len &&
551 strncasecmp(ctx.value.ptr, comp_type->name, comp_type->name_len) == 0)
552 /* this Content-Type should be compressed */
553 break;
554 }
555 /* this Content-Type should not be compressed */
556 if (comp_type == NULL)
557 goto fail;
558 }
559 }
560 else { /* no content-type header */
561 if ((s->be->comp && s->be->comp->types) ||
562 (strm_fe(s)->comp && strm_fe(s)->comp->types))
563 goto fail; /* a content-type was required */
564 }
565
566 /* limit compression rate */
567 if (global.comp_rate_lim > 0)
568 if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim)
569 goto fail;
570
571 /* limit cpu usage */
Willy Tarreau81036f22019-05-20 19:24:50 +0200572 if (ti->idle_pct < compress_min_idle)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100573 goto fail;
574
575 /* initialize compression */
576 if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0)
577 goto fail;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100578 msg->flags |= HTTP_MSGF_COMPRESSING;
579 return 1;
580
581 deinit_comp_ctx:
582 st->comp_algo->end(&st->comp_ctx);
583 fail:
584 st->comp_algo = NULL;
585 return 0;
586}
587
Christopher Faulet3d97c902015-12-09 14:59:38 +0100588/***********************************************************************/
Christopher Faulete6902cd2018-11-30 22:29:48 +0100589static int
590htx_compression_buffer_init(struct htx *htx, struct buffer *out)
591{
592 /* output stream requires at least 10 bytes for the gzip header, plus
593 * at least 8 bytes for the gzip trailer (crc+len), plus a possible
594 * plus at most 5 bytes per 32kB block and 2 bytes to close the stream.
595 */
596 if (htx_free_space(htx) < 20 + 5 * ((htx->data + 32767) >> 15))
597 return -1;
598 b_reset(out);
599 return 0;
600}
601
Christopher Faulete6902cd2018-11-30 22:29:48 +0100602static int
603htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
604 struct buffer *out)
605{
606 return st->comp_algo->add_data(st->comp_ctx, data, len, out);
607}
608
Christopher Faulete6902cd2018-11-30 22:29:48 +0100609static int
610htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end)
611{
612 if (end)
613 return st->comp_algo->finish(st->comp_ctx, out);
614 else
615 return st->comp_algo->flush(st->comp_ctx, out);
616}
617
Christopher Faulet3d97c902015-12-09 14:59:38 +0100618
619/***********************************************************************/
Christopher Faulet92d36382015-11-05 13:35:03 +0100620struct flt_ops comp_ops = {
Christopher Faulete6902cd2018-11-30 22:29:48 +0100621 .init = comp_flt_init,
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +0200622 .init_per_thread = comp_flt_init_per_thread,
623 .deinit_per_thread = comp_flt_deinit_per_thread,
Christopher Faulet92d36382015-11-05 13:35:03 +0100624
625 .channel_start_analyze = comp_start_analyze,
Christopher Faulet92d36382015-11-05 13:35:03 +0100626 .channel_end_analyze = comp_end_analyze,
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200627 .channel_post_analyze = comp_http_post_analyze,
Christopher Faulet92d36382015-11-05 13:35:03 +0100628
Christopher Faulet1339d742016-05-11 16:48:33 +0200629 .http_headers = comp_http_headers,
Christopher Faulete6902cd2018-11-30 22:29:48 +0100630 .http_payload = comp_http_payload,
631 .http_end = comp_http_end,
Christopher Faulet92d36382015-11-05 13:35:03 +0100632};
633
Christopher Faulet3d97c902015-12-09 14:59:38 +0100634static int
635parse_compression_options(char **args, int section, struct proxy *proxy,
636 struct proxy *defpx, const char *file, int line,
637 char **err)
638{
Christopher Faulet92d36382015-11-05 13:35:03 +0100639 struct comp *comp;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100640
641 if (proxy->comp == NULL) {
Vincent Bernat02779b62016-04-03 13:48:43 +0200642 comp = calloc(1, sizeof(*comp));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100643 proxy->comp = comp;
644 }
645 else
646 comp = proxy->comp;
647
648 if (!strcmp(args[1], "algo")) {
649 struct comp_ctx *ctx;
650 int cur_arg = 2;
651
652 if (!*args[cur_arg]) {
653 memprintf(err, "parsing [%s:%d] : '%s' expects <algorithm>\n",
654 file, line, args[0]);
655 return -1;
656 }
657 while (*(args[cur_arg])) {
658 if (comp_append_algo(comp, args[cur_arg]) < 0) {
659 memprintf(err, "'%s' : '%s' is not a supported algorithm.\n",
660 args[0], args[cur_arg]);
661 return -1;
662 }
663 if (proxy->comp->algos->init(&ctx, 9) == 0)
664 proxy->comp->algos->end(&ctx);
665 else {
666 memprintf(err, "'%s' : Can't init '%s' algorithm.\n",
667 args[0], args[cur_arg]);
668 return -1;
669 }
670 cur_arg++;
671 continue;
672 }
673 }
674 else if (!strcmp(args[1], "offload"))
675 comp->offload = 1;
676 else if (!strcmp(args[1], "type")) {
677 int cur_arg = 2;
678
679 if (!*args[cur_arg]) {
680 memprintf(err, "'%s' expects <type>\n", args[0]);
681 return -1;
682 }
683 while (*(args[cur_arg])) {
684 comp_append_type(comp, args[cur_arg]);
685 cur_arg++;
686 continue;
687 }
688 }
689 else {
690 memprintf(err, "'%s' expects 'algo', 'type' or 'offload'\n",
691 args[0]);
692 return -1;
693 }
694
695 return 0;
696}
697
Christopher Faulet92d36382015-11-05 13:35:03 +0100698static int
699parse_http_comp_flt(char **args, int *cur_arg, struct proxy *px,
Thierry Fournier3610c392016-04-13 18:27:51 +0200700 struct flt_conf *fconf, char **err, void *private)
Christopher Faulet92d36382015-11-05 13:35:03 +0100701{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100702 struct flt_conf *fc, *back;
Christopher Faulet92d36382015-11-05 13:35:03 +0100703
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100704 list_for_each_entry_safe(fc, back, &px->filter_configs, list) {
705 if (fc->id == http_comp_flt_id) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100706 memprintf(err, "%s: Proxy supports only one compression filter\n", px->id);
707 return -1;
708 }
709 }
710
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100711 fconf->id = http_comp_flt_id;
712 fconf->conf = NULL;
713 fconf->ops = &comp_ops;
Christopher Faulet92d36382015-11-05 13:35:03 +0100714 (*cur_arg)++;
715
716 return 0;
717}
718
719
720int
Christopher Fauletc9df7f72018-12-10 16:14:04 +0100721check_implicit_http_comp_flt(struct proxy *proxy)
Christopher Faulet92d36382015-11-05 13:35:03 +0100722{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100723 struct flt_conf *fconf;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100724 int explicit = 0;
725 int comp = 0;
Christopher Faulet92d36382015-11-05 13:35:03 +0100726 int err = 0;
727
728 if (proxy->comp == NULL)
729 goto end;
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100730 if (!LIST_ISEMPTY(&proxy->filter_configs)) {
731 list_for_each_entry(fconf, &proxy->filter_configs, list) {
732 if (fconf->id == http_comp_flt_id)
Christopher Faulet27d93c32018-12-15 22:32:02 +0100733 comp = 1;
734 else if (fconf->id == cache_store_flt_id) {
735 if (comp) {
736 ha_alert("config: %s '%s': unable to enable the compression filter "
737 "before any cache filter.\n",
738 proxy_type_str(proxy), proxy->id);
739 err++;
740 goto end;
741 }
742 }
743 else
744 explicit = 1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100745 }
Christopher Faulet27d93c32018-12-15 22:32:02 +0100746 }
747 if (comp)
748 goto end;
749 else if (explicit) {
750 ha_alert("config: %s '%s': require an explicit filter declaration to use "
751 "HTTP compression\n", proxy_type_str(proxy), proxy->id);
Christopher Faulet92d36382015-11-05 13:35:03 +0100752 err++;
753 goto end;
754 }
755
Christopher Faulet27d93c32018-12-15 22:32:02 +0100756 /* Implicit declaration of the compression filter is always the last
757 * one */
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100758 fconf = calloc(1, sizeof(*fconf));
759 if (!fconf) {
Christopher Faulet767a84b2017-11-24 16:50:31 +0100760 ha_alert("config: %s '%s': out of memory\n",
761 proxy_type_str(proxy), proxy->id);
Christopher Faulet92d36382015-11-05 13:35:03 +0100762 err++;
763 goto end;
764 }
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100765 fconf->id = http_comp_flt_id;
766 fconf->conf = NULL;
767 fconf->ops = &comp_ops;
768 LIST_ADDQ(&proxy->filter_configs, &fconf->list);
Christopher Faulet92d36382015-11-05 13:35:03 +0100769 end:
770 return err;
771}
772
773/*
774 * boolean, returns true if compression is used (either gzip or deflate) in the
775 * response.
776 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100777static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100778smp_fetch_res_comp(const struct arg *args, struct sample *smp, const char *kw,
779 void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100780{
Willy Tarreaube508f12016-03-10 11:47:01 +0100781 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100782
Christopher Faulet3d97c902015-12-09 14:59:38 +0100783 smp->data.type = SMP_T_BOOL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100784 smp->data.u.sint = (txn && (txn->rsp.flags & HTTP_MSGF_COMPRESSING));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100785 return 1;
786}
787
Christopher Faulet92d36382015-11-05 13:35:03 +0100788/*
789 * string, returns algo
790 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100791static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100792smp_fetch_res_comp_algo(const struct arg *args, struct sample *smp,
793 const char *kw, void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100794{
Willy Tarreaube508f12016-03-10 11:47:01 +0100795 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100796 struct filter *filter;
797 struct comp_state *st;
798
Christopher Faulet03d85532017-09-15 10:14:43 +0200799 if (!txn || !(txn->rsp.flags & HTTP_MSGF_COMPRESSING))
Christopher Faulet3d97c902015-12-09 14:59:38 +0100800 return 0;
801
Christopher Fauletfcf035c2015-12-03 11:48:03 +0100802 list_for_each_entry(filter, &strm_flt(smp->strm)->filters, list) {
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100803 if (FLT_ID(filter) != http_comp_flt_id)
Christopher Faulet92d36382015-11-05 13:35:03 +0100804 continue;
805
806 if (!(st = filter->ctx))
807 break;
808
809 smp->data.type = SMP_T_STR;
810 smp->flags = SMP_F_CONST;
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200811 smp->data.u.str.area = st->comp_algo->cfg_name;
812 smp->data.u.str.data = st->comp_algo->cfg_name_len;
Christopher Faulet92d36382015-11-05 13:35:03 +0100813 return 1;
814 }
815 return 0;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100816}
817
818/* Declare the config parser for "compression" keyword */
819static struct cfg_kw_list cfg_kws = {ILH, {
820 { CFG_LISTEN, "compression", parse_compression_options },
821 { 0, NULL, NULL },
822 }
823};
824
Willy Tarreau0108d902018-11-25 19:14:37 +0100825INITCALL1(STG_REGISTER, cfg_register_keywords, &cfg_kws);
826
Christopher Faulet92d36382015-11-05 13:35:03 +0100827/* Declare the filter parser for "compression" keyword */
828static struct flt_kw_list filter_kws = { "COMP", { }, {
Thierry Fournier3610c392016-04-13 18:27:51 +0200829 { "compression", parse_http_comp_flt, NULL },
830 { NULL, NULL, NULL },
Christopher Faulet92d36382015-11-05 13:35:03 +0100831 }
832};
833
Willy Tarreau0108d902018-11-25 19:14:37 +0100834INITCALL1(STG_REGISTER, flt_register_keywords, &filter_kws);
835
Christopher Faulet3d97c902015-12-09 14:59:38 +0100836/* Note: must not be declared <const> as its list will be overwritten */
837static struct sample_fetch_kw_list sample_fetch_keywords = {ILH, {
Christopher Faulet92d36382015-11-05 13:35:03 +0100838 { "res.comp", smp_fetch_res_comp, 0, NULL, SMP_T_BOOL, SMP_USE_HRSHP },
839 { "res.comp_algo", smp_fetch_res_comp_algo, 0, NULL, SMP_T_STR, SMP_USE_HRSHP },
840 { /* END */ },
841 }
842};
Christopher Faulet3d97c902015-12-09 14:59:38 +0100843
Willy Tarreau0108d902018-11-25 19:14:37 +0100844INITCALL1(STG_REGISTER, sample_register_fetches, &sample_fetch_keywords);