blob: 5ab6a01c78815ff9707dfd50c2fd00049e6f87f5 [file] [log] [blame]
Christopher Faulet3d97c902015-12-09 14:59:38 +01001/*
2 * Stream filters related variables and functions.
3 *
4 * Copyright (C) 2015 Qualys Inc., Christopher Faulet <cfaulet@qualys.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020013#include <haproxy/api.h>
Willy Tarreau0a3bd392020-06-04 08:52:38 +020014#include <haproxy/compression.h>
Willy Tarreau2741c8c2020-06-02 11:28:02 +020015#include <haproxy/dynbuf.h>
Willy Tarreaucd72d8c2020-06-02 19:11:26 +020016#include <haproxy/http.h>
Willy Tarreau87735332020-06-04 09:08:41 +020017#include <haproxy/http_htx.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010018#include <common/cfgparse.h>
Willy Tarreau16f958c2020-06-03 08:44:35 +020019#include <haproxy/htx.h>
Willy Tarreau853b2972020-05-27 18:01:47 +020020#include <haproxy/list.h>
Willy Tarreaue6ce10b2020-06-04 15:33:47 +020021#include <haproxy/sample.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020022#include <haproxy/tools.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010023
Christopher Faulet3d97c902015-12-09 14:59:38 +010024#include <types/filters.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010025#include <types/proxy.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010026
Christopher Faulet92d36382015-11-05 13:35:03 +010027#include <proto/filters.h>
Christopher Fauletfc9cfe42019-07-16 14:54:53 +020028#include <proto/http_ana.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010029#include <proto/stream.h>
30
Christopher Fauletf4a4ef72018-12-07 17:39:53 +010031const char *http_comp_flt_id = "compression filter";
Christopher Faulet92d36382015-11-05 13:35:03 +010032
33struct flt_ops comp_ops;
34
Christopher Faulet92d36382015-11-05 13:35:03 +010035struct comp_state {
36 struct comp_ctx *comp_ctx; /* compression context */
37 struct comp_algo *comp_algo; /* compression algorithm if not NULL */
Christopher Faulet92d36382015-11-05 13:35:03 +010038};
39
Willy Tarreau8ceae722018-11-26 11:58:30 +010040/* Pools used to allocate comp_state structs */
41DECLARE_STATIC_POOL(pool_head_comp_state, "comp_state", sizeof(struct comp_state));
42
43static THREAD_LOCAL struct buffer tmpbuf;
44static THREAD_LOCAL struct buffer zbuf;
Willy Tarreau8ceae722018-11-26 11:58:30 +010045
Christopher Faulet92d36382015-11-05 13:35:03 +010046static int select_compression_request_header(struct comp_state *st,
47 struct stream *s,
48 struct http_msg *msg);
49static int select_compression_response_header(struct comp_state *st,
50 struct stream *s,
51 struct http_msg *msg);
Christopher Faulet27d93c32018-12-15 22:32:02 +010052static int set_compression_response_header(struct comp_state *st,
53 struct stream *s,
54 struct http_msg *msg);
Christopher Faulet92d36382015-11-05 13:35:03 +010055
Christopher Faulete6902cd2018-11-30 22:29:48 +010056static int htx_compression_buffer_init(struct htx *htx, struct buffer *out);
57static int htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
58 struct buffer *out);
59static int htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end);
60
Christopher Faulet92d36382015-11-05 13:35:03 +010061/***********************************************************************/
62static int
Christopher Faulete6902cd2018-11-30 22:29:48 +010063comp_flt_init(struct proxy *px, struct flt_conf *fconf)
64{
Christopher Faulet6e540952018-12-03 22:43:41 +010065 fconf->flags |= FLT_CFG_FL_HTX;
Christopher Faulete6902cd2018-11-30 22:29:48 +010066 return 0;
67}
68
69static int
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020070comp_flt_init_per_thread(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010071{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020072 if (!tmpbuf.size && b_alloc(&tmpbuf) == NULL)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010073 return -1;
Willy Tarreauc9fa0482018-07-10 17:43:27 +020074 if (!zbuf.size && b_alloc(&zbuf) == NULL)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010075 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +010076 return 0;
77}
78
79static void
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020080comp_flt_deinit_per_thread(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010081{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020082 if (tmpbuf.size)
Christopher Faulet92d36382015-11-05 13:35:03 +010083 b_free(&tmpbuf);
Willy Tarreauc9fa0482018-07-10 17:43:27 +020084 if (zbuf.size)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010085 b_free(&zbuf);
Christopher Faulet92d36382015-11-05 13:35:03 +010086}
87
88static int
Christopher Faulet5e896512020-03-06 14:59:05 +010089comp_strm_init(struct stream *s, struct filter *filter)
Christopher Faulet92d36382015-11-05 13:35:03 +010090{
Christopher Faulet5e896512020-03-06 14:59:05 +010091 struct comp_state *st;
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020092
Christopher Faulet5e896512020-03-06 14:59:05 +010093 st = pool_alloc_dirty(pool_head_comp_state);
94 if (st == NULL)
95 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +010096
Christopher Faulet5e896512020-03-06 14:59:05 +010097 st->comp_algo = NULL;
98 st->comp_ctx = NULL;
99 filter->ctx = st;
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200100
Christopher Faulet5e896512020-03-06 14:59:05 +0100101 /* Register post-analyzer on AN_RES_WAIT_HTTP because we need to
102 * analyze response headers before http-response rules execution
103 * to be sure we can use res.comp and res.comp_algo sample
104 * fetches */
105 filter->post_analyzers |= AN_RES_WAIT_HTTP;
Christopher Faulet92d36382015-11-05 13:35:03 +0100106 return 1;
107}
108
Christopher Faulet5e896512020-03-06 14:59:05 +0100109static void
110comp_strm_deinit(struct stream *s, struct filter *filter)
Christopher Faulet92d36382015-11-05 13:35:03 +0100111{
112 struct comp_state *st = filter->ctx;
Christopher Faulet92d36382015-11-05 13:35:03 +0100113
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200114 if (!st)
Christopher Faulet5e896512020-03-06 14:59:05 +0100115 return;
Christopher Faulet92d36382015-11-05 13:35:03 +0100116
Christopher Faulet92d36382015-11-05 13:35:03 +0100117 /* release any possible compression context */
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200118 if (st->comp_algo)
119 st->comp_algo->end(&st->comp_ctx);
Willy Tarreaubafbe012017-11-24 17:34:44 +0100120 pool_free(pool_head_comp_state, st);
Christopher Faulet92d36382015-11-05 13:35:03 +0100121 filter->ctx = NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100122}
123
124static int
Christopher Faulet1339d742016-05-11 16:48:33 +0200125comp_http_headers(struct stream *s, struct filter *filter, struct http_msg *msg)
126{
127 struct comp_state *st = filter->ctx;
128
129 if (!strm_fe(s)->comp && !s->be->comp)
130 goto end;
131
132 if (!(msg->chn->flags & CF_ISRESP))
133 select_compression_request_header(st, s, msg);
134 else {
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200135 /* Response headers have already been checked in
136 * comp_http_post_analyze callback. */
Christopher Faulet1339d742016-05-11 16:48:33 +0200137 if (st->comp_algo) {
Christopher Faulet27d93c32018-12-15 22:32:02 +0100138 if (!set_compression_response_header(st, s, msg))
139 goto end;
Christopher Faulet1339d742016-05-11 16:48:33 +0200140 register_data_filter(s, msg->chn, filter);
Christopher Faulet1339d742016-05-11 16:48:33 +0200141 }
142 }
143
144 end:
145 return 1;
146}
147
148static int
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200149comp_http_post_analyze(struct stream *s, struct filter *filter,
150 struct channel *chn, unsigned an_bit)
151{
152 struct http_txn *txn = s->txn;
153 struct http_msg *msg = &txn->rsp;
154 struct comp_state *st = filter->ctx;
155
156 if (an_bit != AN_RES_WAIT_HTTP)
157 goto end;
158
159 if (!strm_fe(s)->comp && !s->be->comp)
160 goto end;
161
162 select_compression_response_header(st, s, msg);
163
164 end:
165 return 1;
166}
167
168static int
Christopher Faulete6902cd2018-11-30 22:29:48 +0100169comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg,
170 unsigned int offset, unsigned int len)
171{
172 struct comp_state *st = filter->ctx;
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100173 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6a62bf2020-03-02 16:20:05 +0100174 struct htx_ret htxret = htx_find_offset(htx, offset);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100175 struct htx_blk *blk;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100176 int ret, consumed = 0, to_forward = 0;
177
Christopher Faulete6a62bf2020-03-02 16:20:05 +0100178 blk = htxret.blk;
179 offset = htxret.ret;
180 for (; blk && len; blk = htx_get_next_blk(htx, blk)) {
Christopher Faulete6902cd2018-11-30 22:29:48 +0100181 enum htx_blk_type type = htx_get_blk_type(blk);
182 uint32_t sz = htx_get_blksz(blk);
183 struct ist v;
184
185 switch (type) {
186 case HTX_BLK_UNUSED:
187 break;
188
189 case HTX_BLK_DATA:
190 v = htx_get_blk_value(htx, blk);
191 v.ptr += offset;
192 v.len -= offset;
193 if (v.len > len)
194 v.len = len;
195 if (htx_compression_buffer_init(htx, &trash) < 0) {
196 msg->chn->flags |= CF_WAKE_WRITE;
197 goto end;
198 }
199 ret = htx_compression_buffer_add_data(st, v.ptr, v.len, &trash);
200 if (ret < 0)
201 goto error;
202 if (htx_compression_buffer_end(st, &trash, 0) < 0)
203 goto error;
204 len -= ret;
205 consumed += ret;
206 to_forward += b_data(&trash);
207 if (ret == sz && !b_data(&trash)) {
208 offset = 0;
209 blk = htx_remove_blk(htx, blk);
210 continue;
211 }
212 v.len = ret;
213 blk = htx_replace_blk_value(htx, blk, v, ist2(b_head(&trash), b_data(&trash)));
214 break;
215
Christopher Faulete6902cd2018-11-30 22:29:48 +0100216 case HTX_BLK_TLR:
Christopher Faulet2d7c5392019-06-03 10:41:26 +0200217 case HTX_BLK_EOT:
Christopher Faulete6902cd2018-11-30 22:29:48 +0100218 case HTX_BLK_EOM:
219 if (msg->flags & HTTP_MSGF_COMPRESSING) {
220 if (htx_compression_buffer_init(htx, &trash) < 0) {
221 msg->chn->flags |= CF_WAKE_WRITE;
222 goto end;
223 }
224 if (htx_compression_buffer_end(st, &trash, 1) < 0)
225 goto error;
Christopher Fauletd238ae32018-12-21 15:10:25 +0100226 if (b_data(&trash)) {
Christopher Faulet86bc8df2019-06-11 10:38:38 +0200227 struct htx_blk *last = htx_add_last_data(htx, ist2(b_head(&trash), b_data(&trash)));
228 if (!last)
229 goto error;
230 blk = htx_get_next_blk(htx, last);
Christopher Fauletd238ae32018-12-21 15:10:25 +0100231 if (!blk)
232 goto error;
233 to_forward += b_data(&trash);
234 }
Christopher Faulete6902cd2018-11-30 22:29:48 +0100235 msg->flags &= ~HTTP_MSGF_COMPRESSING;
236 /* We let the mux add last empty chunk and empty trailers */
237 }
238 /* fall through */
239
240 default:
241 sz -= offset;
242 if (sz > len)
243 sz = len;
244 consumed += sz;
245 to_forward += sz;
246 len -= sz;
247 break;
248 }
249
250 offset = 0;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100251 }
252
253 end:
254 if (to_forward != consumed)
255 flt_update_offsets(filter, msg->chn, to_forward - consumed);
256
257 if (st->comp_ctx && st->comp_ctx->cur_lvl > 0) {
Willy Tarreauef6fd852019-02-04 11:48:03 +0100258 update_freq_ctr(&global.comp_bps_in, consumed);
Olivier Houchard43da3432019-03-08 18:50:27 +0100259 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_in, consumed);
260 _HA_ATOMIC_ADD(&s->be->be_counters.comp_in, consumed);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100261 update_freq_ctr(&global.comp_bps_out, to_forward);
Olivier Houchard43da3432019-03-08 18:50:27 +0100262 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_out, to_forward);
263 _HA_ATOMIC_ADD(&s->be->be_counters.comp_out, to_forward);
Willy Tarreauef6fd852019-02-04 11:48:03 +0100264 } else {
Olivier Houchard43da3432019-03-08 18:50:27 +0100265 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_byp, consumed);
266 _HA_ATOMIC_ADD(&s->be->be_counters.comp_byp, consumed);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100267 }
268 return to_forward;
269
270 error:
271 return -1;
272}
273
Christopher Faulet2fb28802015-12-01 10:40:57 +0100274
Christopher Faulet92d36382015-11-05 13:35:03 +0100275static int
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200276comp_http_end(struct stream *s, struct filter *filter,
277 struct http_msg *msg)
278{
279 struct comp_state *st = filter->ctx;
280
281 if (!(msg->chn->flags & CF_ISRESP) || !st || !st->comp_algo)
282 goto end;
283
284 if (strm_fe(s)->mode == PR_MODE_HTTP)
Olivier Houchard43da3432019-03-08 18:50:27 +0100285 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.p.http.comp_rsp, 1);
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200286 if ((s->flags & SF_BE_ASSIGNED) && (s->be->mode == PR_MODE_HTTP))
Olivier Houchard43da3432019-03-08 18:50:27 +0100287 _HA_ATOMIC_ADD(&s->be->be_counters.p.http.comp_rsp, 1);
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200288 end:
289 return 1;
290}
Christopher Faulet27d93c32018-12-15 22:32:02 +0100291
Christopher Faulet89f2b162019-07-15 21:16:04 +0200292/***********************************************************************/
Christopher Faulet27d93c32018-12-15 22:32:02 +0100293static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200294set_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulet27d93c32018-12-15 22:32:02 +0100295{
296 struct htx *htx = htxbuf(&msg->chn->buf);
Tim Duesterhusb229f012019-01-29 16:38:56 +0100297 struct http_hdr_ctx ctx;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100298
299 /*
300 * Add Content-Encoding header when it's not identity encoding.
301 * RFC 2616 : Identity encoding: This content-coding is used only in the
302 * Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding
303 * header.
304 */
305 if (st->comp_algo->cfg_name_len != 8 || memcmp(st->comp_algo->cfg_name, "identity", 8) != 0) {
306 struct ist v = ist2(st->comp_algo->ua_name, st->comp_algo->ua_name_len);
307
308 if (!http_add_header(htx, ist("Content-Encoding"), v))
309 goto error;
310 }
311
312 /* remove Content-Length header */
313 if (msg->flags & HTTP_MSGF_CNT_LEN) {
Christopher Faulet27d93c32018-12-15 22:32:02 +0100314 ctx.blk = NULL;
315 while (http_find_header(htx, ist("Content-Length"), &ctx, 1))
316 http_remove_header(htx, &ctx);
317 }
318
319 /* add "Transfer-Encoding: chunked" header */
320 if (!(msg->flags & HTTP_MSGF_TE_CHNK)) {
321 if (!http_add_header(htx, ist("Transfer-Encoding"), ist("chunked")))
322 goto error;
323 }
324
Tim Duesterhusb229f012019-01-29 16:38:56 +0100325 /* convert "ETag" header to a weak ETag */
326 ctx.blk = NULL;
327 if (http_find_header(htx, ist("ETag"), &ctx, 1)) {
328 if (ctx.value.ptr[0] == '"') {
329 /* This a strong ETag. Convert it to a weak one. */
330 struct ist v = ist2(trash.area, 0);
331 if (istcat(&v, ist("W/"), trash.size) == -1 || istcat(&v, ctx.value, trash.size) == -1)
332 goto error;
333
334 if (!http_replace_header_value(htx, &ctx, v))
335 goto error;
336 }
337 }
338
Tim Duesterhus721d6862019-06-17 16:10:07 +0200339 if (!http_add_header(htx, ist("Vary"), ist("Accept-Encoding")))
340 goto error;
341
Christopher Faulet27d93c32018-12-15 22:32:02 +0100342 return 1;
343
344 error:
345 st->comp_algo->end(&st->comp_ctx);
346 st->comp_algo = NULL;
347 return 0;
348}
349
Christopher Faulet3d97c902015-12-09 14:59:38 +0100350/*
351 * Selects a compression algorithm depending on the client request.
352 */
Christopher Faulete6902cd2018-11-30 22:29:48 +0100353static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200354select_compression_request_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100355{
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100356 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100357 struct http_hdr_ctx ctx;
358 struct comp_algo *comp_algo = NULL;
359 struct comp_algo *comp_algo_back = NULL;
360
361 /* Disable compression for older user agents announcing themselves as "Mozilla/4"
362 * unless they are known good (MSIE 6 with XP SP2, or MSIE 7 and later).
363 * See http://zoompf.com/2012/02/lose-the-wait-http-compression for more details.
364 */
365 ctx.blk = NULL;
366 if (http_find_header(htx, ist("User-Agent"), &ctx, 1) &&
367 ctx.value.len >= 9 &&
368 memcmp(ctx.value.ptr, "Mozilla/4", 9) == 0 &&
369 (ctx.value.len < 31 ||
370 memcmp(ctx.value.ptr + 25, "MSIE ", 5) != 0 ||
371 *(ctx.value.ptr + 30) < '6' ||
372 (*(ctx.value.ptr + 30) == '6' &&
373 (ctx.value.len < 54 || memcmp(ctx.value.ptr + 51, "SV1", 3) != 0)))) {
374 st->comp_algo = NULL;
375 return 0;
376 }
377
378 /* search for the algo in the backend in priority or the frontend */
379 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
380 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
381 int best_q = 0;
382
383 ctx.blk = NULL;
384 while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 0)) {
385 const char *qval;
386 int q;
387 int toklen;
388
389 /* try to isolate the token from the optional q-value */
390 toklen = 0;
391 while (toklen < ctx.value.len && HTTP_IS_TOKEN(*(ctx.value.ptr + toklen)))
392 toklen++;
393
394 qval = ctx.value.ptr + toklen;
395 while (1) {
396 while (qval < ctx.value.ptr + ctx.value.len && HTTP_IS_LWS(*qval))
397 qval++;
398
399 if (qval >= ctx.value.ptr + ctx.value.len || *qval != ';') {
400 qval = NULL;
401 break;
402 }
403 qval++;
404
405 while (qval < ctx.value.ptr + ctx.value.len && HTTP_IS_LWS(*qval))
406 qval++;
407
408 if (qval >= ctx.value.ptr + ctx.value.len) {
409 qval = NULL;
410 break;
411 }
412 if (strncmp(qval, "q=", MIN(ctx.value.ptr + ctx.value.len - qval, 2)) == 0)
413 break;
414
415 while (qval < ctx.value.ptr + ctx.value.len && *qval != ';')
416 qval++;
417 }
418
419 /* here we have qval pointing to the first "q=" attribute or NULL if not found */
420 q = qval ? http_parse_qvalue(qval + 2, NULL) : 1000;
421
422 if (q <= best_q)
423 continue;
424
425 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
426 if (*(ctx.value.ptr) == '*' ||
427 word_match(ctx.value.ptr, toklen, comp_algo->ua_name, comp_algo->ua_name_len)) {
428 st->comp_algo = comp_algo;
429 best_q = q;
430 break;
431 }
432 }
433 }
434 }
435
436 /* remove all occurrences of the header when "compression offload" is set */
437 if (st->comp_algo) {
438 if ((s->be->comp && s->be->comp->offload) ||
439 (strm_fe(s)->comp && strm_fe(s)->comp->offload)) {
440 http_remove_header(htx, &ctx);
441 ctx.blk = NULL;
442 while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 1))
443 http_remove_header(htx, &ctx);
444 }
Christopher Faulet3d97c902015-12-09 14:59:38 +0100445 return 1;
446 }
447
448 /* identity is implicit does not require headers */
Christopher Faulet92d36382015-11-05 13:35:03 +0100449 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
450 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100451 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
452 if (comp_algo->cfg_name_len == 8 && memcmp(comp_algo->cfg_name, "identity", 8) == 0) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100453 st->comp_algo = comp_algo;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100454 return 1;
455 }
456 }
457 }
458
Christopher Faulet92d36382015-11-05 13:35:03 +0100459 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100460 return 0;
461}
462
463/*
464 * Selects a comression algorithm depending of the server response.
465 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100466static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200467select_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100468{
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100469 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100470 struct http_txn *txn = s->txn;
471 struct http_hdr_ctx ctx;
472 struct comp_type *comp_type;
473
474 /* no common compression algorithm was found in request header */
475 if (st->comp_algo == NULL)
476 goto fail;
477
Christopher Faulet1d3613a2019-01-07 14:41:59 +0100478 /* compression already in progress */
479 if (msg->flags & HTTP_MSGF_COMPRESSING)
480 goto fail;
481
Christopher Faulete6902cd2018-11-30 22:29:48 +0100482 /* HTTP < 1.1 should not be compressed */
483 if (!(msg->flags & HTTP_MSGF_VER_11) || !(txn->req.flags & HTTP_MSGF_VER_11))
484 goto fail;
485
486 if (txn->meth == HTTP_METH_HEAD)
487 goto fail;
488
489 /* compress 200,201,202,203 responses only */
490 if ((txn->status != 200) &&
491 (txn->status != 201) &&
492 (txn->status != 202) &&
493 (txn->status != 203))
494 goto fail;
495
Christopher Fauletc963eb22018-12-21 14:53:54 +0100496 if (!(msg->flags & HTTP_MSGF_XFER_LEN) || msg->flags & HTTP_MSGF_BODYLESS)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100497 goto fail;
498
499 /* content is already compressed */
500 ctx.blk = NULL;
501 if (http_find_header(htx, ist("Content-Encoding"), &ctx, 1))
502 goto fail;
503
504 /* no compression when Cache-Control: no-transform is present in the message */
505 ctx.blk = NULL;
506 while (http_find_header(htx, ist("Cache-Control"), &ctx, 0)) {
507 if (word_match(ctx.value.ptr, ctx.value.len, "no-transform", 12))
508 goto fail;
509 }
510
Tim Duesterhusb229f012019-01-29 16:38:56 +0100511 /* no compression when ETag is malformed */
512 ctx.blk = NULL;
513 if (http_find_header(htx, ist("ETag"), &ctx, 1)) {
514 if (!(((ctx.value.len >= 4 && memcmp(ctx.value.ptr, "W/\"", 3) == 0) || /* Either a weak ETag */
515 (ctx.value.len >= 2 && ctx.value.ptr[0] == '"')) && /* or strong ETag */
516 ctx.value.ptr[ctx.value.len - 1] == '"')) {
517 goto fail;
518 }
519 }
520 /* no compression when multiple ETags are present
521 * Note: Do not reset ctx.blk!
522 */
523 if (http_find_header(htx, ist("ETag"), &ctx, 1))
524 goto fail;
525
Christopher Faulete6902cd2018-11-30 22:29:48 +0100526 comp_type = NULL;
527
528 /* we don't want to compress multipart content-types, nor content-types that are
529 * not listed in the "compression type" directive if any. If no content-type was
530 * found but configuration requires one, we don't compress either. Backend has
531 * the priority.
532 */
533 ctx.blk = NULL;
534 if (http_find_header(htx, ist("Content-Type"), &ctx, 1)) {
535 if (ctx.value.len >= 9 && strncasecmp("multipart", ctx.value.ptr, 9) == 0)
536 goto fail;
537
538 if ((s->be->comp && (comp_type = s->be->comp->types)) ||
539 (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types))) {
540 for (; comp_type; comp_type = comp_type->next) {
541 if (ctx.value.len >= comp_type->name_len &&
542 strncasecmp(ctx.value.ptr, comp_type->name, comp_type->name_len) == 0)
543 /* this Content-Type should be compressed */
544 break;
545 }
546 /* this Content-Type should not be compressed */
547 if (comp_type == NULL)
548 goto fail;
549 }
550 }
551 else { /* no content-type header */
552 if ((s->be->comp && s->be->comp->types) ||
553 (strm_fe(s)->comp && strm_fe(s)->comp->types))
554 goto fail; /* a content-type was required */
555 }
556
557 /* limit compression rate */
558 if (global.comp_rate_lim > 0)
559 if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim)
560 goto fail;
561
562 /* limit cpu usage */
Willy Tarreau81036f22019-05-20 19:24:50 +0200563 if (ti->idle_pct < compress_min_idle)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100564 goto fail;
565
566 /* initialize compression */
567 if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0)
568 goto fail;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100569 msg->flags |= HTTP_MSGF_COMPRESSING;
570 return 1;
571
Christopher Faulete6902cd2018-11-30 22:29:48 +0100572 fail:
573 st->comp_algo = NULL;
574 return 0;
575}
576
Christopher Faulet3d97c902015-12-09 14:59:38 +0100577/***********************************************************************/
Christopher Faulete6902cd2018-11-30 22:29:48 +0100578static int
579htx_compression_buffer_init(struct htx *htx, struct buffer *out)
580{
581 /* output stream requires at least 10 bytes for the gzip header, plus
582 * at least 8 bytes for the gzip trailer (crc+len), plus a possible
583 * plus at most 5 bytes per 32kB block and 2 bytes to close the stream.
584 */
585 if (htx_free_space(htx) < 20 + 5 * ((htx->data + 32767) >> 15))
586 return -1;
587 b_reset(out);
588 return 0;
589}
590
Christopher Faulete6902cd2018-11-30 22:29:48 +0100591static int
592htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
593 struct buffer *out)
594{
595 return st->comp_algo->add_data(st->comp_ctx, data, len, out);
596}
597
Christopher Faulete6902cd2018-11-30 22:29:48 +0100598static int
599htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end)
600{
601 if (end)
602 return st->comp_algo->finish(st->comp_ctx, out);
603 else
604 return st->comp_algo->flush(st->comp_ctx, out);
605}
606
Christopher Faulet3d97c902015-12-09 14:59:38 +0100607
608/***********************************************************************/
Christopher Faulet92d36382015-11-05 13:35:03 +0100609struct flt_ops comp_ops = {
Christopher Faulete6902cd2018-11-30 22:29:48 +0100610 .init = comp_flt_init,
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +0200611 .init_per_thread = comp_flt_init_per_thread,
612 .deinit_per_thread = comp_flt_deinit_per_thread,
Christopher Faulet92d36382015-11-05 13:35:03 +0100613
Christopher Faulet5e896512020-03-06 14:59:05 +0100614 .attach = comp_strm_init,
615 .detach = comp_strm_deinit,
616
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200617 .channel_post_analyze = comp_http_post_analyze,
Christopher Faulet92d36382015-11-05 13:35:03 +0100618
Christopher Faulet1339d742016-05-11 16:48:33 +0200619 .http_headers = comp_http_headers,
Christopher Faulete6902cd2018-11-30 22:29:48 +0100620 .http_payload = comp_http_payload,
621 .http_end = comp_http_end,
Christopher Faulet92d36382015-11-05 13:35:03 +0100622};
623
Christopher Faulet3d97c902015-12-09 14:59:38 +0100624static int
625parse_compression_options(char **args, int section, struct proxy *proxy,
626 struct proxy *defpx, const char *file, int line,
627 char **err)
628{
Christopher Faulet92d36382015-11-05 13:35:03 +0100629 struct comp *comp;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100630
631 if (proxy->comp == NULL) {
Vincent Bernat02779b62016-04-03 13:48:43 +0200632 comp = calloc(1, sizeof(*comp));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100633 proxy->comp = comp;
634 }
635 else
636 comp = proxy->comp;
637
638 if (!strcmp(args[1], "algo")) {
639 struct comp_ctx *ctx;
640 int cur_arg = 2;
641
642 if (!*args[cur_arg]) {
643 memprintf(err, "parsing [%s:%d] : '%s' expects <algorithm>\n",
644 file, line, args[0]);
645 return -1;
646 }
647 while (*(args[cur_arg])) {
648 if (comp_append_algo(comp, args[cur_arg]) < 0) {
649 memprintf(err, "'%s' : '%s' is not a supported algorithm.\n",
650 args[0], args[cur_arg]);
651 return -1;
652 }
653 if (proxy->comp->algos->init(&ctx, 9) == 0)
654 proxy->comp->algos->end(&ctx);
655 else {
656 memprintf(err, "'%s' : Can't init '%s' algorithm.\n",
657 args[0], args[cur_arg]);
658 return -1;
659 }
660 cur_arg++;
661 continue;
662 }
663 }
664 else if (!strcmp(args[1], "offload"))
665 comp->offload = 1;
666 else if (!strcmp(args[1], "type")) {
667 int cur_arg = 2;
668
669 if (!*args[cur_arg]) {
670 memprintf(err, "'%s' expects <type>\n", args[0]);
671 return -1;
672 }
673 while (*(args[cur_arg])) {
674 comp_append_type(comp, args[cur_arg]);
675 cur_arg++;
676 continue;
677 }
678 }
679 else {
680 memprintf(err, "'%s' expects 'algo', 'type' or 'offload'\n",
681 args[0]);
682 return -1;
683 }
684
685 return 0;
686}
687
Christopher Faulet92d36382015-11-05 13:35:03 +0100688static int
689parse_http_comp_flt(char **args, int *cur_arg, struct proxy *px,
Thierry Fournier3610c392016-04-13 18:27:51 +0200690 struct flt_conf *fconf, char **err, void *private)
Christopher Faulet92d36382015-11-05 13:35:03 +0100691{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100692 struct flt_conf *fc, *back;
Christopher Faulet92d36382015-11-05 13:35:03 +0100693
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100694 list_for_each_entry_safe(fc, back, &px->filter_configs, list) {
695 if (fc->id == http_comp_flt_id) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100696 memprintf(err, "%s: Proxy supports only one compression filter\n", px->id);
697 return -1;
698 }
699 }
700
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100701 fconf->id = http_comp_flt_id;
702 fconf->conf = NULL;
703 fconf->ops = &comp_ops;
Christopher Faulet92d36382015-11-05 13:35:03 +0100704 (*cur_arg)++;
705
706 return 0;
707}
708
709
710int
Christopher Fauletc9df7f72018-12-10 16:14:04 +0100711check_implicit_http_comp_flt(struct proxy *proxy)
Christopher Faulet92d36382015-11-05 13:35:03 +0100712{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100713 struct flt_conf *fconf;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100714 int explicit = 0;
715 int comp = 0;
Christopher Faulet92d36382015-11-05 13:35:03 +0100716 int err = 0;
717
718 if (proxy->comp == NULL)
719 goto end;
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100720 if (!LIST_ISEMPTY(&proxy->filter_configs)) {
721 list_for_each_entry(fconf, &proxy->filter_configs, list) {
722 if (fconf->id == http_comp_flt_id)
Christopher Faulet27d93c32018-12-15 22:32:02 +0100723 comp = 1;
724 else if (fconf->id == cache_store_flt_id) {
725 if (comp) {
726 ha_alert("config: %s '%s': unable to enable the compression filter "
727 "before any cache filter.\n",
728 proxy_type_str(proxy), proxy->id);
729 err++;
730 goto end;
731 }
732 }
Christopher Faulet78fbb9f2019-08-11 23:11:03 +0200733 else if (fconf->id == fcgi_flt_id)
734 continue;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100735 else
736 explicit = 1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100737 }
Christopher Faulet27d93c32018-12-15 22:32:02 +0100738 }
739 if (comp)
740 goto end;
741 else if (explicit) {
742 ha_alert("config: %s '%s': require an explicit filter declaration to use "
743 "HTTP compression\n", proxy_type_str(proxy), proxy->id);
Christopher Faulet92d36382015-11-05 13:35:03 +0100744 err++;
745 goto end;
746 }
747
Christopher Faulet27d93c32018-12-15 22:32:02 +0100748 /* Implicit declaration of the compression filter is always the last
749 * one */
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100750 fconf = calloc(1, sizeof(*fconf));
751 if (!fconf) {
Christopher Faulet767a84b2017-11-24 16:50:31 +0100752 ha_alert("config: %s '%s': out of memory\n",
753 proxy_type_str(proxy), proxy->id);
Christopher Faulet92d36382015-11-05 13:35:03 +0100754 err++;
755 goto end;
756 }
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100757 fconf->id = http_comp_flt_id;
758 fconf->conf = NULL;
759 fconf->ops = &comp_ops;
760 LIST_ADDQ(&proxy->filter_configs, &fconf->list);
Christopher Faulet92d36382015-11-05 13:35:03 +0100761 end:
762 return err;
763}
764
765/*
766 * boolean, returns true if compression is used (either gzip or deflate) in the
767 * response.
768 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100769static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100770smp_fetch_res_comp(const struct arg *args, struct sample *smp, const char *kw,
771 void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100772{
Willy Tarreaube508f12016-03-10 11:47:01 +0100773 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100774
Christopher Faulet3d97c902015-12-09 14:59:38 +0100775 smp->data.type = SMP_T_BOOL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100776 smp->data.u.sint = (txn && (txn->rsp.flags & HTTP_MSGF_COMPRESSING));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100777 return 1;
778}
779
Christopher Faulet92d36382015-11-05 13:35:03 +0100780/*
781 * string, returns algo
782 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100783static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100784smp_fetch_res_comp_algo(const struct arg *args, struct sample *smp,
785 const char *kw, void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100786{
Willy Tarreaube508f12016-03-10 11:47:01 +0100787 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100788 struct filter *filter;
789 struct comp_state *st;
790
Christopher Faulet03d85532017-09-15 10:14:43 +0200791 if (!txn || !(txn->rsp.flags & HTTP_MSGF_COMPRESSING))
Christopher Faulet3d97c902015-12-09 14:59:38 +0100792 return 0;
793
Christopher Fauletfcf035c2015-12-03 11:48:03 +0100794 list_for_each_entry(filter, &strm_flt(smp->strm)->filters, list) {
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100795 if (FLT_ID(filter) != http_comp_flt_id)
Christopher Faulet92d36382015-11-05 13:35:03 +0100796 continue;
797
798 if (!(st = filter->ctx))
799 break;
800
801 smp->data.type = SMP_T_STR;
802 smp->flags = SMP_F_CONST;
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200803 smp->data.u.str.area = st->comp_algo->cfg_name;
804 smp->data.u.str.data = st->comp_algo->cfg_name_len;
Christopher Faulet92d36382015-11-05 13:35:03 +0100805 return 1;
806 }
807 return 0;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100808}
809
810/* Declare the config parser for "compression" keyword */
811static struct cfg_kw_list cfg_kws = {ILH, {
812 { CFG_LISTEN, "compression", parse_compression_options },
813 { 0, NULL, NULL },
814 }
815};
816
Willy Tarreau0108d902018-11-25 19:14:37 +0100817INITCALL1(STG_REGISTER, cfg_register_keywords, &cfg_kws);
818
Christopher Faulet92d36382015-11-05 13:35:03 +0100819/* Declare the filter parser for "compression" keyword */
820static struct flt_kw_list filter_kws = { "COMP", { }, {
Thierry Fournier3610c392016-04-13 18:27:51 +0200821 { "compression", parse_http_comp_flt, NULL },
822 { NULL, NULL, NULL },
Christopher Faulet92d36382015-11-05 13:35:03 +0100823 }
824};
825
Willy Tarreau0108d902018-11-25 19:14:37 +0100826INITCALL1(STG_REGISTER, flt_register_keywords, &filter_kws);
827
Christopher Faulet3d97c902015-12-09 14:59:38 +0100828/* Note: must not be declared <const> as its list will be overwritten */
829static struct sample_fetch_kw_list sample_fetch_keywords = {ILH, {
Christopher Faulet92d36382015-11-05 13:35:03 +0100830 { "res.comp", smp_fetch_res_comp, 0, NULL, SMP_T_BOOL, SMP_USE_HRSHP },
831 { "res.comp_algo", smp_fetch_res_comp_algo, 0, NULL, SMP_T_STR, SMP_USE_HRSHP },
832 { /* END */ },
833 }
834};
Christopher Faulet3d97c902015-12-09 14:59:38 +0100835
Willy Tarreau0108d902018-11-25 19:14:37 +0100836INITCALL1(STG_REGISTER, sample_register_fetches, &sample_fetch_keywords);