blob: dc7898472b1ae501b68d3564a1697aa4032ac729 [file] [log] [blame]
Christopher Faulet3d97c902015-12-09 14:59:38 +01001/*
2 * Stream filters related variables and functions.
3 *
4 * Copyright (C) 2015 Qualys Inc., Christopher Faulet <cfaulet@qualys.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
13#include <common/buffer.h>
14#include <common/cfgparse.h>
Willy Tarreaub96b77e2018-12-11 10:22:41 +010015#include <common/htx.h>
Willy Tarreau0108d902018-11-25 19:14:37 +010016#include <common/initcall.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010017#include <common/mini-clist.h>
18#include <common/standard.h>
19
20#include <types/compression.h>
21#include <types/filters.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010022#include <types/proxy.h>
23#include <types/sample.h>
24
25#include <proto/compression.h>
Christopher Faulet92d36382015-11-05 13:35:03 +010026#include <proto/filters.h>
Christopher Faulete6902cd2018-11-30 22:29:48 +010027#include <proto/http_htx.h>
Christopher Fauletfc9cfe42019-07-16 14:54:53 +020028#include <proto/http_ana.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010029#include <proto/sample.h>
30#include <proto/stream.h>
31
Christopher Fauletf4a4ef72018-12-07 17:39:53 +010032const char *http_comp_flt_id = "compression filter";
Christopher Faulet92d36382015-11-05 13:35:03 +010033
34struct flt_ops comp_ops;
35
Christopher Faulet92d36382015-11-05 13:35:03 +010036struct comp_state {
37 struct comp_ctx *comp_ctx; /* compression context */
38 struct comp_algo *comp_algo; /* compression algorithm if not NULL */
Christopher Faulet92d36382015-11-05 13:35:03 +010039};
40
Willy Tarreau8ceae722018-11-26 11:58:30 +010041/* Pools used to allocate comp_state structs */
42DECLARE_STATIC_POOL(pool_head_comp_state, "comp_state", sizeof(struct comp_state));
43
44static THREAD_LOCAL struct buffer tmpbuf;
45static THREAD_LOCAL struct buffer zbuf;
Willy Tarreau8ceae722018-11-26 11:58:30 +010046
Christopher Faulet92d36382015-11-05 13:35:03 +010047static int select_compression_request_header(struct comp_state *st,
48 struct stream *s,
49 struct http_msg *msg);
50static int select_compression_response_header(struct comp_state *st,
51 struct stream *s,
52 struct http_msg *msg);
Christopher Faulet27d93c32018-12-15 22:32:02 +010053static int set_compression_response_header(struct comp_state *st,
54 struct stream *s,
55 struct http_msg *msg);
Christopher Faulet92d36382015-11-05 13:35:03 +010056
Christopher Faulete6902cd2018-11-30 22:29:48 +010057static int htx_compression_buffer_init(struct htx *htx, struct buffer *out);
58static int htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
59 struct buffer *out);
60static int htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end);
61
Christopher Faulet92d36382015-11-05 13:35:03 +010062/***********************************************************************/
63static int
Christopher Faulete6902cd2018-11-30 22:29:48 +010064comp_flt_init(struct proxy *px, struct flt_conf *fconf)
65{
Christopher Faulet6e540952018-12-03 22:43:41 +010066 fconf->flags |= FLT_CFG_FL_HTX;
Christopher Faulete6902cd2018-11-30 22:29:48 +010067 return 0;
68}
69
70static int
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020071comp_flt_init_per_thread(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010072{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020073 if (!tmpbuf.size && b_alloc(&tmpbuf) == NULL)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010074 return -1;
Willy Tarreauc9fa0482018-07-10 17:43:27 +020075 if (!zbuf.size && b_alloc(&zbuf) == NULL)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010076 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +010077 return 0;
78}
79
80static void
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020081comp_flt_deinit_per_thread(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010082{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020083 if (tmpbuf.size)
Christopher Faulet92d36382015-11-05 13:35:03 +010084 b_free(&tmpbuf);
Willy Tarreauc9fa0482018-07-10 17:43:27 +020085 if (zbuf.size)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010086 b_free(&zbuf);
Christopher Faulet92d36382015-11-05 13:35:03 +010087}
88
89static int
Christopher Faulet5e896512020-03-06 14:59:05 +010090comp_strm_init(struct stream *s, struct filter *filter)
Christopher Faulet92d36382015-11-05 13:35:03 +010091{
Christopher Faulet5e896512020-03-06 14:59:05 +010092 struct comp_state *st;
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020093
Christopher Faulet5e896512020-03-06 14:59:05 +010094 st = pool_alloc_dirty(pool_head_comp_state);
95 if (st == NULL)
96 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +010097
Christopher Faulet5e896512020-03-06 14:59:05 +010098 st->comp_algo = NULL;
99 st->comp_ctx = NULL;
100 filter->ctx = st;
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200101
Christopher Faulet5e896512020-03-06 14:59:05 +0100102 /* Register post-analyzer on AN_RES_WAIT_HTTP because we need to
103 * analyze response headers before http-response rules execution
104 * to be sure we can use res.comp and res.comp_algo sample
105 * fetches */
106 filter->post_analyzers |= AN_RES_WAIT_HTTP;
Christopher Faulet92d36382015-11-05 13:35:03 +0100107 return 1;
108}
109
Christopher Faulet5e896512020-03-06 14:59:05 +0100110static void
111comp_strm_deinit(struct stream *s, struct filter *filter)
Christopher Faulet92d36382015-11-05 13:35:03 +0100112{
113 struct comp_state *st = filter->ctx;
Christopher Faulet92d36382015-11-05 13:35:03 +0100114
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200115 if (!st)
Christopher Faulet5e896512020-03-06 14:59:05 +0100116 return;
Christopher Faulet92d36382015-11-05 13:35:03 +0100117
Christopher Faulet92d36382015-11-05 13:35:03 +0100118 /* release any possible compression context */
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200119 if (st->comp_algo)
120 st->comp_algo->end(&st->comp_ctx);
Willy Tarreaubafbe012017-11-24 17:34:44 +0100121 pool_free(pool_head_comp_state, st);
Christopher Faulet92d36382015-11-05 13:35:03 +0100122 filter->ctx = NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100123}
124
125static int
Christopher Faulet1339d742016-05-11 16:48:33 +0200126comp_http_headers(struct stream *s, struct filter *filter, struct http_msg *msg)
127{
128 struct comp_state *st = filter->ctx;
129
130 if (!strm_fe(s)->comp && !s->be->comp)
131 goto end;
132
133 if (!(msg->chn->flags & CF_ISRESP))
134 select_compression_request_header(st, s, msg);
135 else {
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200136 /* Response headers have already been checked in
137 * comp_http_post_analyze callback. */
Christopher Faulet1339d742016-05-11 16:48:33 +0200138 if (st->comp_algo) {
Christopher Faulet27d93c32018-12-15 22:32:02 +0100139 if (!set_compression_response_header(st, s, msg))
140 goto end;
Christopher Faulet1339d742016-05-11 16:48:33 +0200141 register_data_filter(s, msg->chn, filter);
Christopher Faulet1339d742016-05-11 16:48:33 +0200142 }
143 }
144
145 end:
146 return 1;
147}
148
149static int
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200150comp_http_post_analyze(struct stream *s, struct filter *filter,
151 struct channel *chn, unsigned an_bit)
152{
153 struct http_txn *txn = s->txn;
154 struct http_msg *msg = &txn->rsp;
155 struct comp_state *st = filter->ctx;
156
157 if (an_bit != AN_RES_WAIT_HTTP)
158 goto end;
159
160 if (!strm_fe(s)->comp && !s->be->comp)
161 goto end;
162
163 select_compression_response_header(st, s, msg);
164
165 end:
166 return 1;
167}
168
169static int
Christopher Faulete6902cd2018-11-30 22:29:48 +0100170comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg,
171 unsigned int offset, unsigned int len)
172{
173 struct comp_state *st = filter->ctx;
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100174 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6a62bf2020-03-02 16:20:05 +0100175 struct htx_ret htxret = htx_find_offset(htx, offset);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100176 struct htx_blk *blk;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100177 int ret, consumed = 0, to_forward = 0;
178
Christopher Faulete6a62bf2020-03-02 16:20:05 +0100179 blk = htxret.blk;
180 offset = htxret.ret;
181 for (; blk && len; blk = htx_get_next_blk(htx, blk)) {
Christopher Faulete6902cd2018-11-30 22:29:48 +0100182 enum htx_blk_type type = htx_get_blk_type(blk);
183 uint32_t sz = htx_get_blksz(blk);
184 struct ist v;
185
186 switch (type) {
187 case HTX_BLK_UNUSED:
188 break;
189
190 case HTX_BLK_DATA:
191 v = htx_get_blk_value(htx, blk);
192 v.ptr += offset;
193 v.len -= offset;
194 if (v.len > len)
195 v.len = len;
196 if (htx_compression_buffer_init(htx, &trash) < 0) {
197 msg->chn->flags |= CF_WAKE_WRITE;
198 goto end;
199 }
200 ret = htx_compression_buffer_add_data(st, v.ptr, v.len, &trash);
201 if (ret < 0)
202 goto error;
203 if (htx_compression_buffer_end(st, &trash, 0) < 0)
204 goto error;
205 len -= ret;
206 consumed += ret;
207 to_forward += b_data(&trash);
208 if (ret == sz && !b_data(&trash)) {
209 offset = 0;
210 blk = htx_remove_blk(htx, blk);
211 continue;
212 }
213 v.len = ret;
214 blk = htx_replace_blk_value(htx, blk, v, ist2(b_head(&trash), b_data(&trash)));
215 break;
216
Christopher Faulete6902cd2018-11-30 22:29:48 +0100217 case HTX_BLK_TLR:
Christopher Faulet2d7c5392019-06-03 10:41:26 +0200218 case HTX_BLK_EOT:
Christopher Faulete6902cd2018-11-30 22:29:48 +0100219 case HTX_BLK_EOM:
220 if (msg->flags & HTTP_MSGF_COMPRESSING) {
221 if (htx_compression_buffer_init(htx, &trash) < 0) {
222 msg->chn->flags |= CF_WAKE_WRITE;
223 goto end;
224 }
225 if (htx_compression_buffer_end(st, &trash, 1) < 0)
226 goto error;
Christopher Fauletd238ae32018-12-21 15:10:25 +0100227 if (b_data(&trash)) {
Christopher Faulet86bc8df2019-06-11 10:38:38 +0200228 struct htx_blk *last = htx_add_last_data(htx, ist2(b_head(&trash), b_data(&trash)));
229 if (!last)
230 goto error;
231 blk = htx_get_next_blk(htx, last);
Christopher Fauletd238ae32018-12-21 15:10:25 +0100232 if (!blk)
233 goto error;
234 to_forward += b_data(&trash);
235 }
Christopher Faulete6902cd2018-11-30 22:29:48 +0100236 msg->flags &= ~HTTP_MSGF_COMPRESSING;
237 /* We let the mux add last empty chunk and empty trailers */
238 }
239 /* fall through */
240
241 default:
242 sz -= offset;
243 if (sz > len)
244 sz = len;
245 consumed += sz;
246 to_forward += sz;
247 len -= sz;
248 break;
249 }
250
251 offset = 0;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100252 }
253
254 end:
255 if (to_forward != consumed)
256 flt_update_offsets(filter, msg->chn, to_forward - consumed);
257
258 if (st->comp_ctx && st->comp_ctx->cur_lvl > 0) {
Willy Tarreauef6fd852019-02-04 11:48:03 +0100259 update_freq_ctr(&global.comp_bps_in, consumed);
Olivier Houchard43da3432019-03-08 18:50:27 +0100260 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_in, consumed);
261 _HA_ATOMIC_ADD(&s->be->be_counters.comp_in, consumed);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100262 update_freq_ctr(&global.comp_bps_out, to_forward);
Olivier Houchard43da3432019-03-08 18:50:27 +0100263 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_out, to_forward);
264 _HA_ATOMIC_ADD(&s->be->be_counters.comp_out, to_forward);
Willy Tarreauef6fd852019-02-04 11:48:03 +0100265 } else {
Olivier Houchard43da3432019-03-08 18:50:27 +0100266 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_byp, consumed);
267 _HA_ATOMIC_ADD(&s->be->be_counters.comp_byp, consumed);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100268 }
269 return to_forward;
270
271 error:
272 return -1;
273}
274
Christopher Faulet2fb28802015-12-01 10:40:57 +0100275
Christopher Faulet92d36382015-11-05 13:35:03 +0100276static int
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200277comp_http_end(struct stream *s, struct filter *filter,
278 struct http_msg *msg)
279{
280 struct comp_state *st = filter->ctx;
281
282 if (!(msg->chn->flags & CF_ISRESP) || !st || !st->comp_algo)
283 goto end;
284
285 if (strm_fe(s)->mode == PR_MODE_HTTP)
Olivier Houchard43da3432019-03-08 18:50:27 +0100286 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.p.http.comp_rsp, 1);
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200287 if ((s->flags & SF_BE_ASSIGNED) && (s->be->mode == PR_MODE_HTTP))
Olivier Houchard43da3432019-03-08 18:50:27 +0100288 _HA_ATOMIC_ADD(&s->be->be_counters.p.http.comp_rsp, 1);
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200289 end:
290 return 1;
291}
Christopher Faulet27d93c32018-12-15 22:32:02 +0100292
Christopher Faulet89f2b162019-07-15 21:16:04 +0200293/***********************************************************************/
Christopher Faulet27d93c32018-12-15 22:32:02 +0100294static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200295set_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulet27d93c32018-12-15 22:32:02 +0100296{
297 struct htx *htx = htxbuf(&msg->chn->buf);
Tim Duesterhusb229f012019-01-29 16:38:56 +0100298 struct http_hdr_ctx ctx;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100299
300 /*
301 * Add Content-Encoding header when it's not identity encoding.
302 * RFC 2616 : Identity encoding: This content-coding is used only in the
303 * Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding
304 * header.
305 */
306 if (st->comp_algo->cfg_name_len != 8 || memcmp(st->comp_algo->cfg_name, "identity", 8) != 0) {
307 struct ist v = ist2(st->comp_algo->ua_name, st->comp_algo->ua_name_len);
308
309 if (!http_add_header(htx, ist("Content-Encoding"), v))
310 goto error;
311 }
312
313 /* remove Content-Length header */
314 if (msg->flags & HTTP_MSGF_CNT_LEN) {
Christopher Faulet27d93c32018-12-15 22:32:02 +0100315 ctx.blk = NULL;
316 while (http_find_header(htx, ist("Content-Length"), &ctx, 1))
317 http_remove_header(htx, &ctx);
318 }
319
320 /* add "Transfer-Encoding: chunked" header */
321 if (!(msg->flags & HTTP_MSGF_TE_CHNK)) {
322 if (!http_add_header(htx, ist("Transfer-Encoding"), ist("chunked")))
323 goto error;
324 }
325
Tim Duesterhusb229f012019-01-29 16:38:56 +0100326 /* convert "ETag" header to a weak ETag */
327 ctx.blk = NULL;
328 if (http_find_header(htx, ist("ETag"), &ctx, 1)) {
329 if (ctx.value.ptr[0] == '"') {
330 /* This a strong ETag. Convert it to a weak one. */
331 struct ist v = ist2(trash.area, 0);
332 if (istcat(&v, ist("W/"), trash.size) == -1 || istcat(&v, ctx.value, trash.size) == -1)
333 goto error;
334
335 if (!http_replace_header_value(htx, &ctx, v))
336 goto error;
337 }
338 }
339
Tim Duesterhus721d6862019-06-17 16:10:07 +0200340 if (!http_add_header(htx, ist("Vary"), ist("Accept-Encoding")))
341 goto error;
342
Christopher Faulet27d93c32018-12-15 22:32:02 +0100343 return 1;
344
345 error:
346 st->comp_algo->end(&st->comp_ctx);
347 st->comp_algo = NULL;
348 return 0;
349}
350
Christopher Faulet3d97c902015-12-09 14:59:38 +0100351/*
352 * Selects a compression algorithm depending on the client request.
353 */
Christopher Faulete6902cd2018-11-30 22:29:48 +0100354static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200355select_compression_request_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100356{
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100357 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100358 struct http_hdr_ctx ctx;
359 struct comp_algo *comp_algo = NULL;
360 struct comp_algo *comp_algo_back = NULL;
361
362 /* Disable compression for older user agents announcing themselves as "Mozilla/4"
363 * unless they are known good (MSIE 6 with XP SP2, or MSIE 7 and later).
364 * See http://zoompf.com/2012/02/lose-the-wait-http-compression for more details.
365 */
366 ctx.blk = NULL;
367 if (http_find_header(htx, ist("User-Agent"), &ctx, 1) &&
368 ctx.value.len >= 9 &&
369 memcmp(ctx.value.ptr, "Mozilla/4", 9) == 0 &&
370 (ctx.value.len < 31 ||
371 memcmp(ctx.value.ptr + 25, "MSIE ", 5) != 0 ||
372 *(ctx.value.ptr + 30) < '6' ||
373 (*(ctx.value.ptr + 30) == '6' &&
374 (ctx.value.len < 54 || memcmp(ctx.value.ptr + 51, "SV1", 3) != 0)))) {
375 st->comp_algo = NULL;
376 return 0;
377 }
378
379 /* search for the algo in the backend in priority or the frontend */
380 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
381 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
382 int best_q = 0;
383
384 ctx.blk = NULL;
385 while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 0)) {
386 const char *qval;
387 int q;
388 int toklen;
389
390 /* try to isolate the token from the optional q-value */
391 toklen = 0;
392 while (toklen < ctx.value.len && HTTP_IS_TOKEN(*(ctx.value.ptr + toklen)))
393 toklen++;
394
395 qval = ctx.value.ptr + toklen;
396 while (1) {
397 while (qval < ctx.value.ptr + ctx.value.len && HTTP_IS_LWS(*qval))
398 qval++;
399
400 if (qval >= ctx.value.ptr + ctx.value.len || *qval != ';') {
401 qval = NULL;
402 break;
403 }
404 qval++;
405
406 while (qval < ctx.value.ptr + ctx.value.len && HTTP_IS_LWS(*qval))
407 qval++;
408
409 if (qval >= ctx.value.ptr + ctx.value.len) {
410 qval = NULL;
411 break;
412 }
413 if (strncmp(qval, "q=", MIN(ctx.value.ptr + ctx.value.len - qval, 2)) == 0)
414 break;
415
416 while (qval < ctx.value.ptr + ctx.value.len && *qval != ';')
417 qval++;
418 }
419
420 /* here we have qval pointing to the first "q=" attribute or NULL if not found */
421 q = qval ? http_parse_qvalue(qval + 2, NULL) : 1000;
422
423 if (q <= best_q)
424 continue;
425
426 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
427 if (*(ctx.value.ptr) == '*' ||
428 word_match(ctx.value.ptr, toklen, comp_algo->ua_name, comp_algo->ua_name_len)) {
429 st->comp_algo = comp_algo;
430 best_q = q;
431 break;
432 }
433 }
434 }
435 }
436
437 /* remove all occurrences of the header when "compression offload" is set */
438 if (st->comp_algo) {
439 if ((s->be->comp && s->be->comp->offload) ||
440 (strm_fe(s)->comp && strm_fe(s)->comp->offload)) {
441 http_remove_header(htx, &ctx);
442 ctx.blk = NULL;
443 while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 1))
444 http_remove_header(htx, &ctx);
445 }
Christopher Faulet3d97c902015-12-09 14:59:38 +0100446 return 1;
447 }
448
449 /* identity is implicit does not require headers */
Christopher Faulet92d36382015-11-05 13:35:03 +0100450 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
451 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100452 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
453 if (comp_algo->cfg_name_len == 8 && memcmp(comp_algo->cfg_name, "identity", 8) == 0) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100454 st->comp_algo = comp_algo;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100455 return 1;
456 }
457 }
458 }
459
Christopher Faulet92d36382015-11-05 13:35:03 +0100460 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100461 return 0;
462}
463
464/*
465 * Selects a comression algorithm depending of the server response.
466 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100467static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200468select_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100469{
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100470 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100471 struct http_txn *txn = s->txn;
472 struct http_hdr_ctx ctx;
473 struct comp_type *comp_type;
474
475 /* no common compression algorithm was found in request header */
476 if (st->comp_algo == NULL)
477 goto fail;
478
Christopher Faulet1d3613a2019-01-07 14:41:59 +0100479 /* compression already in progress */
480 if (msg->flags & HTTP_MSGF_COMPRESSING)
481 goto fail;
482
Christopher Faulete6902cd2018-11-30 22:29:48 +0100483 /* HTTP < 1.1 should not be compressed */
484 if (!(msg->flags & HTTP_MSGF_VER_11) || !(txn->req.flags & HTTP_MSGF_VER_11))
485 goto fail;
486
487 if (txn->meth == HTTP_METH_HEAD)
488 goto fail;
489
490 /* compress 200,201,202,203 responses only */
491 if ((txn->status != 200) &&
492 (txn->status != 201) &&
493 (txn->status != 202) &&
494 (txn->status != 203))
495 goto fail;
496
Christopher Fauletc963eb22018-12-21 14:53:54 +0100497 if (!(msg->flags & HTTP_MSGF_XFER_LEN) || msg->flags & HTTP_MSGF_BODYLESS)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100498 goto fail;
499
500 /* content is already compressed */
501 ctx.blk = NULL;
502 if (http_find_header(htx, ist("Content-Encoding"), &ctx, 1))
503 goto fail;
504
505 /* no compression when Cache-Control: no-transform is present in the message */
506 ctx.blk = NULL;
507 while (http_find_header(htx, ist("Cache-Control"), &ctx, 0)) {
508 if (word_match(ctx.value.ptr, ctx.value.len, "no-transform", 12))
509 goto fail;
510 }
511
Tim Duesterhusb229f012019-01-29 16:38:56 +0100512 /* no compression when ETag is malformed */
513 ctx.blk = NULL;
514 if (http_find_header(htx, ist("ETag"), &ctx, 1)) {
515 if (!(((ctx.value.len >= 4 && memcmp(ctx.value.ptr, "W/\"", 3) == 0) || /* Either a weak ETag */
516 (ctx.value.len >= 2 && ctx.value.ptr[0] == '"')) && /* or strong ETag */
517 ctx.value.ptr[ctx.value.len - 1] == '"')) {
518 goto fail;
519 }
520 }
521 /* no compression when multiple ETags are present
522 * Note: Do not reset ctx.blk!
523 */
524 if (http_find_header(htx, ist("ETag"), &ctx, 1))
525 goto fail;
526
Christopher Faulete6902cd2018-11-30 22:29:48 +0100527 comp_type = NULL;
528
529 /* we don't want to compress multipart content-types, nor content-types that are
530 * not listed in the "compression type" directive if any. If no content-type was
531 * found but configuration requires one, we don't compress either. Backend has
532 * the priority.
533 */
534 ctx.blk = NULL;
535 if (http_find_header(htx, ist("Content-Type"), &ctx, 1)) {
536 if (ctx.value.len >= 9 && strncasecmp("multipart", ctx.value.ptr, 9) == 0)
537 goto fail;
538
539 if ((s->be->comp && (comp_type = s->be->comp->types)) ||
540 (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types))) {
541 for (; comp_type; comp_type = comp_type->next) {
542 if (ctx.value.len >= comp_type->name_len &&
543 strncasecmp(ctx.value.ptr, comp_type->name, comp_type->name_len) == 0)
544 /* this Content-Type should be compressed */
545 break;
546 }
547 /* this Content-Type should not be compressed */
548 if (comp_type == NULL)
549 goto fail;
550 }
551 }
552 else { /* no content-type header */
553 if ((s->be->comp && s->be->comp->types) ||
554 (strm_fe(s)->comp && strm_fe(s)->comp->types))
555 goto fail; /* a content-type was required */
556 }
557
558 /* limit compression rate */
559 if (global.comp_rate_lim > 0)
560 if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim)
561 goto fail;
562
563 /* limit cpu usage */
Willy Tarreau81036f22019-05-20 19:24:50 +0200564 if (ti->idle_pct < compress_min_idle)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100565 goto fail;
566
567 /* initialize compression */
568 if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0)
569 goto fail;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100570 msg->flags |= HTTP_MSGF_COMPRESSING;
571 return 1;
572
Christopher Faulete6902cd2018-11-30 22:29:48 +0100573 fail:
574 st->comp_algo = NULL;
575 return 0;
576}
577
Christopher Faulet3d97c902015-12-09 14:59:38 +0100578/***********************************************************************/
Christopher Faulete6902cd2018-11-30 22:29:48 +0100579static int
580htx_compression_buffer_init(struct htx *htx, struct buffer *out)
581{
582 /* output stream requires at least 10 bytes for the gzip header, plus
583 * at least 8 bytes for the gzip trailer (crc+len), plus a possible
584 * plus at most 5 bytes per 32kB block and 2 bytes to close the stream.
585 */
586 if (htx_free_space(htx) < 20 + 5 * ((htx->data + 32767) >> 15))
587 return -1;
588 b_reset(out);
589 return 0;
590}
591
Christopher Faulete6902cd2018-11-30 22:29:48 +0100592static int
593htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
594 struct buffer *out)
595{
596 return st->comp_algo->add_data(st->comp_ctx, data, len, out);
597}
598
Christopher Faulete6902cd2018-11-30 22:29:48 +0100599static int
600htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end)
601{
602 if (end)
603 return st->comp_algo->finish(st->comp_ctx, out);
604 else
605 return st->comp_algo->flush(st->comp_ctx, out);
606}
607
Christopher Faulet3d97c902015-12-09 14:59:38 +0100608
609/***********************************************************************/
Christopher Faulet92d36382015-11-05 13:35:03 +0100610struct flt_ops comp_ops = {
Christopher Faulete6902cd2018-11-30 22:29:48 +0100611 .init = comp_flt_init,
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +0200612 .init_per_thread = comp_flt_init_per_thread,
613 .deinit_per_thread = comp_flt_deinit_per_thread,
Christopher Faulet92d36382015-11-05 13:35:03 +0100614
Christopher Faulet5e896512020-03-06 14:59:05 +0100615 .attach = comp_strm_init,
616 .detach = comp_strm_deinit,
617
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200618 .channel_post_analyze = comp_http_post_analyze,
Christopher Faulet92d36382015-11-05 13:35:03 +0100619
Christopher Faulet1339d742016-05-11 16:48:33 +0200620 .http_headers = comp_http_headers,
Christopher Faulete6902cd2018-11-30 22:29:48 +0100621 .http_payload = comp_http_payload,
622 .http_end = comp_http_end,
Christopher Faulet92d36382015-11-05 13:35:03 +0100623};
624
Christopher Faulet3d97c902015-12-09 14:59:38 +0100625static int
626parse_compression_options(char **args, int section, struct proxy *proxy,
627 struct proxy *defpx, const char *file, int line,
628 char **err)
629{
Christopher Faulet92d36382015-11-05 13:35:03 +0100630 struct comp *comp;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100631
632 if (proxy->comp == NULL) {
Vincent Bernat02779b62016-04-03 13:48:43 +0200633 comp = calloc(1, sizeof(*comp));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100634 proxy->comp = comp;
635 }
636 else
637 comp = proxy->comp;
638
639 if (!strcmp(args[1], "algo")) {
640 struct comp_ctx *ctx;
641 int cur_arg = 2;
642
643 if (!*args[cur_arg]) {
644 memprintf(err, "parsing [%s:%d] : '%s' expects <algorithm>\n",
645 file, line, args[0]);
646 return -1;
647 }
648 while (*(args[cur_arg])) {
649 if (comp_append_algo(comp, args[cur_arg]) < 0) {
650 memprintf(err, "'%s' : '%s' is not a supported algorithm.\n",
651 args[0], args[cur_arg]);
652 return -1;
653 }
654 if (proxy->comp->algos->init(&ctx, 9) == 0)
655 proxy->comp->algos->end(&ctx);
656 else {
657 memprintf(err, "'%s' : Can't init '%s' algorithm.\n",
658 args[0], args[cur_arg]);
659 return -1;
660 }
661 cur_arg++;
662 continue;
663 }
664 }
665 else if (!strcmp(args[1], "offload"))
666 comp->offload = 1;
667 else if (!strcmp(args[1], "type")) {
668 int cur_arg = 2;
669
670 if (!*args[cur_arg]) {
671 memprintf(err, "'%s' expects <type>\n", args[0]);
672 return -1;
673 }
674 while (*(args[cur_arg])) {
675 comp_append_type(comp, args[cur_arg]);
676 cur_arg++;
677 continue;
678 }
679 }
680 else {
681 memprintf(err, "'%s' expects 'algo', 'type' or 'offload'\n",
682 args[0]);
683 return -1;
684 }
685
686 return 0;
687}
688
Christopher Faulet92d36382015-11-05 13:35:03 +0100689static int
690parse_http_comp_flt(char **args, int *cur_arg, struct proxy *px,
Thierry Fournier3610c392016-04-13 18:27:51 +0200691 struct flt_conf *fconf, char **err, void *private)
Christopher Faulet92d36382015-11-05 13:35:03 +0100692{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100693 struct flt_conf *fc, *back;
Christopher Faulet92d36382015-11-05 13:35:03 +0100694
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100695 list_for_each_entry_safe(fc, back, &px->filter_configs, list) {
696 if (fc->id == http_comp_flt_id) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100697 memprintf(err, "%s: Proxy supports only one compression filter\n", px->id);
698 return -1;
699 }
700 }
701
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100702 fconf->id = http_comp_flt_id;
703 fconf->conf = NULL;
704 fconf->ops = &comp_ops;
Christopher Faulet92d36382015-11-05 13:35:03 +0100705 (*cur_arg)++;
706
707 return 0;
708}
709
710
711int
Christopher Fauletc9df7f72018-12-10 16:14:04 +0100712check_implicit_http_comp_flt(struct proxy *proxy)
Christopher Faulet92d36382015-11-05 13:35:03 +0100713{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100714 struct flt_conf *fconf;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100715 int explicit = 0;
716 int comp = 0;
Christopher Faulet92d36382015-11-05 13:35:03 +0100717 int err = 0;
718
719 if (proxy->comp == NULL)
720 goto end;
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100721 if (!LIST_ISEMPTY(&proxy->filter_configs)) {
722 list_for_each_entry(fconf, &proxy->filter_configs, list) {
723 if (fconf->id == http_comp_flt_id)
Christopher Faulet27d93c32018-12-15 22:32:02 +0100724 comp = 1;
725 else if (fconf->id == cache_store_flt_id) {
726 if (comp) {
727 ha_alert("config: %s '%s': unable to enable the compression filter "
728 "before any cache filter.\n",
729 proxy_type_str(proxy), proxy->id);
730 err++;
731 goto end;
732 }
733 }
Christopher Faulet78fbb9f2019-08-11 23:11:03 +0200734 else if (fconf->id == fcgi_flt_id)
735 continue;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100736 else
737 explicit = 1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100738 }
Christopher Faulet27d93c32018-12-15 22:32:02 +0100739 }
740 if (comp)
741 goto end;
742 else if (explicit) {
743 ha_alert("config: %s '%s': require an explicit filter declaration to use "
744 "HTTP compression\n", proxy_type_str(proxy), proxy->id);
Christopher Faulet92d36382015-11-05 13:35:03 +0100745 err++;
746 goto end;
747 }
748
Christopher Faulet27d93c32018-12-15 22:32:02 +0100749 /* Implicit declaration of the compression filter is always the last
750 * one */
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100751 fconf = calloc(1, sizeof(*fconf));
752 if (!fconf) {
Christopher Faulet767a84b2017-11-24 16:50:31 +0100753 ha_alert("config: %s '%s': out of memory\n",
754 proxy_type_str(proxy), proxy->id);
Christopher Faulet92d36382015-11-05 13:35:03 +0100755 err++;
756 goto end;
757 }
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100758 fconf->id = http_comp_flt_id;
759 fconf->conf = NULL;
760 fconf->ops = &comp_ops;
761 LIST_ADDQ(&proxy->filter_configs, &fconf->list);
Christopher Faulet92d36382015-11-05 13:35:03 +0100762 end:
763 return err;
764}
765
766/*
767 * boolean, returns true if compression is used (either gzip or deflate) in the
768 * response.
769 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100770static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100771smp_fetch_res_comp(const struct arg *args, struct sample *smp, const char *kw,
772 void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100773{
Willy Tarreaube508f12016-03-10 11:47:01 +0100774 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100775
Christopher Faulet3d97c902015-12-09 14:59:38 +0100776 smp->data.type = SMP_T_BOOL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100777 smp->data.u.sint = (txn && (txn->rsp.flags & HTTP_MSGF_COMPRESSING));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100778 return 1;
779}
780
Christopher Faulet92d36382015-11-05 13:35:03 +0100781/*
782 * string, returns algo
783 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100784static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100785smp_fetch_res_comp_algo(const struct arg *args, struct sample *smp,
786 const char *kw, void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100787{
Willy Tarreaube508f12016-03-10 11:47:01 +0100788 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100789 struct filter *filter;
790 struct comp_state *st;
791
Christopher Faulet03d85532017-09-15 10:14:43 +0200792 if (!txn || !(txn->rsp.flags & HTTP_MSGF_COMPRESSING))
Christopher Faulet3d97c902015-12-09 14:59:38 +0100793 return 0;
794
Christopher Fauletfcf035c2015-12-03 11:48:03 +0100795 list_for_each_entry(filter, &strm_flt(smp->strm)->filters, list) {
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100796 if (FLT_ID(filter) != http_comp_flt_id)
Christopher Faulet92d36382015-11-05 13:35:03 +0100797 continue;
798
799 if (!(st = filter->ctx))
800 break;
801
802 smp->data.type = SMP_T_STR;
803 smp->flags = SMP_F_CONST;
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200804 smp->data.u.str.area = st->comp_algo->cfg_name;
805 smp->data.u.str.data = st->comp_algo->cfg_name_len;
Christopher Faulet92d36382015-11-05 13:35:03 +0100806 return 1;
807 }
808 return 0;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100809}
810
811/* Declare the config parser for "compression" keyword */
812static struct cfg_kw_list cfg_kws = {ILH, {
813 { CFG_LISTEN, "compression", parse_compression_options },
814 { 0, NULL, NULL },
815 }
816};
817
Willy Tarreau0108d902018-11-25 19:14:37 +0100818INITCALL1(STG_REGISTER, cfg_register_keywords, &cfg_kws);
819
Christopher Faulet92d36382015-11-05 13:35:03 +0100820/* Declare the filter parser for "compression" keyword */
821static struct flt_kw_list filter_kws = { "COMP", { }, {
Thierry Fournier3610c392016-04-13 18:27:51 +0200822 { "compression", parse_http_comp_flt, NULL },
823 { NULL, NULL, NULL },
Christopher Faulet92d36382015-11-05 13:35:03 +0100824 }
825};
826
Willy Tarreau0108d902018-11-25 19:14:37 +0100827INITCALL1(STG_REGISTER, flt_register_keywords, &filter_kws);
828
Christopher Faulet3d97c902015-12-09 14:59:38 +0100829/* Note: must not be declared <const> as its list will be overwritten */
830static struct sample_fetch_kw_list sample_fetch_keywords = {ILH, {
Christopher Faulet92d36382015-11-05 13:35:03 +0100831 { "res.comp", smp_fetch_res_comp, 0, NULL, SMP_T_BOOL, SMP_USE_HRSHP },
832 { "res.comp_algo", smp_fetch_res_comp_algo, 0, NULL, SMP_T_STR, SMP_USE_HRSHP },
833 { /* END */ },
834 }
835};
Christopher Faulet3d97c902015-12-09 14:59:38 +0100836
Willy Tarreau0108d902018-11-25 19:14:37 +0100837INITCALL1(STG_REGISTER, sample_register_fetches, &sample_fetch_keywords);