blob: ac4a3bf2e52d964a6e951fd6540e63a358045b87 [file] [log] [blame]
Christopher Faulet3d97c902015-12-09 14:59:38 +01001/*
2 * Stream filters related variables and functions.
3 *
4 * Copyright (C) 2015 Qualys Inc., Christopher Faulet <cfaulet@qualys.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020013#include <haproxy/api.h>
Willy Tarreau2741c8c2020-06-02 11:28:02 +020014#include <haproxy/dynbuf.h>
Willy Tarreaucd72d8c2020-06-02 19:11:26 +020015#include <haproxy/http.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010016#include <common/cfgparse.h>
Willy Tarreau16f958c2020-06-03 08:44:35 +020017#include <haproxy/htx.h>
Willy Tarreau853b2972020-05-27 18:01:47 +020018#include <haproxy/list.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010019#include <common/standard.h>
20
21#include <types/compression.h>
22#include <types/filters.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010023#include <types/proxy.h>
24#include <types/sample.h>
25
26#include <proto/compression.h>
Christopher Faulet92d36382015-11-05 13:35:03 +010027#include <proto/filters.h>
Christopher Faulete6902cd2018-11-30 22:29:48 +010028#include <proto/http_htx.h>
Christopher Fauletfc9cfe42019-07-16 14:54:53 +020029#include <proto/http_ana.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010030#include <proto/sample.h>
31#include <proto/stream.h>
32
Christopher Fauletf4a4ef72018-12-07 17:39:53 +010033const char *http_comp_flt_id = "compression filter";
Christopher Faulet92d36382015-11-05 13:35:03 +010034
35struct flt_ops comp_ops;
36
Christopher Faulet92d36382015-11-05 13:35:03 +010037struct comp_state {
38 struct comp_ctx *comp_ctx; /* compression context */
39 struct comp_algo *comp_algo; /* compression algorithm if not NULL */
Christopher Faulet92d36382015-11-05 13:35:03 +010040};
41
Willy Tarreau8ceae722018-11-26 11:58:30 +010042/* Pools used to allocate comp_state structs */
43DECLARE_STATIC_POOL(pool_head_comp_state, "comp_state", sizeof(struct comp_state));
44
45static THREAD_LOCAL struct buffer tmpbuf;
46static THREAD_LOCAL struct buffer zbuf;
Willy Tarreau8ceae722018-11-26 11:58:30 +010047
Christopher Faulet92d36382015-11-05 13:35:03 +010048static int select_compression_request_header(struct comp_state *st,
49 struct stream *s,
50 struct http_msg *msg);
51static int select_compression_response_header(struct comp_state *st,
52 struct stream *s,
53 struct http_msg *msg);
Christopher Faulet27d93c32018-12-15 22:32:02 +010054static int set_compression_response_header(struct comp_state *st,
55 struct stream *s,
56 struct http_msg *msg);
Christopher Faulet92d36382015-11-05 13:35:03 +010057
Christopher Faulete6902cd2018-11-30 22:29:48 +010058static int htx_compression_buffer_init(struct htx *htx, struct buffer *out);
59static int htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
60 struct buffer *out);
61static int htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end);
62
Christopher Faulet92d36382015-11-05 13:35:03 +010063/***********************************************************************/
64static int
Christopher Faulete6902cd2018-11-30 22:29:48 +010065comp_flt_init(struct proxy *px, struct flt_conf *fconf)
66{
Christopher Faulet6e540952018-12-03 22:43:41 +010067 fconf->flags |= FLT_CFG_FL_HTX;
Christopher Faulete6902cd2018-11-30 22:29:48 +010068 return 0;
69}
70
71static int
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020072comp_flt_init_per_thread(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010073{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020074 if (!tmpbuf.size && b_alloc(&tmpbuf) == NULL)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010075 return -1;
Willy Tarreauc9fa0482018-07-10 17:43:27 +020076 if (!zbuf.size && b_alloc(&zbuf) == NULL)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010077 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +010078 return 0;
79}
80
81static void
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020082comp_flt_deinit_per_thread(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010083{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020084 if (tmpbuf.size)
Christopher Faulet92d36382015-11-05 13:35:03 +010085 b_free(&tmpbuf);
Willy Tarreauc9fa0482018-07-10 17:43:27 +020086 if (zbuf.size)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010087 b_free(&zbuf);
Christopher Faulet92d36382015-11-05 13:35:03 +010088}
89
90static int
Christopher Faulet5e896512020-03-06 14:59:05 +010091comp_strm_init(struct stream *s, struct filter *filter)
Christopher Faulet92d36382015-11-05 13:35:03 +010092{
Christopher Faulet5e896512020-03-06 14:59:05 +010093 struct comp_state *st;
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020094
Christopher Faulet5e896512020-03-06 14:59:05 +010095 st = pool_alloc_dirty(pool_head_comp_state);
96 if (st == NULL)
97 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +010098
Christopher Faulet5e896512020-03-06 14:59:05 +010099 st->comp_algo = NULL;
100 st->comp_ctx = NULL;
101 filter->ctx = st;
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200102
Christopher Faulet5e896512020-03-06 14:59:05 +0100103 /* Register post-analyzer on AN_RES_WAIT_HTTP because we need to
104 * analyze response headers before http-response rules execution
105 * to be sure we can use res.comp and res.comp_algo sample
106 * fetches */
107 filter->post_analyzers |= AN_RES_WAIT_HTTP;
Christopher Faulet92d36382015-11-05 13:35:03 +0100108 return 1;
109}
110
Christopher Faulet5e896512020-03-06 14:59:05 +0100111static void
112comp_strm_deinit(struct stream *s, struct filter *filter)
Christopher Faulet92d36382015-11-05 13:35:03 +0100113{
114 struct comp_state *st = filter->ctx;
Christopher Faulet92d36382015-11-05 13:35:03 +0100115
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200116 if (!st)
Christopher Faulet5e896512020-03-06 14:59:05 +0100117 return;
Christopher Faulet92d36382015-11-05 13:35:03 +0100118
Christopher Faulet92d36382015-11-05 13:35:03 +0100119 /* release any possible compression context */
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200120 if (st->comp_algo)
121 st->comp_algo->end(&st->comp_ctx);
Willy Tarreaubafbe012017-11-24 17:34:44 +0100122 pool_free(pool_head_comp_state, st);
Christopher Faulet92d36382015-11-05 13:35:03 +0100123 filter->ctx = NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100124}
125
126static int
Christopher Faulet1339d742016-05-11 16:48:33 +0200127comp_http_headers(struct stream *s, struct filter *filter, struct http_msg *msg)
128{
129 struct comp_state *st = filter->ctx;
130
131 if (!strm_fe(s)->comp && !s->be->comp)
132 goto end;
133
134 if (!(msg->chn->flags & CF_ISRESP))
135 select_compression_request_header(st, s, msg);
136 else {
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200137 /* Response headers have already been checked in
138 * comp_http_post_analyze callback. */
Christopher Faulet1339d742016-05-11 16:48:33 +0200139 if (st->comp_algo) {
Christopher Faulet27d93c32018-12-15 22:32:02 +0100140 if (!set_compression_response_header(st, s, msg))
141 goto end;
Christopher Faulet1339d742016-05-11 16:48:33 +0200142 register_data_filter(s, msg->chn, filter);
Christopher Faulet1339d742016-05-11 16:48:33 +0200143 }
144 }
145
146 end:
147 return 1;
148}
149
150static int
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200151comp_http_post_analyze(struct stream *s, struct filter *filter,
152 struct channel *chn, unsigned an_bit)
153{
154 struct http_txn *txn = s->txn;
155 struct http_msg *msg = &txn->rsp;
156 struct comp_state *st = filter->ctx;
157
158 if (an_bit != AN_RES_WAIT_HTTP)
159 goto end;
160
161 if (!strm_fe(s)->comp && !s->be->comp)
162 goto end;
163
164 select_compression_response_header(st, s, msg);
165
166 end:
167 return 1;
168}
169
170static int
Christopher Faulete6902cd2018-11-30 22:29:48 +0100171comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg,
172 unsigned int offset, unsigned int len)
173{
174 struct comp_state *st = filter->ctx;
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100175 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6a62bf2020-03-02 16:20:05 +0100176 struct htx_ret htxret = htx_find_offset(htx, offset);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100177 struct htx_blk *blk;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100178 int ret, consumed = 0, to_forward = 0;
179
Christopher Faulete6a62bf2020-03-02 16:20:05 +0100180 blk = htxret.blk;
181 offset = htxret.ret;
182 for (; blk && len; blk = htx_get_next_blk(htx, blk)) {
Christopher Faulete6902cd2018-11-30 22:29:48 +0100183 enum htx_blk_type type = htx_get_blk_type(blk);
184 uint32_t sz = htx_get_blksz(blk);
185 struct ist v;
186
187 switch (type) {
188 case HTX_BLK_UNUSED:
189 break;
190
191 case HTX_BLK_DATA:
192 v = htx_get_blk_value(htx, blk);
193 v.ptr += offset;
194 v.len -= offset;
195 if (v.len > len)
196 v.len = len;
197 if (htx_compression_buffer_init(htx, &trash) < 0) {
198 msg->chn->flags |= CF_WAKE_WRITE;
199 goto end;
200 }
201 ret = htx_compression_buffer_add_data(st, v.ptr, v.len, &trash);
202 if (ret < 0)
203 goto error;
204 if (htx_compression_buffer_end(st, &trash, 0) < 0)
205 goto error;
206 len -= ret;
207 consumed += ret;
208 to_forward += b_data(&trash);
209 if (ret == sz && !b_data(&trash)) {
210 offset = 0;
211 blk = htx_remove_blk(htx, blk);
212 continue;
213 }
214 v.len = ret;
215 blk = htx_replace_blk_value(htx, blk, v, ist2(b_head(&trash), b_data(&trash)));
216 break;
217
Christopher Faulete6902cd2018-11-30 22:29:48 +0100218 case HTX_BLK_TLR:
Christopher Faulet2d7c5392019-06-03 10:41:26 +0200219 case HTX_BLK_EOT:
Christopher Faulete6902cd2018-11-30 22:29:48 +0100220 case HTX_BLK_EOM:
221 if (msg->flags & HTTP_MSGF_COMPRESSING) {
222 if (htx_compression_buffer_init(htx, &trash) < 0) {
223 msg->chn->flags |= CF_WAKE_WRITE;
224 goto end;
225 }
226 if (htx_compression_buffer_end(st, &trash, 1) < 0)
227 goto error;
Christopher Fauletd238ae32018-12-21 15:10:25 +0100228 if (b_data(&trash)) {
Christopher Faulet86bc8df2019-06-11 10:38:38 +0200229 struct htx_blk *last = htx_add_last_data(htx, ist2(b_head(&trash), b_data(&trash)));
230 if (!last)
231 goto error;
232 blk = htx_get_next_blk(htx, last);
Christopher Fauletd238ae32018-12-21 15:10:25 +0100233 if (!blk)
234 goto error;
235 to_forward += b_data(&trash);
236 }
Christopher Faulete6902cd2018-11-30 22:29:48 +0100237 msg->flags &= ~HTTP_MSGF_COMPRESSING;
238 /* We let the mux add last empty chunk and empty trailers */
239 }
240 /* fall through */
241
242 default:
243 sz -= offset;
244 if (sz > len)
245 sz = len;
246 consumed += sz;
247 to_forward += sz;
248 len -= sz;
249 break;
250 }
251
252 offset = 0;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100253 }
254
255 end:
256 if (to_forward != consumed)
257 flt_update_offsets(filter, msg->chn, to_forward - consumed);
258
259 if (st->comp_ctx && st->comp_ctx->cur_lvl > 0) {
Willy Tarreauef6fd852019-02-04 11:48:03 +0100260 update_freq_ctr(&global.comp_bps_in, consumed);
Olivier Houchard43da3432019-03-08 18:50:27 +0100261 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_in, consumed);
262 _HA_ATOMIC_ADD(&s->be->be_counters.comp_in, consumed);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100263 update_freq_ctr(&global.comp_bps_out, to_forward);
Olivier Houchard43da3432019-03-08 18:50:27 +0100264 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_out, to_forward);
265 _HA_ATOMIC_ADD(&s->be->be_counters.comp_out, to_forward);
Willy Tarreauef6fd852019-02-04 11:48:03 +0100266 } else {
Olivier Houchard43da3432019-03-08 18:50:27 +0100267 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_byp, consumed);
268 _HA_ATOMIC_ADD(&s->be->be_counters.comp_byp, consumed);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100269 }
270 return to_forward;
271
272 error:
273 return -1;
274}
275
Christopher Faulet2fb28802015-12-01 10:40:57 +0100276
Christopher Faulet92d36382015-11-05 13:35:03 +0100277static int
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200278comp_http_end(struct stream *s, struct filter *filter,
279 struct http_msg *msg)
280{
281 struct comp_state *st = filter->ctx;
282
283 if (!(msg->chn->flags & CF_ISRESP) || !st || !st->comp_algo)
284 goto end;
285
286 if (strm_fe(s)->mode == PR_MODE_HTTP)
Olivier Houchard43da3432019-03-08 18:50:27 +0100287 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.p.http.comp_rsp, 1);
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200288 if ((s->flags & SF_BE_ASSIGNED) && (s->be->mode == PR_MODE_HTTP))
Olivier Houchard43da3432019-03-08 18:50:27 +0100289 _HA_ATOMIC_ADD(&s->be->be_counters.p.http.comp_rsp, 1);
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200290 end:
291 return 1;
292}
Christopher Faulet27d93c32018-12-15 22:32:02 +0100293
Christopher Faulet89f2b162019-07-15 21:16:04 +0200294/***********************************************************************/
Christopher Faulet27d93c32018-12-15 22:32:02 +0100295static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200296set_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulet27d93c32018-12-15 22:32:02 +0100297{
298 struct htx *htx = htxbuf(&msg->chn->buf);
Tim Duesterhusb229f012019-01-29 16:38:56 +0100299 struct http_hdr_ctx ctx;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100300
301 /*
302 * Add Content-Encoding header when it's not identity encoding.
303 * RFC 2616 : Identity encoding: This content-coding is used only in the
304 * Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding
305 * header.
306 */
307 if (st->comp_algo->cfg_name_len != 8 || memcmp(st->comp_algo->cfg_name, "identity", 8) != 0) {
308 struct ist v = ist2(st->comp_algo->ua_name, st->comp_algo->ua_name_len);
309
310 if (!http_add_header(htx, ist("Content-Encoding"), v))
311 goto error;
312 }
313
314 /* remove Content-Length header */
315 if (msg->flags & HTTP_MSGF_CNT_LEN) {
Christopher Faulet27d93c32018-12-15 22:32:02 +0100316 ctx.blk = NULL;
317 while (http_find_header(htx, ist("Content-Length"), &ctx, 1))
318 http_remove_header(htx, &ctx);
319 }
320
321 /* add "Transfer-Encoding: chunked" header */
322 if (!(msg->flags & HTTP_MSGF_TE_CHNK)) {
323 if (!http_add_header(htx, ist("Transfer-Encoding"), ist("chunked")))
324 goto error;
325 }
326
Tim Duesterhusb229f012019-01-29 16:38:56 +0100327 /* convert "ETag" header to a weak ETag */
328 ctx.blk = NULL;
329 if (http_find_header(htx, ist("ETag"), &ctx, 1)) {
330 if (ctx.value.ptr[0] == '"') {
331 /* This a strong ETag. Convert it to a weak one. */
332 struct ist v = ist2(trash.area, 0);
333 if (istcat(&v, ist("W/"), trash.size) == -1 || istcat(&v, ctx.value, trash.size) == -1)
334 goto error;
335
336 if (!http_replace_header_value(htx, &ctx, v))
337 goto error;
338 }
339 }
340
Tim Duesterhus721d6862019-06-17 16:10:07 +0200341 if (!http_add_header(htx, ist("Vary"), ist("Accept-Encoding")))
342 goto error;
343
Christopher Faulet27d93c32018-12-15 22:32:02 +0100344 return 1;
345
346 error:
347 st->comp_algo->end(&st->comp_ctx);
348 st->comp_algo = NULL;
349 return 0;
350}
351
Christopher Faulet3d97c902015-12-09 14:59:38 +0100352/*
353 * Selects a compression algorithm depending on the client request.
354 */
Christopher Faulete6902cd2018-11-30 22:29:48 +0100355static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200356select_compression_request_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100357{
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100358 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100359 struct http_hdr_ctx ctx;
360 struct comp_algo *comp_algo = NULL;
361 struct comp_algo *comp_algo_back = NULL;
362
363 /* Disable compression for older user agents announcing themselves as "Mozilla/4"
364 * unless they are known good (MSIE 6 with XP SP2, or MSIE 7 and later).
365 * See http://zoompf.com/2012/02/lose-the-wait-http-compression for more details.
366 */
367 ctx.blk = NULL;
368 if (http_find_header(htx, ist("User-Agent"), &ctx, 1) &&
369 ctx.value.len >= 9 &&
370 memcmp(ctx.value.ptr, "Mozilla/4", 9) == 0 &&
371 (ctx.value.len < 31 ||
372 memcmp(ctx.value.ptr + 25, "MSIE ", 5) != 0 ||
373 *(ctx.value.ptr + 30) < '6' ||
374 (*(ctx.value.ptr + 30) == '6' &&
375 (ctx.value.len < 54 || memcmp(ctx.value.ptr + 51, "SV1", 3) != 0)))) {
376 st->comp_algo = NULL;
377 return 0;
378 }
379
380 /* search for the algo in the backend in priority or the frontend */
381 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
382 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
383 int best_q = 0;
384
385 ctx.blk = NULL;
386 while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 0)) {
387 const char *qval;
388 int q;
389 int toklen;
390
391 /* try to isolate the token from the optional q-value */
392 toklen = 0;
393 while (toklen < ctx.value.len && HTTP_IS_TOKEN(*(ctx.value.ptr + toklen)))
394 toklen++;
395
396 qval = ctx.value.ptr + toklen;
397 while (1) {
398 while (qval < ctx.value.ptr + ctx.value.len && HTTP_IS_LWS(*qval))
399 qval++;
400
401 if (qval >= ctx.value.ptr + ctx.value.len || *qval != ';') {
402 qval = NULL;
403 break;
404 }
405 qval++;
406
407 while (qval < ctx.value.ptr + ctx.value.len && HTTP_IS_LWS(*qval))
408 qval++;
409
410 if (qval >= ctx.value.ptr + ctx.value.len) {
411 qval = NULL;
412 break;
413 }
414 if (strncmp(qval, "q=", MIN(ctx.value.ptr + ctx.value.len - qval, 2)) == 0)
415 break;
416
417 while (qval < ctx.value.ptr + ctx.value.len && *qval != ';')
418 qval++;
419 }
420
421 /* here we have qval pointing to the first "q=" attribute or NULL if not found */
422 q = qval ? http_parse_qvalue(qval + 2, NULL) : 1000;
423
424 if (q <= best_q)
425 continue;
426
427 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
428 if (*(ctx.value.ptr) == '*' ||
429 word_match(ctx.value.ptr, toklen, comp_algo->ua_name, comp_algo->ua_name_len)) {
430 st->comp_algo = comp_algo;
431 best_q = q;
432 break;
433 }
434 }
435 }
436 }
437
438 /* remove all occurrences of the header when "compression offload" is set */
439 if (st->comp_algo) {
440 if ((s->be->comp && s->be->comp->offload) ||
441 (strm_fe(s)->comp && strm_fe(s)->comp->offload)) {
442 http_remove_header(htx, &ctx);
443 ctx.blk = NULL;
444 while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 1))
445 http_remove_header(htx, &ctx);
446 }
Christopher Faulet3d97c902015-12-09 14:59:38 +0100447 return 1;
448 }
449
450 /* identity is implicit does not require headers */
Christopher Faulet92d36382015-11-05 13:35:03 +0100451 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
452 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100453 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
454 if (comp_algo->cfg_name_len == 8 && memcmp(comp_algo->cfg_name, "identity", 8) == 0) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100455 st->comp_algo = comp_algo;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100456 return 1;
457 }
458 }
459 }
460
Christopher Faulet92d36382015-11-05 13:35:03 +0100461 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100462 return 0;
463}
464
465/*
466 * Selects a comression algorithm depending of the server response.
467 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100468static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200469select_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100470{
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100471 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100472 struct http_txn *txn = s->txn;
473 struct http_hdr_ctx ctx;
474 struct comp_type *comp_type;
475
476 /* no common compression algorithm was found in request header */
477 if (st->comp_algo == NULL)
478 goto fail;
479
Christopher Faulet1d3613a2019-01-07 14:41:59 +0100480 /* compression already in progress */
481 if (msg->flags & HTTP_MSGF_COMPRESSING)
482 goto fail;
483
Christopher Faulete6902cd2018-11-30 22:29:48 +0100484 /* HTTP < 1.1 should not be compressed */
485 if (!(msg->flags & HTTP_MSGF_VER_11) || !(txn->req.flags & HTTP_MSGF_VER_11))
486 goto fail;
487
488 if (txn->meth == HTTP_METH_HEAD)
489 goto fail;
490
491 /* compress 200,201,202,203 responses only */
492 if ((txn->status != 200) &&
493 (txn->status != 201) &&
494 (txn->status != 202) &&
495 (txn->status != 203))
496 goto fail;
497
Christopher Fauletc963eb22018-12-21 14:53:54 +0100498 if (!(msg->flags & HTTP_MSGF_XFER_LEN) || msg->flags & HTTP_MSGF_BODYLESS)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100499 goto fail;
500
501 /* content is already compressed */
502 ctx.blk = NULL;
503 if (http_find_header(htx, ist("Content-Encoding"), &ctx, 1))
504 goto fail;
505
506 /* no compression when Cache-Control: no-transform is present in the message */
507 ctx.blk = NULL;
508 while (http_find_header(htx, ist("Cache-Control"), &ctx, 0)) {
509 if (word_match(ctx.value.ptr, ctx.value.len, "no-transform", 12))
510 goto fail;
511 }
512
Tim Duesterhusb229f012019-01-29 16:38:56 +0100513 /* no compression when ETag is malformed */
514 ctx.blk = NULL;
515 if (http_find_header(htx, ist("ETag"), &ctx, 1)) {
516 if (!(((ctx.value.len >= 4 && memcmp(ctx.value.ptr, "W/\"", 3) == 0) || /* Either a weak ETag */
517 (ctx.value.len >= 2 && ctx.value.ptr[0] == '"')) && /* or strong ETag */
518 ctx.value.ptr[ctx.value.len - 1] == '"')) {
519 goto fail;
520 }
521 }
522 /* no compression when multiple ETags are present
523 * Note: Do not reset ctx.blk!
524 */
525 if (http_find_header(htx, ist("ETag"), &ctx, 1))
526 goto fail;
527
Christopher Faulete6902cd2018-11-30 22:29:48 +0100528 comp_type = NULL;
529
530 /* we don't want to compress multipart content-types, nor content-types that are
531 * not listed in the "compression type" directive if any. If no content-type was
532 * found but configuration requires one, we don't compress either. Backend has
533 * the priority.
534 */
535 ctx.blk = NULL;
536 if (http_find_header(htx, ist("Content-Type"), &ctx, 1)) {
537 if (ctx.value.len >= 9 && strncasecmp("multipart", ctx.value.ptr, 9) == 0)
538 goto fail;
539
540 if ((s->be->comp && (comp_type = s->be->comp->types)) ||
541 (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types))) {
542 for (; comp_type; comp_type = comp_type->next) {
543 if (ctx.value.len >= comp_type->name_len &&
544 strncasecmp(ctx.value.ptr, comp_type->name, comp_type->name_len) == 0)
545 /* this Content-Type should be compressed */
546 break;
547 }
548 /* this Content-Type should not be compressed */
549 if (comp_type == NULL)
550 goto fail;
551 }
552 }
553 else { /* no content-type header */
554 if ((s->be->comp && s->be->comp->types) ||
555 (strm_fe(s)->comp && strm_fe(s)->comp->types))
556 goto fail; /* a content-type was required */
557 }
558
559 /* limit compression rate */
560 if (global.comp_rate_lim > 0)
561 if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim)
562 goto fail;
563
564 /* limit cpu usage */
Willy Tarreau81036f22019-05-20 19:24:50 +0200565 if (ti->idle_pct < compress_min_idle)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100566 goto fail;
567
568 /* initialize compression */
569 if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0)
570 goto fail;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100571 msg->flags |= HTTP_MSGF_COMPRESSING;
572 return 1;
573
Christopher Faulete6902cd2018-11-30 22:29:48 +0100574 fail:
575 st->comp_algo = NULL;
576 return 0;
577}
578
Christopher Faulet3d97c902015-12-09 14:59:38 +0100579/***********************************************************************/
Christopher Faulete6902cd2018-11-30 22:29:48 +0100580static int
581htx_compression_buffer_init(struct htx *htx, struct buffer *out)
582{
583 /* output stream requires at least 10 bytes for the gzip header, plus
584 * at least 8 bytes for the gzip trailer (crc+len), plus a possible
585 * plus at most 5 bytes per 32kB block and 2 bytes to close the stream.
586 */
587 if (htx_free_space(htx) < 20 + 5 * ((htx->data + 32767) >> 15))
588 return -1;
589 b_reset(out);
590 return 0;
591}
592
Christopher Faulete6902cd2018-11-30 22:29:48 +0100593static int
594htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
595 struct buffer *out)
596{
597 return st->comp_algo->add_data(st->comp_ctx, data, len, out);
598}
599
Christopher Faulete6902cd2018-11-30 22:29:48 +0100600static int
601htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end)
602{
603 if (end)
604 return st->comp_algo->finish(st->comp_ctx, out);
605 else
606 return st->comp_algo->flush(st->comp_ctx, out);
607}
608
Christopher Faulet3d97c902015-12-09 14:59:38 +0100609
610/***********************************************************************/
Christopher Faulet92d36382015-11-05 13:35:03 +0100611struct flt_ops comp_ops = {
Christopher Faulete6902cd2018-11-30 22:29:48 +0100612 .init = comp_flt_init,
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +0200613 .init_per_thread = comp_flt_init_per_thread,
614 .deinit_per_thread = comp_flt_deinit_per_thread,
Christopher Faulet92d36382015-11-05 13:35:03 +0100615
Christopher Faulet5e896512020-03-06 14:59:05 +0100616 .attach = comp_strm_init,
617 .detach = comp_strm_deinit,
618
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200619 .channel_post_analyze = comp_http_post_analyze,
Christopher Faulet92d36382015-11-05 13:35:03 +0100620
Christopher Faulet1339d742016-05-11 16:48:33 +0200621 .http_headers = comp_http_headers,
Christopher Faulete6902cd2018-11-30 22:29:48 +0100622 .http_payload = comp_http_payload,
623 .http_end = comp_http_end,
Christopher Faulet92d36382015-11-05 13:35:03 +0100624};
625
Christopher Faulet3d97c902015-12-09 14:59:38 +0100626static int
627parse_compression_options(char **args, int section, struct proxy *proxy,
628 struct proxy *defpx, const char *file, int line,
629 char **err)
630{
Christopher Faulet92d36382015-11-05 13:35:03 +0100631 struct comp *comp;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100632
633 if (proxy->comp == NULL) {
Vincent Bernat02779b62016-04-03 13:48:43 +0200634 comp = calloc(1, sizeof(*comp));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100635 proxy->comp = comp;
636 }
637 else
638 comp = proxy->comp;
639
640 if (!strcmp(args[1], "algo")) {
641 struct comp_ctx *ctx;
642 int cur_arg = 2;
643
644 if (!*args[cur_arg]) {
645 memprintf(err, "parsing [%s:%d] : '%s' expects <algorithm>\n",
646 file, line, args[0]);
647 return -1;
648 }
649 while (*(args[cur_arg])) {
650 if (comp_append_algo(comp, args[cur_arg]) < 0) {
651 memprintf(err, "'%s' : '%s' is not a supported algorithm.\n",
652 args[0], args[cur_arg]);
653 return -1;
654 }
655 if (proxy->comp->algos->init(&ctx, 9) == 0)
656 proxy->comp->algos->end(&ctx);
657 else {
658 memprintf(err, "'%s' : Can't init '%s' algorithm.\n",
659 args[0], args[cur_arg]);
660 return -1;
661 }
662 cur_arg++;
663 continue;
664 }
665 }
666 else if (!strcmp(args[1], "offload"))
667 comp->offload = 1;
668 else if (!strcmp(args[1], "type")) {
669 int cur_arg = 2;
670
671 if (!*args[cur_arg]) {
672 memprintf(err, "'%s' expects <type>\n", args[0]);
673 return -1;
674 }
675 while (*(args[cur_arg])) {
676 comp_append_type(comp, args[cur_arg]);
677 cur_arg++;
678 continue;
679 }
680 }
681 else {
682 memprintf(err, "'%s' expects 'algo', 'type' or 'offload'\n",
683 args[0]);
684 return -1;
685 }
686
687 return 0;
688}
689
Christopher Faulet92d36382015-11-05 13:35:03 +0100690static int
691parse_http_comp_flt(char **args, int *cur_arg, struct proxy *px,
Thierry Fournier3610c392016-04-13 18:27:51 +0200692 struct flt_conf *fconf, char **err, void *private)
Christopher Faulet92d36382015-11-05 13:35:03 +0100693{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100694 struct flt_conf *fc, *back;
Christopher Faulet92d36382015-11-05 13:35:03 +0100695
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100696 list_for_each_entry_safe(fc, back, &px->filter_configs, list) {
697 if (fc->id == http_comp_flt_id) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100698 memprintf(err, "%s: Proxy supports only one compression filter\n", px->id);
699 return -1;
700 }
701 }
702
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100703 fconf->id = http_comp_flt_id;
704 fconf->conf = NULL;
705 fconf->ops = &comp_ops;
Christopher Faulet92d36382015-11-05 13:35:03 +0100706 (*cur_arg)++;
707
708 return 0;
709}
710
711
712int
Christopher Fauletc9df7f72018-12-10 16:14:04 +0100713check_implicit_http_comp_flt(struct proxy *proxy)
Christopher Faulet92d36382015-11-05 13:35:03 +0100714{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100715 struct flt_conf *fconf;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100716 int explicit = 0;
717 int comp = 0;
Christopher Faulet92d36382015-11-05 13:35:03 +0100718 int err = 0;
719
720 if (proxy->comp == NULL)
721 goto end;
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100722 if (!LIST_ISEMPTY(&proxy->filter_configs)) {
723 list_for_each_entry(fconf, &proxy->filter_configs, list) {
724 if (fconf->id == http_comp_flt_id)
Christopher Faulet27d93c32018-12-15 22:32:02 +0100725 comp = 1;
726 else if (fconf->id == cache_store_flt_id) {
727 if (comp) {
728 ha_alert("config: %s '%s': unable to enable the compression filter "
729 "before any cache filter.\n",
730 proxy_type_str(proxy), proxy->id);
731 err++;
732 goto end;
733 }
734 }
Christopher Faulet78fbb9f2019-08-11 23:11:03 +0200735 else if (fconf->id == fcgi_flt_id)
736 continue;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100737 else
738 explicit = 1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100739 }
Christopher Faulet27d93c32018-12-15 22:32:02 +0100740 }
741 if (comp)
742 goto end;
743 else if (explicit) {
744 ha_alert("config: %s '%s': require an explicit filter declaration to use "
745 "HTTP compression\n", proxy_type_str(proxy), proxy->id);
Christopher Faulet92d36382015-11-05 13:35:03 +0100746 err++;
747 goto end;
748 }
749
Christopher Faulet27d93c32018-12-15 22:32:02 +0100750 /* Implicit declaration of the compression filter is always the last
751 * one */
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100752 fconf = calloc(1, sizeof(*fconf));
753 if (!fconf) {
Christopher Faulet767a84b2017-11-24 16:50:31 +0100754 ha_alert("config: %s '%s': out of memory\n",
755 proxy_type_str(proxy), proxy->id);
Christopher Faulet92d36382015-11-05 13:35:03 +0100756 err++;
757 goto end;
758 }
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100759 fconf->id = http_comp_flt_id;
760 fconf->conf = NULL;
761 fconf->ops = &comp_ops;
762 LIST_ADDQ(&proxy->filter_configs, &fconf->list);
Christopher Faulet92d36382015-11-05 13:35:03 +0100763 end:
764 return err;
765}
766
767/*
768 * boolean, returns true if compression is used (either gzip or deflate) in the
769 * response.
770 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100771static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100772smp_fetch_res_comp(const struct arg *args, struct sample *smp, const char *kw,
773 void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100774{
Willy Tarreaube508f12016-03-10 11:47:01 +0100775 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100776
Christopher Faulet3d97c902015-12-09 14:59:38 +0100777 smp->data.type = SMP_T_BOOL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100778 smp->data.u.sint = (txn && (txn->rsp.flags & HTTP_MSGF_COMPRESSING));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100779 return 1;
780}
781
Christopher Faulet92d36382015-11-05 13:35:03 +0100782/*
783 * string, returns algo
784 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100785static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100786smp_fetch_res_comp_algo(const struct arg *args, struct sample *smp,
787 const char *kw, void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100788{
Willy Tarreaube508f12016-03-10 11:47:01 +0100789 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100790 struct filter *filter;
791 struct comp_state *st;
792
Christopher Faulet03d85532017-09-15 10:14:43 +0200793 if (!txn || !(txn->rsp.flags & HTTP_MSGF_COMPRESSING))
Christopher Faulet3d97c902015-12-09 14:59:38 +0100794 return 0;
795
Christopher Fauletfcf035c2015-12-03 11:48:03 +0100796 list_for_each_entry(filter, &strm_flt(smp->strm)->filters, list) {
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100797 if (FLT_ID(filter) != http_comp_flt_id)
Christopher Faulet92d36382015-11-05 13:35:03 +0100798 continue;
799
800 if (!(st = filter->ctx))
801 break;
802
803 smp->data.type = SMP_T_STR;
804 smp->flags = SMP_F_CONST;
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200805 smp->data.u.str.area = st->comp_algo->cfg_name;
806 smp->data.u.str.data = st->comp_algo->cfg_name_len;
Christopher Faulet92d36382015-11-05 13:35:03 +0100807 return 1;
808 }
809 return 0;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100810}
811
812/* Declare the config parser for "compression" keyword */
813static struct cfg_kw_list cfg_kws = {ILH, {
814 { CFG_LISTEN, "compression", parse_compression_options },
815 { 0, NULL, NULL },
816 }
817};
818
Willy Tarreau0108d902018-11-25 19:14:37 +0100819INITCALL1(STG_REGISTER, cfg_register_keywords, &cfg_kws);
820
Christopher Faulet92d36382015-11-05 13:35:03 +0100821/* Declare the filter parser for "compression" keyword */
822static struct flt_kw_list filter_kws = { "COMP", { }, {
Thierry Fournier3610c392016-04-13 18:27:51 +0200823 { "compression", parse_http_comp_flt, NULL },
824 { NULL, NULL, NULL },
Christopher Faulet92d36382015-11-05 13:35:03 +0100825 }
826};
827
Willy Tarreau0108d902018-11-25 19:14:37 +0100828INITCALL1(STG_REGISTER, flt_register_keywords, &filter_kws);
829
Christopher Faulet3d97c902015-12-09 14:59:38 +0100830/* Note: must not be declared <const> as its list will be overwritten */
831static struct sample_fetch_kw_list sample_fetch_keywords = {ILH, {
Christopher Faulet92d36382015-11-05 13:35:03 +0100832 { "res.comp", smp_fetch_res_comp, 0, NULL, SMP_T_BOOL, SMP_USE_HRSHP },
833 { "res.comp_algo", smp_fetch_res_comp_algo, 0, NULL, SMP_T_STR, SMP_USE_HRSHP },
834 { /* END */ },
835 }
836};
Christopher Faulet3d97c902015-12-09 14:59:38 +0100837
Willy Tarreau0108d902018-11-25 19:14:37 +0100838INITCALL1(STG_REGISTER, sample_register_fetches, &sample_fetch_keywords);