blob: e98ff46ba0337ef09a2ba6edb954d44eac3b454a [file] [log] [blame]
Christopher Faulet3d97c902015-12-09 14:59:38 +01001/*
2 * Stream filters related variables and functions.
3 *
4 * Copyright (C) 2015 Qualys Inc., Christopher Faulet <cfaulet@qualys.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
13#include <common/buffer.h>
14#include <common/cfgparse.h>
Willy Tarreaub96b77e2018-12-11 10:22:41 +010015#include <common/htx.h>
Willy Tarreau0108d902018-11-25 19:14:37 +010016#include <common/initcall.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010017#include <common/mini-clist.h>
18#include <common/standard.h>
19
20#include <types/compression.h>
21#include <types/filters.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010022#include <types/proxy.h>
23#include <types/sample.h>
24
25#include <proto/compression.h>
Christopher Faulet92d36382015-11-05 13:35:03 +010026#include <proto/filters.h>
Christopher Faulete6902cd2018-11-30 22:29:48 +010027#include <proto/http_htx.h>
Christopher Fauletfc9cfe42019-07-16 14:54:53 +020028#include <proto/http_ana.h>
Christopher Faulet3d97c902015-12-09 14:59:38 +010029#include <proto/sample.h>
30#include <proto/stream.h>
31
Christopher Fauletf4a4ef72018-12-07 17:39:53 +010032const char *http_comp_flt_id = "compression filter";
Christopher Faulet92d36382015-11-05 13:35:03 +010033
34struct flt_ops comp_ops;
35
Christopher Faulet92d36382015-11-05 13:35:03 +010036struct comp_state {
37 struct comp_ctx *comp_ctx; /* compression context */
38 struct comp_algo *comp_algo; /* compression algorithm if not NULL */
Christopher Faulet92d36382015-11-05 13:35:03 +010039};
40
Willy Tarreau8ceae722018-11-26 11:58:30 +010041/* Pools used to allocate comp_state structs */
42DECLARE_STATIC_POOL(pool_head_comp_state, "comp_state", sizeof(struct comp_state));
43
44static THREAD_LOCAL struct buffer tmpbuf;
45static THREAD_LOCAL struct buffer zbuf;
Willy Tarreau8ceae722018-11-26 11:58:30 +010046
Christopher Faulet92d36382015-11-05 13:35:03 +010047static int select_compression_request_header(struct comp_state *st,
48 struct stream *s,
49 struct http_msg *msg);
50static int select_compression_response_header(struct comp_state *st,
51 struct stream *s,
52 struct http_msg *msg);
Christopher Faulet27d93c32018-12-15 22:32:02 +010053static int set_compression_response_header(struct comp_state *st,
54 struct stream *s,
55 struct http_msg *msg);
Christopher Faulet92d36382015-11-05 13:35:03 +010056
Christopher Faulete6902cd2018-11-30 22:29:48 +010057static int htx_compression_buffer_init(struct htx *htx, struct buffer *out);
58static int htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
59 struct buffer *out);
60static int htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end);
61
Christopher Faulet92d36382015-11-05 13:35:03 +010062/***********************************************************************/
63static int
Christopher Faulete6902cd2018-11-30 22:29:48 +010064comp_flt_init(struct proxy *px, struct flt_conf *fconf)
65{
Christopher Faulet6e540952018-12-03 22:43:41 +010066 fconf->flags |= FLT_CFG_FL_HTX;
Christopher Faulete6902cd2018-11-30 22:29:48 +010067 return 0;
68}
69
70static int
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020071comp_flt_init_per_thread(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010072{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020073 if (!tmpbuf.size && b_alloc(&tmpbuf) == NULL)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010074 return -1;
Willy Tarreauc9fa0482018-07-10 17:43:27 +020075 if (!zbuf.size && b_alloc(&zbuf) == NULL)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010076 return -1;
Christopher Faulet92d36382015-11-05 13:35:03 +010077 return 0;
78}
79
80static void
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020081comp_flt_deinit_per_thread(struct proxy *px, struct flt_conf *fconf)
Christopher Faulet92d36382015-11-05 13:35:03 +010082{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020083 if (tmpbuf.size)
Christopher Faulet92d36382015-11-05 13:35:03 +010084 b_free(&tmpbuf);
Willy Tarreauc9fa0482018-07-10 17:43:27 +020085 if (zbuf.size)
Christopher Fauletb77c5c22015-12-07 16:48:42 +010086 b_free(&zbuf);
Christopher Faulet92d36382015-11-05 13:35:03 +010087}
88
89static int
90comp_start_analyze(struct stream *s, struct filter *filter, struct channel *chn)
91{
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +020092
Christopher Faulet92d36382015-11-05 13:35:03 +010093 if (filter->ctx == NULL) {
94 struct comp_state *st;
95
Willy Tarreaubafbe012017-11-24 17:34:44 +010096 st = pool_alloc_dirty(pool_head_comp_state);
Christopher Fauleta03d4ad2017-06-26 16:53:33 +020097 if (st == NULL)
Christopher Faulet92d36382015-11-05 13:35:03 +010098 return -1;
99
Christopher Faulet89f2b162019-07-15 21:16:04 +0200100 st->comp_algo = NULL;
101 st->comp_ctx = NULL;
102 filter->ctx = st;
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200103
104 /* Register post-analyzer on AN_RES_WAIT_HTTP because we need to
105 * analyze response headers before http-response rules execution
106 * to be sure we can use res.comp and res.comp_algo sample
107 * fetches */
108 filter->post_analyzers |= AN_RES_WAIT_HTTP;
Christopher Faulet92d36382015-11-05 13:35:03 +0100109 }
110 return 1;
111}
112
113static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100114comp_end_analyze(struct stream *s, struct filter *filter, struct channel *chn)
115{
116 struct comp_state *st = filter->ctx;
Christopher Faulet92d36382015-11-05 13:35:03 +0100117
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200118 if (!st)
Christopher Faulet92d36382015-11-05 13:35:03 +0100119 goto end;
120
Christopher Faulet92d36382015-11-05 13:35:03 +0100121 /* release any possible compression context */
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200122 if (st->comp_algo)
123 st->comp_algo->end(&st->comp_ctx);
Willy Tarreaubafbe012017-11-24 17:34:44 +0100124 pool_free(pool_head_comp_state, st);
Christopher Faulet92d36382015-11-05 13:35:03 +0100125 filter->ctx = NULL;
126 end:
127 return 1;
128}
129
130static int
Christopher Faulet1339d742016-05-11 16:48:33 +0200131comp_http_headers(struct stream *s, struct filter *filter, struct http_msg *msg)
132{
133 struct comp_state *st = filter->ctx;
134
135 if (!strm_fe(s)->comp && !s->be->comp)
136 goto end;
137
138 if (!(msg->chn->flags & CF_ISRESP))
139 select_compression_request_header(st, s, msg);
140 else {
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200141 /* Response headers have already been checked in
142 * comp_http_post_analyze callback. */
Christopher Faulet1339d742016-05-11 16:48:33 +0200143 if (st->comp_algo) {
Christopher Faulet27d93c32018-12-15 22:32:02 +0100144 if (!set_compression_response_header(st, s, msg))
145 goto end;
Christopher Faulet1339d742016-05-11 16:48:33 +0200146 register_data_filter(s, msg->chn, filter);
Christopher Faulet1339d742016-05-11 16:48:33 +0200147 }
148 }
149
150 end:
151 return 1;
152}
153
154static int
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200155comp_http_post_analyze(struct stream *s, struct filter *filter,
156 struct channel *chn, unsigned an_bit)
157{
158 struct http_txn *txn = s->txn;
159 struct http_msg *msg = &txn->rsp;
160 struct comp_state *st = filter->ctx;
161
162 if (an_bit != AN_RES_WAIT_HTTP)
163 goto end;
164
165 if (!strm_fe(s)->comp && !s->be->comp)
166 goto end;
167
168 select_compression_response_header(st, s, msg);
169
170 end:
171 return 1;
172}
173
174static int
Christopher Faulete6902cd2018-11-30 22:29:48 +0100175comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg,
176 unsigned int offset, unsigned int len)
177{
178 struct comp_state *st = filter->ctx;
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100179 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100180 struct htx_blk *blk;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100181 int ret, consumed = 0, to_forward = 0;
182
Christopher Fauletee847d42019-05-23 11:55:33 +0200183 for (blk = htx_get_first_blk(htx); blk && len; blk = htx_get_next_blk(htx, blk)) {
Christopher Faulete6902cd2018-11-30 22:29:48 +0100184 enum htx_blk_type type = htx_get_blk_type(blk);
185 uint32_t sz = htx_get_blksz(blk);
186 struct ist v;
187
Christopher Fauletee847d42019-05-23 11:55:33 +0200188 if (offset >= sz) {
189 offset -= sz;
190 continue;
191 }
192
Christopher Faulete6902cd2018-11-30 22:29:48 +0100193 switch (type) {
194 case HTX_BLK_UNUSED:
195 break;
196
197 case HTX_BLK_DATA:
198 v = htx_get_blk_value(htx, blk);
199 v.ptr += offset;
200 v.len -= offset;
201 if (v.len > len)
202 v.len = len;
203 if (htx_compression_buffer_init(htx, &trash) < 0) {
204 msg->chn->flags |= CF_WAKE_WRITE;
205 goto end;
206 }
207 ret = htx_compression_buffer_add_data(st, v.ptr, v.len, &trash);
208 if (ret < 0)
209 goto error;
210 if (htx_compression_buffer_end(st, &trash, 0) < 0)
211 goto error;
212 len -= ret;
213 consumed += ret;
214 to_forward += b_data(&trash);
215 if (ret == sz && !b_data(&trash)) {
216 offset = 0;
217 blk = htx_remove_blk(htx, blk);
218 continue;
219 }
220 v.len = ret;
221 blk = htx_replace_blk_value(htx, blk, v, ist2(b_head(&trash), b_data(&trash)));
222 break;
223
Christopher Faulete6902cd2018-11-30 22:29:48 +0100224 case HTX_BLK_TLR:
Christopher Faulet2d7c5392019-06-03 10:41:26 +0200225 case HTX_BLK_EOT:
Christopher Faulete6902cd2018-11-30 22:29:48 +0100226 case HTX_BLK_EOM:
227 if (msg->flags & HTTP_MSGF_COMPRESSING) {
228 if (htx_compression_buffer_init(htx, &trash) < 0) {
229 msg->chn->flags |= CF_WAKE_WRITE;
230 goto end;
231 }
232 if (htx_compression_buffer_end(st, &trash, 1) < 0)
233 goto error;
Christopher Fauletd238ae32018-12-21 15:10:25 +0100234 if (b_data(&trash)) {
Christopher Faulet86bc8df2019-06-11 10:38:38 +0200235 struct htx_blk *last = htx_add_last_data(htx, ist2(b_head(&trash), b_data(&trash)));
236 if (!last)
237 goto error;
238 blk = htx_get_next_blk(htx, last);
Christopher Fauletd238ae32018-12-21 15:10:25 +0100239 if (!blk)
240 goto error;
241 to_forward += b_data(&trash);
242 }
Christopher Faulete6902cd2018-11-30 22:29:48 +0100243 msg->flags &= ~HTTP_MSGF_COMPRESSING;
244 /* We let the mux add last empty chunk and empty trailers */
245 }
246 /* fall through */
247
248 default:
249 sz -= offset;
250 if (sz > len)
251 sz = len;
252 consumed += sz;
253 to_forward += sz;
254 len -= sz;
255 break;
256 }
257
258 offset = 0;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100259 }
260
261 end:
262 if (to_forward != consumed)
263 flt_update_offsets(filter, msg->chn, to_forward - consumed);
264
265 if (st->comp_ctx && st->comp_ctx->cur_lvl > 0) {
Willy Tarreauef6fd852019-02-04 11:48:03 +0100266 update_freq_ctr(&global.comp_bps_in, consumed);
Olivier Houchard43da3432019-03-08 18:50:27 +0100267 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_in, consumed);
268 _HA_ATOMIC_ADD(&s->be->be_counters.comp_in, consumed);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100269 update_freq_ctr(&global.comp_bps_out, to_forward);
Olivier Houchard43da3432019-03-08 18:50:27 +0100270 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_out, to_forward);
271 _HA_ATOMIC_ADD(&s->be->be_counters.comp_out, to_forward);
Willy Tarreauef6fd852019-02-04 11:48:03 +0100272 } else {
Olivier Houchard43da3432019-03-08 18:50:27 +0100273 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_byp, consumed);
274 _HA_ATOMIC_ADD(&s->be->be_counters.comp_byp, consumed);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100275 }
276 return to_forward;
277
278 error:
279 return -1;
280}
281
Christopher Faulet2fb28802015-12-01 10:40:57 +0100282
Christopher Faulet92d36382015-11-05 13:35:03 +0100283static int
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200284comp_http_end(struct stream *s, struct filter *filter,
285 struct http_msg *msg)
286{
287 struct comp_state *st = filter->ctx;
288
289 if (!(msg->chn->flags & CF_ISRESP) || !st || !st->comp_algo)
290 goto end;
291
292 if (strm_fe(s)->mode == PR_MODE_HTTP)
Olivier Houchard43da3432019-03-08 18:50:27 +0100293 _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.p.http.comp_rsp, 1);
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200294 if ((s->flags & SF_BE_ASSIGNED) && (s->be->mode == PR_MODE_HTTP))
Olivier Houchard43da3432019-03-08 18:50:27 +0100295 _HA_ATOMIC_ADD(&s->be->be_counters.p.http.comp_rsp, 1);
Christopher Fauletd60b3cf2017-06-26 11:47:13 +0200296 end:
297 return 1;
298}
Christopher Faulet27d93c32018-12-15 22:32:02 +0100299
Christopher Faulet89f2b162019-07-15 21:16:04 +0200300/***********************************************************************/
Christopher Faulet27d93c32018-12-15 22:32:02 +0100301static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200302set_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulet27d93c32018-12-15 22:32:02 +0100303{
304 struct htx *htx = htxbuf(&msg->chn->buf);
Tim Duesterhusb229f012019-01-29 16:38:56 +0100305 struct http_hdr_ctx ctx;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100306
307 /*
308 * Add Content-Encoding header when it's not identity encoding.
309 * RFC 2616 : Identity encoding: This content-coding is used only in the
310 * Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding
311 * header.
312 */
313 if (st->comp_algo->cfg_name_len != 8 || memcmp(st->comp_algo->cfg_name, "identity", 8) != 0) {
314 struct ist v = ist2(st->comp_algo->ua_name, st->comp_algo->ua_name_len);
315
316 if (!http_add_header(htx, ist("Content-Encoding"), v))
317 goto error;
318 }
319
320 /* remove Content-Length header */
321 if (msg->flags & HTTP_MSGF_CNT_LEN) {
Christopher Faulet27d93c32018-12-15 22:32:02 +0100322 ctx.blk = NULL;
323 while (http_find_header(htx, ist("Content-Length"), &ctx, 1))
324 http_remove_header(htx, &ctx);
325 }
326
327 /* add "Transfer-Encoding: chunked" header */
328 if (!(msg->flags & HTTP_MSGF_TE_CHNK)) {
329 if (!http_add_header(htx, ist("Transfer-Encoding"), ist("chunked")))
330 goto error;
331 }
332
Tim Duesterhusb229f012019-01-29 16:38:56 +0100333 /* convert "ETag" header to a weak ETag */
334 ctx.blk = NULL;
335 if (http_find_header(htx, ist("ETag"), &ctx, 1)) {
336 if (ctx.value.ptr[0] == '"') {
337 /* This a strong ETag. Convert it to a weak one. */
338 struct ist v = ist2(trash.area, 0);
339 if (istcat(&v, ist("W/"), trash.size) == -1 || istcat(&v, ctx.value, trash.size) == -1)
340 goto error;
341
342 if (!http_replace_header_value(htx, &ctx, v))
343 goto error;
344 }
345 }
346
Tim Duesterhus721d6862019-06-17 16:10:07 +0200347 if (!http_add_header(htx, ist("Vary"), ist("Accept-Encoding")))
348 goto error;
349
Christopher Faulet27d93c32018-12-15 22:32:02 +0100350 return 1;
351
352 error:
353 st->comp_algo->end(&st->comp_ctx);
354 st->comp_algo = NULL;
355 return 0;
356}
357
Christopher Faulet3d97c902015-12-09 14:59:38 +0100358/*
359 * Selects a compression algorithm depending on the client request.
360 */
Christopher Faulete6902cd2018-11-30 22:29:48 +0100361static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200362select_compression_request_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100363{
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100364 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100365 struct http_hdr_ctx ctx;
366 struct comp_algo *comp_algo = NULL;
367 struct comp_algo *comp_algo_back = NULL;
368
369 /* Disable compression for older user agents announcing themselves as "Mozilla/4"
370 * unless they are known good (MSIE 6 with XP SP2, or MSIE 7 and later).
371 * See http://zoompf.com/2012/02/lose-the-wait-http-compression for more details.
372 */
373 ctx.blk = NULL;
374 if (http_find_header(htx, ist("User-Agent"), &ctx, 1) &&
375 ctx.value.len >= 9 &&
376 memcmp(ctx.value.ptr, "Mozilla/4", 9) == 0 &&
377 (ctx.value.len < 31 ||
378 memcmp(ctx.value.ptr + 25, "MSIE ", 5) != 0 ||
379 *(ctx.value.ptr + 30) < '6' ||
380 (*(ctx.value.ptr + 30) == '6' &&
381 (ctx.value.len < 54 || memcmp(ctx.value.ptr + 51, "SV1", 3) != 0)))) {
382 st->comp_algo = NULL;
383 return 0;
384 }
385
386 /* search for the algo in the backend in priority or the frontend */
387 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
388 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
389 int best_q = 0;
390
391 ctx.blk = NULL;
392 while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 0)) {
393 const char *qval;
394 int q;
395 int toklen;
396
397 /* try to isolate the token from the optional q-value */
398 toklen = 0;
399 while (toklen < ctx.value.len && HTTP_IS_TOKEN(*(ctx.value.ptr + toklen)))
400 toklen++;
401
402 qval = ctx.value.ptr + toklen;
403 while (1) {
404 while (qval < ctx.value.ptr + ctx.value.len && HTTP_IS_LWS(*qval))
405 qval++;
406
407 if (qval >= ctx.value.ptr + ctx.value.len || *qval != ';') {
408 qval = NULL;
409 break;
410 }
411 qval++;
412
413 while (qval < ctx.value.ptr + ctx.value.len && HTTP_IS_LWS(*qval))
414 qval++;
415
416 if (qval >= ctx.value.ptr + ctx.value.len) {
417 qval = NULL;
418 break;
419 }
420 if (strncmp(qval, "q=", MIN(ctx.value.ptr + ctx.value.len - qval, 2)) == 0)
421 break;
422
423 while (qval < ctx.value.ptr + ctx.value.len && *qval != ';')
424 qval++;
425 }
426
427 /* here we have qval pointing to the first "q=" attribute or NULL if not found */
428 q = qval ? http_parse_qvalue(qval + 2, NULL) : 1000;
429
430 if (q <= best_q)
431 continue;
432
433 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
434 if (*(ctx.value.ptr) == '*' ||
435 word_match(ctx.value.ptr, toklen, comp_algo->ua_name, comp_algo->ua_name_len)) {
436 st->comp_algo = comp_algo;
437 best_q = q;
438 break;
439 }
440 }
441 }
442 }
443
444 /* remove all occurrences of the header when "compression offload" is set */
445 if (st->comp_algo) {
446 if ((s->be->comp && s->be->comp->offload) ||
447 (strm_fe(s)->comp && strm_fe(s)->comp->offload)) {
448 http_remove_header(htx, &ctx);
449 ctx.blk = NULL;
450 while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 1))
451 http_remove_header(htx, &ctx);
452 }
Christopher Faulet3d97c902015-12-09 14:59:38 +0100453 return 1;
454 }
455
456 /* identity is implicit does not require headers */
Christopher Faulet92d36382015-11-05 13:35:03 +0100457 if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
458 (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
Christopher Faulet3d97c902015-12-09 14:59:38 +0100459 for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
460 if (comp_algo->cfg_name_len == 8 && memcmp(comp_algo->cfg_name, "identity", 8) == 0) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100461 st->comp_algo = comp_algo;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100462 return 1;
463 }
464 }
465 }
466
Christopher Faulet92d36382015-11-05 13:35:03 +0100467 st->comp_algo = NULL;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100468 return 0;
469}
470
471/*
472 * Selects a comression algorithm depending of the server response.
473 */
Christopher Faulet92d36382015-11-05 13:35:03 +0100474static int
Christopher Faulet89f2b162019-07-15 21:16:04 +0200475select_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100476{
Christopher Faulet27ba2dc2018-12-05 11:53:24 +0100477 struct htx *htx = htxbuf(&msg->chn->buf);
Christopher Faulete6902cd2018-11-30 22:29:48 +0100478 struct http_txn *txn = s->txn;
479 struct http_hdr_ctx ctx;
480 struct comp_type *comp_type;
481
482 /* no common compression algorithm was found in request header */
483 if (st->comp_algo == NULL)
484 goto fail;
485
Christopher Faulet1d3613a2019-01-07 14:41:59 +0100486 /* compression already in progress */
487 if (msg->flags & HTTP_MSGF_COMPRESSING)
488 goto fail;
489
Christopher Faulete6902cd2018-11-30 22:29:48 +0100490 /* HTTP < 1.1 should not be compressed */
491 if (!(msg->flags & HTTP_MSGF_VER_11) || !(txn->req.flags & HTTP_MSGF_VER_11))
492 goto fail;
493
494 if (txn->meth == HTTP_METH_HEAD)
495 goto fail;
496
497 /* compress 200,201,202,203 responses only */
498 if ((txn->status != 200) &&
499 (txn->status != 201) &&
500 (txn->status != 202) &&
501 (txn->status != 203))
502 goto fail;
503
Christopher Fauletc963eb22018-12-21 14:53:54 +0100504 if (!(msg->flags & HTTP_MSGF_XFER_LEN) || msg->flags & HTTP_MSGF_BODYLESS)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100505 goto fail;
506
507 /* content is already compressed */
508 ctx.blk = NULL;
509 if (http_find_header(htx, ist("Content-Encoding"), &ctx, 1))
510 goto fail;
511
512 /* no compression when Cache-Control: no-transform is present in the message */
513 ctx.blk = NULL;
514 while (http_find_header(htx, ist("Cache-Control"), &ctx, 0)) {
515 if (word_match(ctx.value.ptr, ctx.value.len, "no-transform", 12))
516 goto fail;
517 }
518
Tim Duesterhusb229f012019-01-29 16:38:56 +0100519 /* no compression when ETag is malformed */
520 ctx.blk = NULL;
521 if (http_find_header(htx, ist("ETag"), &ctx, 1)) {
522 if (!(((ctx.value.len >= 4 && memcmp(ctx.value.ptr, "W/\"", 3) == 0) || /* Either a weak ETag */
523 (ctx.value.len >= 2 && ctx.value.ptr[0] == '"')) && /* or strong ETag */
524 ctx.value.ptr[ctx.value.len - 1] == '"')) {
525 goto fail;
526 }
527 }
528 /* no compression when multiple ETags are present
529 * Note: Do not reset ctx.blk!
530 */
531 if (http_find_header(htx, ist("ETag"), &ctx, 1))
532 goto fail;
533
Christopher Faulete6902cd2018-11-30 22:29:48 +0100534 comp_type = NULL;
535
536 /* we don't want to compress multipart content-types, nor content-types that are
537 * not listed in the "compression type" directive if any. If no content-type was
538 * found but configuration requires one, we don't compress either. Backend has
539 * the priority.
540 */
541 ctx.blk = NULL;
542 if (http_find_header(htx, ist("Content-Type"), &ctx, 1)) {
543 if (ctx.value.len >= 9 && strncasecmp("multipart", ctx.value.ptr, 9) == 0)
544 goto fail;
545
546 if ((s->be->comp && (comp_type = s->be->comp->types)) ||
547 (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types))) {
548 for (; comp_type; comp_type = comp_type->next) {
549 if (ctx.value.len >= comp_type->name_len &&
550 strncasecmp(ctx.value.ptr, comp_type->name, comp_type->name_len) == 0)
551 /* this Content-Type should be compressed */
552 break;
553 }
554 /* this Content-Type should not be compressed */
555 if (comp_type == NULL)
556 goto fail;
557 }
558 }
559 else { /* no content-type header */
560 if ((s->be->comp && s->be->comp->types) ||
561 (strm_fe(s)->comp && strm_fe(s)->comp->types))
562 goto fail; /* a content-type was required */
563 }
564
565 /* limit compression rate */
566 if (global.comp_rate_lim > 0)
567 if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim)
568 goto fail;
569
570 /* limit cpu usage */
Willy Tarreau81036f22019-05-20 19:24:50 +0200571 if (ti->idle_pct < compress_min_idle)
Christopher Faulete6902cd2018-11-30 22:29:48 +0100572 goto fail;
573
574 /* initialize compression */
575 if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0)
576 goto fail;
Christopher Faulete6902cd2018-11-30 22:29:48 +0100577 msg->flags |= HTTP_MSGF_COMPRESSING;
578 return 1;
579
580 deinit_comp_ctx:
581 st->comp_algo->end(&st->comp_ctx);
582 fail:
583 st->comp_algo = NULL;
584 return 0;
585}
586
Christopher Faulet3d97c902015-12-09 14:59:38 +0100587/***********************************************************************/
Christopher Faulete6902cd2018-11-30 22:29:48 +0100588static int
589htx_compression_buffer_init(struct htx *htx, struct buffer *out)
590{
591 /* output stream requires at least 10 bytes for the gzip header, plus
592 * at least 8 bytes for the gzip trailer (crc+len), plus a possible
593 * plus at most 5 bytes per 32kB block and 2 bytes to close the stream.
594 */
595 if (htx_free_space(htx) < 20 + 5 * ((htx->data + 32767) >> 15))
596 return -1;
597 b_reset(out);
598 return 0;
599}
600
Christopher Faulete6902cd2018-11-30 22:29:48 +0100601static int
602htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
603 struct buffer *out)
604{
605 return st->comp_algo->add_data(st->comp_ctx, data, len, out);
606}
607
Christopher Faulete6902cd2018-11-30 22:29:48 +0100608static int
609htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end)
610{
611 if (end)
612 return st->comp_algo->finish(st->comp_ctx, out);
613 else
614 return st->comp_algo->flush(st->comp_ctx, out);
615}
616
Christopher Faulet3d97c902015-12-09 14:59:38 +0100617
618/***********************************************************************/
Christopher Faulet92d36382015-11-05 13:35:03 +0100619struct flt_ops comp_ops = {
Christopher Faulete6902cd2018-11-30 22:29:48 +0100620 .init = comp_flt_init,
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +0200621 .init_per_thread = comp_flt_init_per_thread,
622 .deinit_per_thread = comp_flt_deinit_per_thread,
Christopher Faulet92d36382015-11-05 13:35:03 +0100623
624 .channel_start_analyze = comp_start_analyze,
Christopher Faulet92d36382015-11-05 13:35:03 +0100625 .channel_end_analyze = comp_end_analyze,
Christopher Faulet3dc860d2017-09-15 11:39:36 +0200626 .channel_post_analyze = comp_http_post_analyze,
Christopher Faulet92d36382015-11-05 13:35:03 +0100627
Christopher Faulet1339d742016-05-11 16:48:33 +0200628 .http_headers = comp_http_headers,
Christopher Faulete6902cd2018-11-30 22:29:48 +0100629 .http_payload = comp_http_payload,
630 .http_end = comp_http_end,
Christopher Faulet92d36382015-11-05 13:35:03 +0100631};
632
Christopher Faulet3d97c902015-12-09 14:59:38 +0100633static int
634parse_compression_options(char **args, int section, struct proxy *proxy,
635 struct proxy *defpx, const char *file, int line,
636 char **err)
637{
Christopher Faulet92d36382015-11-05 13:35:03 +0100638 struct comp *comp;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100639
640 if (proxy->comp == NULL) {
Vincent Bernat02779b62016-04-03 13:48:43 +0200641 comp = calloc(1, sizeof(*comp));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100642 proxy->comp = comp;
643 }
644 else
645 comp = proxy->comp;
646
647 if (!strcmp(args[1], "algo")) {
648 struct comp_ctx *ctx;
649 int cur_arg = 2;
650
651 if (!*args[cur_arg]) {
652 memprintf(err, "parsing [%s:%d] : '%s' expects <algorithm>\n",
653 file, line, args[0]);
654 return -1;
655 }
656 while (*(args[cur_arg])) {
657 if (comp_append_algo(comp, args[cur_arg]) < 0) {
658 memprintf(err, "'%s' : '%s' is not a supported algorithm.\n",
659 args[0], args[cur_arg]);
660 return -1;
661 }
662 if (proxy->comp->algos->init(&ctx, 9) == 0)
663 proxy->comp->algos->end(&ctx);
664 else {
665 memprintf(err, "'%s' : Can't init '%s' algorithm.\n",
666 args[0], args[cur_arg]);
667 return -1;
668 }
669 cur_arg++;
670 continue;
671 }
672 }
673 else if (!strcmp(args[1], "offload"))
674 comp->offload = 1;
675 else if (!strcmp(args[1], "type")) {
676 int cur_arg = 2;
677
678 if (!*args[cur_arg]) {
679 memprintf(err, "'%s' expects <type>\n", args[0]);
680 return -1;
681 }
682 while (*(args[cur_arg])) {
683 comp_append_type(comp, args[cur_arg]);
684 cur_arg++;
685 continue;
686 }
687 }
688 else {
689 memprintf(err, "'%s' expects 'algo', 'type' or 'offload'\n",
690 args[0]);
691 return -1;
692 }
693
694 return 0;
695}
696
Christopher Faulet92d36382015-11-05 13:35:03 +0100697static int
698parse_http_comp_flt(char **args, int *cur_arg, struct proxy *px,
Thierry Fournier3610c392016-04-13 18:27:51 +0200699 struct flt_conf *fconf, char **err, void *private)
Christopher Faulet92d36382015-11-05 13:35:03 +0100700{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100701 struct flt_conf *fc, *back;
Christopher Faulet92d36382015-11-05 13:35:03 +0100702
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100703 list_for_each_entry_safe(fc, back, &px->filter_configs, list) {
704 if (fc->id == http_comp_flt_id) {
Christopher Faulet92d36382015-11-05 13:35:03 +0100705 memprintf(err, "%s: Proxy supports only one compression filter\n", px->id);
706 return -1;
707 }
708 }
709
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100710 fconf->id = http_comp_flt_id;
711 fconf->conf = NULL;
712 fconf->ops = &comp_ops;
Christopher Faulet92d36382015-11-05 13:35:03 +0100713 (*cur_arg)++;
714
715 return 0;
716}
717
718
719int
Christopher Fauletc9df7f72018-12-10 16:14:04 +0100720check_implicit_http_comp_flt(struct proxy *proxy)
Christopher Faulet92d36382015-11-05 13:35:03 +0100721{
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100722 struct flt_conf *fconf;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100723 int explicit = 0;
724 int comp = 0;
Christopher Faulet92d36382015-11-05 13:35:03 +0100725 int err = 0;
726
727 if (proxy->comp == NULL)
728 goto end;
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100729 if (!LIST_ISEMPTY(&proxy->filter_configs)) {
730 list_for_each_entry(fconf, &proxy->filter_configs, list) {
731 if (fconf->id == http_comp_flt_id)
Christopher Faulet27d93c32018-12-15 22:32:02 +0100732 comp = 1;
733 else if (fconf->id == cache_store_flt_id) {
734 if (comp) {
735 ha_alert("config: %s '%s': unable to enable the compression filter "
736 "before any cache filter.\n",
737 proxy_type_str(proxy), proxy->id);
738 err++;
739 goto end;
740 }
741 }
Christopher Faulet78fbb9f2019-08-11 23:11:03 +0200742 else if (fconf->id == fcgi_flt_id)
743 continue;
Christopher Faulet27d93c32018-12-15 22:32:02 +0100744 else
745 explicit = 1;
Christopher Faulet92d36382015-11-05 13:35:03 +0100746 }
Christopher Faulet27d93c32018-12-15 22:32:02 +0100747 }
748 if (comp)
749 goto end;
750 else if (explicit) {
751 ha_alert("config: %s '%s': require an explicit filter declaration to use "
752 "HTTP compression\n", proxy_type_str(proxy), proxy->id);
Christopher Faulet92d36382015-11-05 13:35:03 +0100753 err++;
754 goto end;
755 }
756
Christopher Faulet27d93c32018-12-15 22:32:02 +0100757 /* Implicit declaration of the compression filter is always the last
758 * one */
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100759 fconf = calloc(1, sizeof(*fconf));
760 if (!fconf) {
Christopher Faulet767a84b2017-11-24 16:50:31 +0100761 ha_alert("config: %s '%s': out of memory\n",
762 proxy_type_str(proxy), proxy->id);
Christopher Faulet92d36382015-11-05 13:35:03 +0100763 err++;
764 goto end;
765 }
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100766 fconf->id = http_comp_flt_id;
767 fconf->conf = NULL;
768 fconf->ops = &comp_ops;
769 LIST_ADDQ(&proxy->filter_configs, &fconf->list);
Christopher Faulet92d36382015-11-05 13:35:03 +0100770 end:
771 return err;
772}
773
774/*
775 * boolean, returns true if compression is used (either gzip or deflate) in the
776 * response.
777 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100778static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100779smp_fetch_res_comp(const struct arg *args, struct sample *smp, const char *kw,
780 void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100781{
Willy Tarreaube508f12016-03-10 11:47:01 +0100782 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100783
Christopher Faulet3d97c902015-12-09 14:59:38 +0100784 smp->data.type = SMP_T_BOOL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100785 smp->data.u.sint = (txn && (txn->rsp.flags & HTTP_MSGF_COMPRESSING));
Christopher Faulet3d97c902015-12-09 14:59:38 +0100786 return 1;
787}
788
Christopher Faulet92d36382015-11-05 13:35:03 +0100789/*
790 * string, returns algo
791 */
Christopher Faulet3d97c902015-12-09 14:59:38 +0100792static int
Christopher Faulet92d36382015-11-05 13:35:03 +0100793smp_fetch_res_comp_algo(const struct arg *args, struct sample *smp,
794 const char *kw, void *private)
Christopher Faulet3d97c902015-12-09 14:59:38 +0100795{
Willy Tarreaube508f12016-03-10 11:47:01 +0100796 struct http_txn *txn = smp->strm ? smp->strm->txn : NULL;
Christopher Faulet92d36382015-11-05 13:35:03 +0100797 struct filter *filter;
798 struct comp_state *st;
799
Christopher Faulet03d85532017-09-15 10:14:43 +0200800 if (!txn || !(txn->rsp.flags & HTTP_MSGF_COMPRESSING))
Christopher Faulet3d97c902015-12-09 14:59:38 +0100801 return 0;
802
Christopher Fauletfcf035c2015-12-03 11:48:03 +0100803 list_for_each_entry(filter, &strm_flt(smp->strm)->filters, list) {
Christopher Faulet443ea1a2016-02-04 13:40:26 +0100804 if (FLT_ID(filter) != http_comp_flt_id)
Christopher Faulet92d36382015-11-05 13:35:03 +0100805 continue;
806
807 if (!(st = filter->ctx))
808 break;
809
810 smp->data.type = SMP_T_STR;
811 smp->flags = SMP_F_CONST;
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200812 smp->data.u.str.area = st->comp_algo->cfg_name;
813 smp->data.u.str.data = st->comp_algo->cfg_name_len;
Christopher Faulet92d36382015-11-05 13:35:03 +0100814 return 1;
815 }
816 return 0;
Christopher Faulet3d97c902015-12-09 14:59:38 +0100817}
818
819/* Declare the config parser for "compression" keyword */
820static struct cfg_kw_list cfg_kws = {ILH, {
821 { CFG_LISTEN, "compression", parse_compression_options },
822 { 0, NULL, NULL },
823 }
824};
825
Willy Tarreau0108d902018-11-25 19:14:37 +0100826INITCALL1(STG_REGISTER, cfg_register_keywords, &cfg_kws);
827
Christopher Faulet92d36382015-11-05 13:35:03 +0100828/* Declare the filter parser for "compression" keyword */
829static struct flt_kw_list filter_kws = { "COMP", { }, {
Thierry Fournier3610c392016-04-13 18:27:51 +0200830 { "compression", parse_http_comp_flt, NULL },
831 { NULL, NULL, NULL },
Christopher Faulet92d36382015-11-05 13:35:03 +0100832 }
833};
834
Willy Tarreau0108d902018-11-25 19:14:37 +0100835INITCALL1(STG_REGISTER, flt_register_keywords, &filter_kws);
836
Christopher Faulet3d97c902015-12-09 14:59:38 +0100837/* Note: must not be declared <const> as its list will be overwritten */
838static struct sample_fetch_kw_list sample_fetch_keywords = {ILH, {
Christopher Faulet92d36382015-11-05 13:35:03 +0100839 { "res.comp", smp_fetch_res_comp, 0, NULL, SMP_T_BOOL, SMP_USE_HRSHP },
840 { "res.comp_algo", smp_fetch_res_comp_algo, 0, NULL, SMP_T_STR, SMP_USE_HRSHP },
841 { /* END */ },
842 }
843};
Christopher Faulet3d97c902015-12-09 14:59:38 +0100844
Willy Tarreau0108d902018-11-25 19:14:37 +0100845INITCALL1(STG_REGISTER, sample_register_fetches, &sample_fetch_keywords);