MEDIUM: HTTP compression (zlib library support)
This commit introduces HTTP compression using the zlib library.
http_response_forward_body has been modified to call the compression
functions.
This feature includes 3 algorithms: identity, gzip and deflate:
* identity: this is mostly for debugging, and it was useful for
developping the compression feature. With Content-Length in input, it
is making each chunk with the data available in the current buffer.
With chunks in input, it is rechunking, the output chunks will be
bigger or smaller depending of the size of the input chunk and the
size of the buffer. Identity does not apply any change on data.
* gzip: same as identity, but applying a gzip compression. The data
are deflated using the Z_NO_FLUSH flag in zlib. When there is no more
data in the input buffer, it flushes the data in the output buffer
(Z_SYNC_FLUSH). At the end of data, when it receives the last chunk in
input, or when there is no more data to read, it writes the end of
data with Z_FINISH and the ending chunk.
* deflate: same as gzip, but with deflate algorithm and zlib format.
Note that this algorithm has ambiguous support on many browsers and
no support at all from recent ones. It is strongly recommended not
to use it for anything else than experimentation.
You can't choose the compression ratio at the moment, it will be set to
Z_BEST_SPEED (1), as tests have shown very little benefit in terms of
compression ration when going above for HTML contents, at the cost of
a massive CPU impact.
Compression will be activated depending of the Accept-Encoding request
header. With identity, it does not take care of that header.
To build HAProxy with zlib support, use USE_ZLIB=1 in the make
parameters.
This work was initially started by David Du Colombier at Exceliance.
diff --git a/Makefile b/Makefile
index 5652d59..02c9291 100644
--- a/Makefile
+++ b/Makefile
@@ -30,6 +30,7 @@
# USE_FUTEX : enable use of futex on kernel 2.6. Automatic.
# USE_ACCEPT4 : enable use of accept4() on linux. Automatic.
# USE_MY_ACCEPT4 : use own implemention of accept4() if glibc < 2.10.
+# USE_ZLIB : enable zlib library support.
#
# Options can be forced by specifying "USE_xxx=1" or can be disabled by using
# "USE_xxx=" (empty string).
@@ -402,6 +403,12 @@
BUILD_OPTIONS += $(call ignore_implicit,USE_GETADDRINFO)
endif
+ifneq ($(USE_ZLIB),)
+OPTIONS_CFLAGS += -DUSE_ZLIB
+BUILD_OPTIONS += $(call ignore_implicit,USE_ZLIB)
+OPTIONS_LDFLAGS += -lz
+endif
+
ifneq ($(USE_POLL),)
OPTIONS_CFLAGS += -DENABLE_POLL
OPTIONS_OBJS += src/ev_poll.o
@@ -594,6 +601,7 @@
src/stream_interface.o src/dumpstats.o src/proto_tcp.o \
src/session.o src/hdr_idx.o src/ev_select.o src/signal.o \
src/acl.o src/sample.o src/memory.o src/freq_ctr.o src/auth.o \
+ src/compression.o
EBTREE_OBJS = $(EBTREE_DIR)/ebtree.o \
$(EBTREE_DIR)/eb32tree.o $(EBTREE_DIR)/eb64tree.o \
diff --git a/README b/README
index b886eee..d403711 100644
--- a/README
+++ b/README
@@ -89,6 +89,10 @@
if the build fails due to missing symbols such as deflateInit(), then try again
with "ADDLIB=-lz".
+It is also possible to include native support for ZLIB to benefit from HTTP
+compression. For this, pass "USE_ZLIB=1" on the "make" command line and ensure
+that zlib is present on the system.
+
By default, the DEBUG variable is set to '-g' to enable debug symbols. It is
not wise to disable it on uncommon systems, because it's often the only way to
get a complete core when you need one. Otherwise, you can set DEBUG to '-s' to
@@ -102,11 +106,12 @@
$ make -f Makefile.bsd REGEX=pcre DEBUG= COPTS.generic="-Os -fomit-frame-pointer -mgnu"
-And on a recent Linux with SSL support :
+And on a recent Linux with SSL and ZLIB support :
- $ make TARGET=linux2628 CPU=native USE_PCRE=1 USE_OPENSSL=1
+ $ make TARGET=linux2628 CPU=native USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1
-In order to build a 32-bit binary on an x86_64 Linux system with SSL support :
+In order to build a 32-bit binary on an x86_64 Linux system with SSL support
+without support for compression but when OpenSSL requires ZLIB anyway :
$ make TARGET=linux26 ARCH=i386 USE_OPENSSL=1 ADDLIB=-lz
diff --git a/doc/configuration.txt b/doc/configuration.txt
index 69f4591..9270e16 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -994,6 +994,7 @@
capture request header - X X -
capture response header - X X -
clitimeout (deprecated) X X X -
+compression X X X X
contimeout (deprecated) X - X X
cookie X - X X
default-server X - X X
@@ -1777,6 +1778,18 @@
See also : "timeout client", "timeout http-request", "timeout server", and
"srvtimeout".
+compression algo [ gzip ] ...
+compression type ...
+ Enable HTTP compression.
+ May be used in sections : defaults | frontend | listen | backend
+ yes | yes | yes | yes
+ Arguments :
+ algo is followed by the list of supported compression algorithms.
+ type is followed by the list of MIME types that will be compressed.
+
+ Examples :
+ compression algo gzip
+ compression type text/html text/plain
contimeout <timeout> (deprecated)
Set the maximum time to wait for a connection attempt to a server to succeed.
diff --git a/include/proto/compression.h b/include/proto/compression.h
new file mode 100644
index 0000000..6f1a26a
--- /dev/null
+++ b/include/proto/compression.h
@@ -0,0 +1,66 @@
+/*
+ * include/proto/compression.h
+ * This file defines function prototypes for compression.
+ *
+ * Copyright 2012 (C) Exceliance, David Du Colombier <dducolombier@exceliance.fr>
+ * William Lallemand <wlallemand@exceliance.fr>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation, version 2.1
+ * exclusively.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _PROTO_COMP_H
+#define _PROTO_COMP_H
+
+#include <types/compression.h>
+
+int comp_append_type(struct comp *comp, const char *type);
+int comp_append_algo(struct comp *comp, const char *algo);
+
+int http_emit_chunk_size(char *out, unsigned int chksz, int add_crlf);
+int http_compression_buffer_init(struct session *s, struct buffer *in, struct buffer *out);
+int http_compression_buffer_add_data(struct session *s, struct buffer *in, struct buffer *out);
+int http_compression_buffer_end(struct session *s, struct buffer **in, struct buffer **out, int end);
+
+int identity_init(void *v, int level);
+int identity_add_data(void *v, const char *in_data, int in_len, char *out_data, int out_len);
+int identity_flush(void *comp_ctx, struct buffer *out, int flag);
+int identity_reset(void *comp_ctx);
+int identity_end(void *comp_ctx);
+
+
+#ifdef USE_ZLIB
+
+int deflate_init(void *comp_ctx, int level);
+int deflate_add_data(void *v, const char *in_data, int in_len, char *out_data, int out_len);
+int deflate_flush(void *comp_ctx, struct buffer *out, int flag);
+int deflate_reset(void *comp_ctx);
+int deflate_end(void *comp_ctx);
+
+int gzip_init(void *comp_ctx, int level);
+int gzip_add_data(void *v, const char *in_data, int in_len, char *out_data, int out_len);
+int gzip_flush(void *comp_ctx, struct buffer *out, int flag);
+int gzip_reset(void *comp_ctx);
+int gzip_end(void *comp_ctx);
+
+#endif /* USE_ZLIB */
+
+#endif /* _PROTO_COMP_H */
+
+/*
+ * Local variables:
+ * c-indent-level: 8
+ * c-basic-offset: 8
+ * End:
+ */
diff --git a/include/types/compression.h b/include/types/compression.h
new file mode 100644
index 0000000..96dd107
--- /dev/null
+++ b/include/types/compression.h
@@ -0,0 +1,63 @@
+/*
+ * include/types/compression.h
+ * This file defines everything related to compression.
+ *
+ * Copyright 2012 Exceliance, David Du Colombier <dducolombier@exceliance.fr>
+ William Lallemand <wlallemand@exceliance.fr>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation, version 2.1
+ * exclusively.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _TYPES_COMP_H
+#define _TYPES_COMP_H
+
+#include <zlib.h>
+
+struct comp {
+ struct comp_algo *algos;
+ struct comp_type *types;
+};
+
+struct comp_algo {
+ char *name;
+ int name_len;
+ int (*init)(void *, int);
+ int (*add_data)(void *v, const char *in_data, int in_len, char *out_data, int out_len);
+ int (*flush)(void *v, struct buffer *out, int flag);
+ int (*reset)(void *v);
+ int (*end)(void *v);
+ struct comp_algo *next;
+};
+
+union comp_ctx {
+ z_stream strm; /* zlib */
+};
+
+struct comp_type {
+ char *name;
+ int name_len;
+ struct comp_type *next;
+};
+
+
+#endif /* _TYPES_COMP_H */
+
+/*
+ * Local variables:
+ * c-indent-level: 8
+ * c-basic-offset: 8
+ * End:
+ */
+
diff --git a/include/types/proxy.h b/include/types/proxy.h
index 9bfa68d..343d4ad 100644
--- a/include/types/proxy.h
+++ b/include/types/proxy.h
@@ -363,6 +363,7 @@
struct list listeners; /* list of listeners belonging to this frontend */
} conf; /* config information */
void *parent; /* parent of the proxy when applicable */
+ struct comp *comp; /* http compression */
};
struct switching_rule {
diff --git a/include/types/session.h b/include/types/session.h
index e3ccde8..4726a19 100644
--- a/include/types/session.h
+++ b/include/types/session.h
@@ -32,6 +32,7 @@
#include <common/mini-clist.h>
#include <types/channel.h>
+#include <types/compression.h>
#include <types/proto_http.h>
#include <types/proxy.h>
#include <types/queue.h>
@@ -155,6 +156,8 @@
void (*srv_error)(struct session *s, /* the function to call upon unrecoverable server errors (or NULL) */
struct stream_interface *si);
unsigned int uniq_id; /* unique ID used for the traces */
+ struct comp_algo *comp_algo; /* HTTP compression algorithm if not NULL */
+ union comp_ctx comp_ctx; /* HTTP compression context */
char *unique_id; /* custom unique ID */
};
diff --git a/src/cfgparse.c b/src/cfgparse.c
index 3f43f58..c6138fd 100644
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -33,6 +33,7 @@
#include <common/uri_auth.h>
#include <types/capture.h>
+#include <types/compression.h>
#include <types/global.h>
#include <types/peers.h>
@@ -41,6 +42,7 @@
#include <proto/backend.h>
#include <proto/channel.h>
#include <proto/checks.h>
+#include <proto/compression.h>
#include <proto/dumpstats.h>
#include <proto/frontend.h>
#include <proto/hdr_idx.h>
@@ -1671,6 +1673,13 @@
if (defproxy.header_unique_id)
curproxy->header_unique_id = strdup(defproxy.header_unique_id);
+ /* default compression options */
+ if (defproxy.comp != NULL) {
+ curproxy->comp = calloc(1, sizeof(struct comp));
+ curproxy->comp->algos = defproxy.comp->algos;
+ curproxy->comp->types = defproxy.comp->types;
+ }
+
curproxy->grace = defproxy.grace;
curproxy->conf.used_listener_id = EB_ROOT;
curproxy->conf.used_server_id = EB_ROOT;
@@ -5236,6 +5245,57 @@
free(err);
}
}
+ else if (!strcmp(args[0], "compression")) {
+ struct comp *comp;
+ if (curproxy->comp == NULL) {
+ comp = calloc(1, sizeof(struct comp));
+ curproxy->comp = comp;
+ } else {
+ comp = curproxy->comp;
+ }
+
+ if (!strcmp(args[1], "algo")) {
+ int cur_arg;
+ cur_arg = 2;
+ if (!*args[cur_arg]) {
+ Alert("parsing [%s:%d] : '%s' expects <algorithm>\n",
+ file, linenum, args[0]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+ while (*(args[cur_arg])) {
+ if (comp_append_algo(comp, args[cur_arg]) < 0) {
+ Alert("parsing [%s:%d] : '%s' : '%s' is not a supported algorithm.\n",
+ file, linenum, args[0], args[cur_arg]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+ cur_arg ++;
+ continue;
+ }
+ }
+ else if (!strcmp(args[1], "type")) {
+ int cur_arg;
+ cur_arg = 2;
+ if (!*args[cur_arg]) {
+ Alert("parsing [%s:%d] : '%s' expects <type>\n",
+ file, linenum, args[0]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+ while (*(args[cur_arg])) {
+ comp_append_type(comp, args[cur_arg]);
+ cur_arg ++;
+ continue;
+ }
+ }
+ else {
+ Alert("parsing [%s:%d] : '%s' expects algo or type\n",
+ file, linenum, args[0]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+ }
else {
struct cfg_kw_list *kwl;
int index;
@@ -5263,7 +5323,7 @@
}
}
}
-
+
Alert("parsing [%s:%d] : unknown keyword '%s' in '%s' section\n", file, linenum, args[0], cursection);
err_code |= ERR_ALERT | ERR_FATAL;
goto out;
diff --git a/src/compression.c b/src/compression.c
new file mode 100644
index 0000000..87449ac
--- /dev/null
+++ b/src/compression.c
@@ -0,0 +1,415 @@
+/*
+ * HTTP compression.
+ *
+ * Copyright 2012 Exceliance, David Du Colombier <dducolombier@exceliance.fr>
+ * William Lallemand <wlallemand@exceliance.fr>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <stdio.h>
+#include <zlib.h>
+
+#include <common/compat.h>
+
+#include <types/global.h>
+#include <types/compression.h>
+
+#include <proto/compression.h>
+#include <proto/proto_http.h>
+
+static const struct comp_algo comp_algos[] =
+{
+ { "identity", 8, identity_init, identity_add_data, identity_flush, identity_reset, identity_end },
+#ifdef USE_ZLIB
+ { "deflate", 7, deflate_init, deflate_add_data, deflate_flush, deflate_reset, deflate_end },
+ { "gzip", 4, gzip_init, deflate_add_data, deflate_flush, deflate_reset, deflate_end },
+#endif /* USE_ZLIB */
+ { NULL, 0, NULL , NULL, NULL, NULL, NULL }
+};
+
+/*
+ * Add a content-type in the configuration
+ */
+int comp_append_type(struct comp *comp, const char *type)
+{
+ struct comp_type *comp_type;
+
+ comp_type = calloc(1, sizeof(struct comp_type));
+ comp_type->name_len = strlen(type);
+ comp_type->name = strdup(type);
+ comp_type->next = comp->types;
+ comp->types = comp_type;
+ return 0;
+}
+
+/*
+ * Add an algorithm in the configuration
+ */
+int comp_append_algo(struct comp *comp, const char *algo)
+{
+ struct comp_algo *comp_algo;
+ int i;
+
+ for (i = 0; comp_algos[i].name; i++) {
+ if (!strcmp(algo, comp_algos[i].name)) {
+ comp_algo = calloc(1, sizeof(struct comp_algo));
+ memmove(comp_algo, &comp_algos[i], sizeof(struct comp_algo));
+ comp_algo->next = comp->algos;
+ comp->algos = comp_algo;
+ return 0;
+ }
+ }
+ return -1;
+}
+
+/* emit the chunksize followed by a CRLF on the output and return the number of
+ * bytes written. Appends <add_crlf> additional CRLF after the first one. Chunk
+ * sizes are truncated to 6 hex digits (16 MB) and padded left. The caller is
+ * responsible for ensuring there is enough room left in the output buffer for
+ * the string (8 bytes * add_crlf*2).
+ */
+int http_emit_chunk_size(char *out, unsigned int chksz, int add_crlf)
+{
+ int shift;
+ int pos = 0;
+
+ for (shift = 20; shift >= 0; shift -= 4)
+ out[pos++] = hextab[(chksz >> shift) & 0xF];
+
+ do {
+ out[pos++] = '\r';
+ out[pos++] = '\n';
+ } while (--add_crlf >= 0);
+
+ return pos;
+}
+
+/*
+ * Init HTTP compression
+ */
+int http_compression_buffer_init(struct session *s, struct buffer *in, struct buffer *out)
+{
+ struct http_msg *msg = &s->txn.rsp;
+ int left;
+
+ /* not enough space */
+ if (in->size - buffer_len(in) < 40)
+ return -1;
+
+ /*
+ * Skip data, we don't need them in the new buffer. They are results
+ * of CHUNK_CRLF and CHUNK_SIZE parsing.
+ */
+ b_adv(in, msg->next);
+ msg->next = 0;
+ msg->sov = 0;
+ msg->sol = 0;
+
+ out->size = global.tune.bufsize;
+ out->i = 0;
+ out->o = 0;
+ out->p = out->data;
+ /* copy output data */
+ if (in->o > 0) {
+ left = in->o - bo_contig_data(in);
+ memcpy(out->data, bo_ptr(in), bo_contig_data(in));
+ out->p += bo_contig_data(in);
+ if (left > 0) { /* second part of the buffer */
+ memcpy(out->p, in->data, left);
+ out->p += left;
+ }
+ out->o = in->o;
+ }
+ out->i += http_emit_chunk_size(out->p, 0, 0);
+
+ return 0;
+}
+
+/*
+ * Add data to compress
+ */
+int http_compression_buffer_add_data(struct session *s, struct buffer *in, struct buffer *out)
+{
+ struct http_msg *msg = &s->txn.rsp;
+ int data_process_len;
+ int left;
+ int ret;
+
+ /*
+ * Skip data, we don't need them in the new buffer. They are results
+ * of CHUNK_CRLF and CHUNK_SIZE parsing.
+ */
+ b_adv(in, msg->next);
+ msg->next = 0;
+ msg->sov = 0;
+ msg->sol = 0;
+
+ /*
+ * select the smallest size between the announced chunk size, the input
+ * data, and the available output buffer size
+ */
+ data_process_len = MIN(in->i, msg->chunk_len);
+ data_process_len = MIN(out->size - buffer_len(out), data_process_len);
+
+ left = data_process_len - bi_contig_data(in);
+ if (left <= 0) {
+ ret = s->comp_algo->add_data(&s->comp_ctx.strm, bi_ptr(in),
+ data_process_len, bi_end(out),
+ out->size - buffer_len(out));
+ if (ret < 0)
+ return -1;
+ out->i += ret;
+
+ } else {
+ ret = s->comp_algo->add_data(&s->comp_ctx.strm, bi_ptr(in), bi_contig_data(in), bi_end(out), out->size - buffer_len(out));
+ if (ret < 0)
+ return -1;
+ out->i += ret;
+ ret = s->comp_algo->add_data(&s->comp_ctx.strm, in->data, left, bi_end(out), out->size - buffer_len(out));
+ if (ret < 0)
+ return -1;
+ out->i += ret;
+ }
+
+ b_adv(in, data_process_len);
+ msg->chunk_len -= data_process_len;
+
+ return 0;
+}
+
+/*
+ * Flush data in process, and write the header and footer of the chunk. Upon
+ * success, in and out buffers are swapped to avoid a copy.
+ */
+int http_compression_buffer_end(struct session *s, struct buffer **in, struct buffer **out, int end)
+{
+ int to_forward;
+ int left;
+ struct http_msg *msg = &s->txn.rsp;
+ struct buffer *ib = *in, *ob = *out;
+ int ret;
+
+ /* flush data here */
+
+ if (end)
+ ret = s->comp_algo->flush(&s->comp_ctx, ob, Z_FINISH); /* end of data */
+ else
+ ret = s->comp_algo->flush(&s->comp_ctx, ob, Z_SYNC_FLUSH); /* end of buffer */
+
+ if (ret < 0)
+ return -1; /* flush failed */
+
+ if (ob->i > 8) {
+ /* more than a chunk size => some data were emitted */
+ char *tail = ob->p + ob->i;
+
+ /* write real size at the begining of the chunk, no need of wrapping */
+ http_emit_chunk_size(ob->p, ob->i - 8, 0);
+
+ /* chunked encoding requires CRLF after data */
+ *tail++ = '\r';
+ *tail++ = '\n';
+
+ if (!(msg->flags & HTTP_MSGF_TE_CHNK) && msg->chunk_len == 0) {
+ /* End of data, 0<CRLF><CRLF> is needed but we're not
+ * in chunked mode on input so we must add it ourselves.
+ */
+ memcpy(tail, "0\r\n\r\n", 5);
+ tail += 5;
+ }
+ ob->i = tail - ob->p;
+ } else {
+ /* no data were sent, cancel the chunk size */
+ ob->i = 0;
+ }
+
+ to_forward = ob->i;
+
+ /* copy the remaining data in the tmp buffer. */
+ if (ib->i > 0) {
+ left = ib->i - bi_contig_data(ib);
+ memcpy(bi_end(ob), bi_ptr(ib), bi_contig_data(ib));
+ ob->i += bi_contig_data(ib);
+ if (left > 0) {
+ memcpy(bi_end(ob), ib->data, left);
+ ob->i += left;
+ }
+ }
+
+ /* swap the buffers */
+ *in = ob;
+ *out = ib;
+
+ /* forward the new chunk without remaining data */
+ b_adv(ob, to_forward);
+
+ /* if there are data between p and next, there are trailers, must forward them */
+ b_adv(ob, msg->next);
+ msg->next = 0;
+
+ return to_forward;
+}
+
+
+/****************************
+ **** Identity algorithm ****
+ ****************************/
+
+/*
+ * Init the identity algorithm
+ */
+int identity_init(void *v, int level)
+{
+ return 0;
+}
+
+/*
+ * Process data
+ * Return size of processed data or -1 on error
+ */
+int identity_add_data(void *comp_ctx, const char *in_data, int in_len, char *out_data, int out_len)
+{
+ if (out_len < in_len)
+ return -1;
+
+ memcpy(out_data, in_data, in_len);
+
+ return in_len;
+}
+
+int identity_flush(void *comp_ctx, struct buffer *out, int flag)
+{
+ return 0;
+}
+
+
+int identity_reset(void *comp_ctx)
+{
+ return 0;
+}
+
+/*
+ * Deinit the algorithm
+ */
+int identity_end(void *comp_ctx)
+{
+ return 0;
+}
+
+
+#ifdef USE_ZLIB
+
+/**************************
+**** gzip algorithm ****
+***************************/
+int gzip_init(void *v, int level)
+{
+ z_stream *strm;
+
+ strm = v;
+
+ strm->zalloc = Z_NULL;
+ strm->zfree = Z_NULL;
+ strm->opaque = Z_NULL;
+
+ if (deflateInit2(strm, level, Z_DEFLATED, MAX_WBITS + 16, 9, Z_DEFAULT_STRATEGY) != Z_OK)
+ return -1;
+
+ return 0;
+}
+/**************************
+**** Deflate algorithm ****
+***************************/
+
+int deflate_init(void *comp_ctx, int level)
+{
+ z_stream *strm;
+
+ strm = comp_ctx;
+
+ strm->zalloc = Z_NULL;
+ strm->zfree = Z_NULL;
+ strm->opaque = Z_NULL;
+
+ if (deflateInit(strm, level) != Z_OK)
+ return -1;
+
+ return 0;
+}
+
+int deflate_add_data(void *comp_ctx, const char *in_data, int in_len, char *out_data, int out_len)
+{
+ z_stream *strm;
+ int ret;
+
+ if (in_len <= 0)
+ return 0;
+
+
+ if (out_len <= 0)
+ return -1;
+
+ strm = comp_ctx;
+
+ strm->next_in = (unsigned char *)in_data;
+ strm->avail_in = in_len;
+ strm->next_out = (unsigned char *)out_data;
+ strm->avail_out = out_len;
+
+ ret = deflate(strm, Z_NO_FLUSH);
+ if (ret != Z_OK)
+ return -1;
+
+ /* deflate update the available data out */
+
+ return out_len - strm->avail_out;
+}
+
+int deflate_flush(void *comp_ctx, struct buffer *out, int flag)
+{
+ int ret;
+ z_stream *strm;
+ int out_len = 0;
+
+ strm = comp_ctx;
+ strm->next_out = (unsigned char *)bi_end(out);
+ strm->avail_out = out->size - buffer_len(out);
+
+ ret = deflate(strm, flag);
+ if (ret != Z_OK && ret != Z_STREAM_END)
+ return -1;
+
+ out_len = (out->size - buffer_len(out)) - strm->avail_out;
+ out->i += out_len;
+
+ return out_len;
+}
+
+int deflate_reset(void *comp_ctx)
+{
+ z_stream *strm;
+
+ strm = comp_ctx;
+ if (deflateReset(strm) == Z_OK)
+ return 0;
+ return -1;
+}
+
+int deflate_end(void *comp_ctx)
+{
+ z_stream *strm;
+
+ strm = comp_ctx;
+ if (deflateEnd(strm) == Z_OK)
+ return 0;
+
+ return -1;
+}
+
+#endif /* USE_ZLIB */
+
diff --git a/src/proto_http.c b/src/proto_http.c
index 4377573..720d66b 100644
--- a/src/proto_http.c
+++ b/src/proto_http.c
@@ -48,6 +48,7 @@
#include <proto/backend.h>
#include <proto/channel.h>
#include <proto/checks.h>
+#include <proto/compression.h>
#include <proto/dumpstats.h>
#include <proto/fd.h>
#include <proto/frontend.h>
@@ -1966,6 +1967,143 @@
return 1;
}
+
+/*
+ * Selects a compression algorithm depending on the client request.
+*/
+
+int select_compression_request_header(struct session *s, struct buffer *req)
+{
+ struct http_txn *txn = &s->txn;
+ struct hdr_ctx ctx;
+ struct comp_algo *comp_algo = NULL;
+
+ ctx.idx = 0;
+ /* no compression when Cache-Control: no-transform found */
+ while (http_find_header2("Cache-Control", 13, req->p, &txn->hdr_idx, &ctx)) {
+ if (word_match(ctx.line + ctx.val, ctx.vlen, "no-transform", 12)) {
+ s->comp_algo = NULL;
+ return 0;
+ }
+ }
+
+ /* search for the algo in the backend in priority or the frontend */
+ if ((s->be->comp && (comp_algo = s->be->comp->algos)) || (s->fe->comp && (comp_algo = s->fe->comp->algos))) {
+ ctx.idx = 0;
+ while (http_find_header2("Accept-Encoding", 15, req->p, &txn->hdr_idx, &ctx)) {
+ for (; comp_algo; comp_algo = comp_algo->next) {
+ if (word_match(ctx.line + ctx.val, ctx.vlen, comp_algo->name, comp_algo->name_len)) {
+ s->comp_algo = comp_algo;
+ return 1;
+ }
+ }
+ }
+ }
+
+ /* identity is implicit does not require headers */
+ if ((s->be->comp && (comp_algo = s->be->comp->algos)) || (s->fe->comp && (comp_algo = s->fe->comp->algos))) {
+ for (; comp_algo; comp_algo = comp_algo->next) {
+ if (comp_algo->add_data == identity_add_data) {
+ s->comp_algo = comp_algo;
+ return 1;
+ }
+ }
+ }
+
+ s->comp_algo = NULL;
+
+ return 0;
+}
+
+/*
+ * Selects a comression algorithm depending of the server response.
+ */
+int select_compression_response_header(struct session *s, struct buffer *res)
+{
+ struct http_txn *txn = &s->txn;
+ struct http_msg *msg = &txn->rsp;
+ struct hdr_ctx ctx;
+ struct comp_type *comp_type;
+ char *hdr_val;
+ int hdr_len;
+
+ /* no common compression algorithm was found in request header */
+ if (s->comp_algo == NULL)
+ goto fail;
+
+ /* HTTP < 1.1 should not be compressed */
+ if (!(msg->flags & HTTP_MSGF_VER_11))
+ goto fail;
+
+ hdr_val = trash;
+ ctx.idx = 0;
+
+ /* Content-Length is null */
+ if (!(msg->flags & HTTP_MSGF_TE_CHNK) && msg->body_len == 0)
+ goto fail;
+
+ /* content is already compressed */
+ if (http_find_header2("Content-Encoding", 16, res->p, &txn->hdr_idx, &ctx))
+ goto fail;
+
+ comp_type = NULL;
+
+ /* if there was a compression content-type option in the backend or the frontend
+ * The backend have priority.
+ */
+ if ((s->be->comp && (comp_type = s->be->comp->types)) || (s->fe->comp && (comp_type = s->fe->comp->types))) {
+ if (http_find_header2("Content-Type", 12, res->p, &txn->hdr_idx, &ctx)) {
+ for (; comp_type; comp_type = comp_type->next) {
+ if (strncmp(ctx.line+ctx.val, comp_type->name, comp_type->name_len) == 0)
+ /* this Content-Type should be compressed */
+ break;
+ }
+ }
+ /* this Content-Type should not be compressed */
+ if (comp_type == NULL)
+ goto fail;
+ }
+
+ ctx.idx = 0;
+
+ /* remove Content-Length header */
+ if ((msg->flags & HTTP_MSGF_CNT_LEN) && http_find_header2("Content-Length", 14, res->p, &txn->hdr_idx, &ctx))
+ http_remove_header2(msg, &txn->hdr_idx, &ctx);
+
+ /* add Transfer-Encoding header */
+ if (!(msg->flags & HTTP_MSGF_TE_CHNK))
+ http_header_add_tail2(&txn->rsp, &txn->hdr_idx, "Transfer-Encoding: chunked", 26);
+
+ /*
+ * Add Content-Encoding header when it's not identity encoding.
+ * RFC 2616 : Identity encoding: This content-coding is used only in the
+ * Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding
+ * header.
+ */
+ if (s->comp_algo->add_data != identity_add_data) {
+ hdr_len = 18;
+ memcpy(hdr_val, "Content-Encoding: ", hdr_len);
+ memcpy(hdr_val + hdr_len, s->comp_algo->name, s->comp_algo->name_len);
+ hdr_len += s->comp_algo->name_len;
+ hdr_val[hdr_len] = '\0';
+ http_header_add_tail2(&txn->rsp, &txn->hdr_idx, hdr_val, hdr_len);
+ }
+
+ /* initialize compression */
+ if (s->comp_algo->init(&s->comp_ctx.strm, 1) < 0)
+ goto fail;
+
+ return 1;
+
+fail:
+ if (s->comp_algo) {
+ s->comp_algo->end(&s->comp_ctx.strm);
+ s->comp_algo = NULL;
+ }
+ return 0;
+}
+
+
/* This stream analyser waits for a complete HTTP request. It returns 1 if the
* processing can continue on next analysers, or zero if it either needs more
* data or wants to immediately abort the request (eg: timeout, error, ...). It
@@ -3328,6 +3466,9 @@
req->buf->i,
req->analysers);
+ if (s->fe->comp || s->be->comp)
+ select_compression_request_header(s, req->buf);
+
/*
* Right now, we know that we have processed the entire headers
* and that unwanted requests have been filtered out. We can do
@@ -4956,6 +5097,9 @@
msg->body_len = msg->chunk_len = cl;
}
+ if (s->fe->comp || s->be->comp)
+ select_compression_response_header(s, rep->buf);
+
/* FIXME: we should also implement the multipart/byterange method.
* For now on, we resort to close mode in this case (unknown length).
*/
@@ -5350,6 +5494,8 @@
struct http_txn *txn = &s->txn;
struct http_msg *msg = &s->txn.rsp;
unsigned int bytes;
+ static struct buffer *tmpbuf = NULL;
+ int compressing = 0;
if (unlikely(msg->msg_state < HTTP_MSG_BODY))
return 0;
@@ -5368,13 +5514,24 @@
/* in most states, we should abort in case of early close */
channel_auto_close(res);
+ /* no data */
+ if (res->buf->i == 0)
+ return 0;
+
+ /* this is the first time we need the compression buffer */
+ if (s->comp_algo != NULL && tmpbuf == NULL) {
+ if ((tmpbuf = pool_alloc2(pool2_buffer)) == NULL)
+ goto aborted_xfer; /* no memory */
+ }
+
if (msg->msg_state < HTTP_MSG_CHUNK_SIZE) {
/* we have msg->sov which points to the first byte of message body.
- * rep->buf->p still points to the beginning of the message and msg->sol
- * is still null. We must save the body in msg->next because it
- * survives buffer re-alignments.
+ * rep->buf.p still points to the beginning of the message and msg->sol
+ * is still null. We forward the headers, we don't need them.
*/
- msg->next = msg->sov;
+ channel_forward(res, msg->sov);
+ msg->next = 0;
+ msg->sov = 0;
if (msg->flags & HTTP_MSGF_TE_CHNK)
msg->msg_state = HTTP_MSG_CHUNK_SIZE;
@@ -5382,20 +5539,32 @@
msg->msg_state = HTTP_MSG_DATA;
}
+ if (s->comp_algo != NULL) {
+ int ret = http_compression_buffer_init(s, res->buf, tmpbuf); /* init a buffer with headers */
+ if (ret < 0)
+ goto missing_data; /* not enough spaces in buffers */
+ compressing = 1;
+ }
+
while (1) {
http_silent_debug(__LINE__, s);
/* we may have some data pending between sol and sov */
- bytes = msg->sov - msg->sol;
- if (msg->chunk_len || bytes) {
- msg->sol = msg->sov;
- msg->next -= bytes; /* will be forwarded */
- msg->chunk_len += bytes;
- msg->chunk_len -= channel_forward(res, msg->chunk_len);
+ if (s->comp_algo == NULL) {
+ bytes = msg->sov - msg->sol;
+ if (msg->chunk_len || bytes) {
+ msg->sol = msg->sov;
+ msg->next -= bytes; /* will be forwarded */
+ msg->chunk_len += bytes;
+ msg->chunk_len -= channel_forward(res, msg->chunk_len);
+ }
}
if (msg->msg_state == HTTP_MSG_DATA) {
/* must still forward */
- if (res->to_forward)
+ if (compressing)
+ http_compression_buffer_add_data(s, res->buf, tmpbuf);
+
+ if (res->to_forward || msg->chunk_len)
goto missing_data;
/* nothing left to forward */
@@ -5418,6 +5587,13 @@
http_capture_bad_message(&s->be->invalid_rep, s, msg, HTTP_MSG_CHUNK_SIZE, s->fe);
goto return_bad_res;
}
+ /* skipping data if we are in compression mode */
+ if (compressing && msg->chunk_len > 0) {
+ b_adv(res->buf, msg->next);
+ msg->next = 0;
+ msg->sov = 0;
+ msg->sol = 0;
+ }
/* otherwise we're in HTTP_MSG_DATA or HTTP_MSG_TRAILERS state */
}
else if (msg->msg_state == HTTP_MSG_CHUNK_CRLF) {
@@ -5431,6 +5607,13 @@
http_capture_bad_message(&s->be->invalid_rep, s, msg, HTTP_MSG_CHUNK_CRLF, s->fe);
goto return_bad_res;
}
+ /* skipping data in buffer for compression */
+ if (compressing) {
+ b_adv(res->buf, msg->next);
+ msg->next = 0;
+ msg->sov = 0;
+ msg->sol = 0;
+ }
/* we're in MSG_CHUNK_SIZE now */
}
else if (msg->msg_state == HTTP_MSG_TRAILERS) {
@@ -5443,11 +5626,20 @@
http_capture_bad_message(&s->be->invalid_rep, s, msg, HTTP_MSG_TRAILERS, s->fe);
goto return_bad_res;
}
+ if (compressing) {
+ http_compression_buffer_end(s, &res->buf, &tmpbuf, 1);
+ compressing = 0;
+ }
/* we're in HTTP_MSG_DONE now */
}
else {
int old_state = msg->msg_state;
+ if (compressing) {
+ http_compression_buffer_end(s, &res->buf, &tmpbuf, 1);
+ compressing = 0;
+ }
+
/* other states, DONE...TUNNEL */
/* for keep-alive we don't want to forward closes on DONE */
if ((txn->flags & TX_CON_WANT_MSK) == TX_CON_WANT_KAL ||
@@ -5476,6 +5668,10 @@
}
missing_data:
+ if (compressing) {
+ http_compression_buffer_end(s, &res->buf, &tmpbuf, 0);
+ compressing = 0;
+ }
/* stop waiting for data if the input is closed before the end */
if (res->flags & CF_SHUTR) {
if (!(s->flags & SN_ERR_MASK))
@@ -5494,12 +5690,14 @@
goto return_bad_res;
/* forward any data pending between sol and sov */
- bytes = msg->sov - msg->sol;
- if (msg->chunk_len || bytes) {
- msg->sol = msg->sov;
- msg->next -= bytes; /* will be forwarded */
- msg->chunk_len += bytes;
- msg->chunk_len -= channel_forward(res, msg->chunk_len);
+ if (s->comp_algo == NULL) {
+ bytes = msg->sov - msg->sol;
+ if (msg->chunk_len || bytes) {
+ msg->sol = msg->sov;
+ msg->next -= bytes; /* will be forwarded */
+ msg->chunk_len += bytes;
+ msg->chunk_len -= channel_forward(res, msg->chunk_len);
+ }
}
/* When TE: chunked is used, we need to get there again to parse remaining
diff --git a/src/session.c b/src/session.c
index 77a214d..f148cb8 100644
--- a/src/session.c
+++ b/src/session.c
@@ -347,6 +347,7 @@
*/
s->be = s->fe;
s->req = s->rep = NULL; /* will be allocated later */
+ s->comp_algo = NULL;
/* Let's count a session now */
proxy_inc_fe_sess_ctr(l, p);
@@ -551,6 +552,11 @@
sess_change_server(s, NULL);
}
+ if (s->comp_algo) {
+ s->comp_algo->end(&s->comp_ctx.strm);
+ s->comp_algo = NULL;
+ }
+
if (s->req->pipe)
put_pipe(s->req->pipe);