diff --git a/src/cache.c b/src/cache.c
index d3bf8dd..58b1ced 100644
--- a/src/cache.c
+++ b/src/cache.c
@@ -960,7 +960,7 @@
  * "vary" on the accept-encoding value.
  * Returns 0 if we found a known encoding in the response, -1 otherwise.
  */
-static int set_secondary_key_encoding(struct htx *htx, char *secondary_key)
+static int set_secondary_key_encoding(struct htx *htx, unsigned int vary_signature, char *secondary_key)
 {
 	unsigned int resp_encoding_bitmap = 0;
 	const struct vary_hashing_information *info = vary_information;
@@ -970,6 +970,11 @@
 	unsigned int encoding_value;
 	struct http_hdr_ctx ctx = { .blk = NULL };
 
+	/* We must not set the accept encoding part of the secondary signature
+	 * if the response does not vary on 'Accept Encoding'. */
+	if (!(vary_signature & VARY_ACCEPT_ENCODING))
+		return 0;
+
 	/* Look for the accept-encoding part of the secondary_key. */
 	while (count < hash_info_count && info->value != VARY_ACCEPT_ENCODING) {
 		offset += info->hash_length;
@@ -1228,7 +1233,7 @@
 	 * We will not cache a response that has an unknown encoding (not
 	 * explicitly supported in parse_encoding_value function). */
 	if (cache->vary_processing_enabled && vary_signature)
-		if (set_secondary_key_encoding(htx, object->secondary_key))
+		if (set_secondary_key_encoding(htx, vary_signature, object->secondary_key))
 		    goto out;
 
 	shctx_lock(shctx);