MEDIUM: 51d: add support for 51Degrees V4 with Hash algorithm
This patch also adds a set of new global options:
- 51degrees-use-performance-graph { on | off }
- 51degrees-use-predictive-graph { on | off }
- 51degrees-drift <number>
- 51degrees-difference <number>
- 51degrees-allow-unmatched { on | off }
To build using the latest 51Degrees V4 engine with Hash algorithm, set
USE_51DEGREES_V4=1.
Other supported build options are 51DEGREES_INC, 51DEGREES_LIB and
51DEGREES_SRC which needs to be set to the directory that contains
headers and C files. For example:
make TARGET=<target> USE_51DEGREES_V4=1 51DEGREES_SRC='51D_REPO_PATH'/src
diff --git a/addons/51degrees/51d.c b/addons/51degrees/51d.c
index 5d68695..a23b468 100644
--- a/addons/51degrees/51d.c
+++ b/addons/51degrees/51d.c
@@ -11,11 +11,19 @@
#include <haproxy/http_ana.h>
#include <haproxy/http_fetch.h>
#include <haproxy/http_htx.h>
+#include <haproxy/htx.h>
#include <haproxy/sample.h>
#include <haproxy/thread.h>
#include <haproxy/tools.h>
#include <haproxy/xxhash.h>
+
+#ifdef USE_51DEGREES_V4
+#include <hash/hash.h>
+#undef MAP_TYPE
+#include <hash/fiftyone.h>
+#else
#include <51Degrees.h>
+#endif
struct _51d_property_names {
struct list list;
@@ -29,10 +37,23 @@
__decl_spinlock(_51d_lru_lock);
#endif
+#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
+#define _51D_HEADERS_BUFFER_SIZE BUFSIZE
+
+static THREAD_LOCAL struct {
+ char **buf;
+ int max;
+ int count;
+} _51d_headers;
+
+static THREAD_LOCAL fiftyoneDegreesResultsHash *_51d_results = NULL;
+#endif
+
static struct {
char property_separator; /* the separator to use in the response for the values. this is taken from 51degrees-property-separator from config. */
struct list property_names; /* list of properties to load into the data set. this is taken from 51degrees-property-name-list from config. */
char *data_file_path;
+#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
int header_count; /* number of HTTP headers related to device detection. */
struct buffer *header_names; /* array of HTTP header names. */
fiftyoneDegreesDataSet data_set; /* data set used with the pattern and trie detection methods. */
@@ -45,6 +66,14 @@
fiftyoneDegreesDeviceOffsets device_offsets; /* Memory used for device offsets. */
#endif
#endif
+#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
+ fiftyoneDegreesResourceManager manager;
+ int use_perf_graph;
+ int use_pred_graph;
+ int drift;
+ int difference;
+ int allow_unmatched;
+#endif
int cache_size;
} global_51degrees = {
.property_separator = ',',
@@ -53,6 +82,14 @@
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
.data_set = { },
#endif
+#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
+ .manager = { },
+ .use_perf_graph = -1,
+ .use_pred_graph = -1,
+ .drift = -1,
+ .difference = -1,
+ .allow_unmatched = -1,
+#endif
.cache_size = 0,
};
@@ -313,6 +350,185 @@
}
#endif
+#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
+static int _51d_use_perf_graph(char **args, int section_type, struct proxy *curpx,
+ const struct proxy *defpx, const char *file, int line,
+ char **err)
+{
+ if (too_many_args(1, args, err, NULL))
+ return -1;
+
+ if (strcmp(args[1], "on") == 0)
+ global_51degrees.use_perf_graph = 1;
+ else if (strcmp(args[1], "off") == 0)
+ global_51degrees.use_perf_graph = 0;
+ else {
+ memprintf(err, "'%s' expects either 'on' or 'off' but got '%s'.", args[0], args[1]);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int _51d_use_pred_graph(char **args, int section_type, struct proxy *curpx,
+ const struct proxy *defpx, const char *file, int line,
+ char **err)
+{
+ if (too_many_args(1, args, err, NULL))
+ return -1;
+
+ if (strcmp(args[1], "on") == 0)
+ global_51degrees.use_pred_graph = 1;
+ else if (strcmp(args[1], "off") == 0)
+ global_51degrees.use_pred_graph = 0;
+ else {
+ memprintf(err, "'%s' expects either 'on' or 'off' but got '%s'.", args[0], args[1]);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int _51d_drift(char **args, int section_type, struct proxy *curpx,
+ const struct proxy *defpx, const char *file, int line,
+ char **err)
+{
+ if (*(args[1]) == 0) {
+ memprintf(err, "'%s' expects a positive numeric value.", args[0]);
+ return -1;
+ }
+
+ global_51degrees.drift = atoi(args[1]);
+ if (global_51degrees.drift < 0) {
+ memprintf(err, "'%s' expects a positive numeric value, got '%s'.",
+ args[0], args[1]);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int _51d_difference(char **args, int section_type, struct proxy *curpx,
+ const struct proxy *defpx, const char *file, int line,
+ char **err)
+{
+ if (*(args[1]) == 0) {
+ memprintf(err, "'%s' expects a positive numeric value.", args[0]);
+ return -1;
+ }
+
+ global_51degrees.difference = atoi(args[1]);
+ if (global_51degrees.difference < 0) {
+ memprintf(err, "'%s' expects a positive numeric value, got '%s'.",
+ args[0], args[1]);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int _51d_allow_unmatched(char **args, int section_type, struct proxy *curpx,
+ const struct proxy *defpx, const char *file, int line,
+ char **err)
+{
+ if (too_many_args(1, args, err, NULL))
+ return -1;
+
+ if (strcmp(args[1], "on") == 0)
+ global_51degrees.allow_unmatched = 1;
+ else if (strcmp(args[1], "off") == 0)
+ global_51degrees.allow_unmatched = 0;
+ else {
+ memprintf(err, "'%s' expects either 'on' or 'off' but got '%s'.", args[0], args[1]);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int _51d_init_internal()
+{
+ fiftyoneDegreesDataSetHash *ds;
+ int hdr_count;
+ int i, ret = 0;
+
+ ds = (fiftyoneDegreesDataSetHash *)fiftyoneDegreesDataSetGet(&global_51degrees.manager);
+
+ hdr_count = ds->b.b.uniqueHeaders->count;
+ if (hdr_count > _51d_headers.max)
+ hdr_count = _51d_headers.max;
+
+ _51d_results = fiftyoneDegreesResultsHashCreate(&global_51degrees.manager, hdr_count, 0);
+ if (!_51d_results)
+ goto out;
+
+ for (i = 0; i < hdr_count; i++) {
+ _51d_headers.buf[i] = malloc(_51D_HEADERS_BUFFER_SIZE);
+ if (!_51d_headers.buf[i])
+ goto out;
+ _51d_headers.count++;
+ }
+
+ /* success */
+ ret = 1;
+
+out:
+ fiftyoneDegreesDataSetRelease((fiftyoneDegreesDataSetBase *)ds);
+ return ret;
+}
+
+static fiftyoneDegreesEvidenceKeyValuePairArray * _51d_get_evidence(struct sample *smp)
+{
+ fiftyoneDegreesEvidenceKeyValuePairArray *evidence;
+ fiftyoneDegreesDataSetHash *ds;
+ size_t size;
+ struct channel *chn;
+ struct htx *htx;
+ struct http_hdr_ctx ctx;
+ struct ist name;
+ int i;
+
+ chn = (smp->strm ? &smp->strm->req : NULL);
+
+ // No need to null check as this has already been carried out in the
+ // calling method
+ htx = smp_prefetch_htx(smp, chn, NULL, 1);
+ ALREADY_CHECKED(htx);
+
+ ds = (fiftyoneDegreesDataSetHash *)_51d_results->b.b.dataSet;
+ size = _51d_headers.count * 2;
+
+ evidence = fiftyoneDegreesEvidenceCreate(size);
+ if (!evidence)
+ return NULL;
+
+ for (i = 0; i < _51d_headers.count; i++) {
+ fiftyoneDegreesHeader *hdr = &ds->b.b.uniqueHeaders->items[i];
+ name = ist2(hdr->name, hdr->nameLength);
+ ctx.blk = NULL;
+
+ if (http_find_header(htx, name, &ctx, 1)) {
+ size_t len = ctx.value.len;
+
+ if (unlikely(len >= _51D_HEADERS_BUFFER_SIZE))
+ len = _51D_HEADERS_BUFFER_SIZE - 1;
+
+ memcpy(_51d_headers.buf[i], ctx.value.ptr, len);
+ _51d_headers.buf[i][len] = '\0';
+
+ fiftyoneDegreesEvidenceAddString(
+ evidence,
+ FIFTYONE_DEGREES_EVIDENCE_HTTP_HEADER_STRING,
+ name.ptr,
+ _51d_headers.buf[i]);
+ }
+ }
+
+ return evidence;
+}
+#endif
+
+#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
static void _51d_process_match(const struct arg *args, struct sample *smp, fiftyoneDegreesWorkset* ws)
{
@@ -325,11 +541,22 @@
const char **requiredProperties = fiftyoneDegreesGetRequiredPropertiesNames(&global_51degrees.data_set);
int requiredPropertiesCount = fiftyoneDegreesGetRequiredPropertiesCount(&global_51degrees.data_set);
#endif
+ const char* property_name;
+ int j;
+
+#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
+static void _51d_process_match(const struct arg *args, struct sample *smp)
+{
+ char valuesBuffer[1024];
+#endif
char no_data[] = "NoData"; /* response when no data could be found */
struct buffer *temp = get_trash_chunk();
- int j, i = 0, found;
- const char* property_name;
+ int i = 0, found;
+
+#if defined(FIFTYONE_DEGREES_HASH_INCLUDED)
+ FIFTYONE_DEGREES_EXCEPTION_CREATE;
+#endif
/* Loop through property names passed to the filter and fetch them from the dataset. */
while (args[i].data.str.area) {
@@ -380,6 +607,19 @@
}
}
#endif
+#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
+ FIFTYONE_DEGREES_EXCEPTION_CLEAR;
+
+ found = fiftyoneDegreesResultsHashGetValuesString(
+ _51d_results, args[i].data.str.area,
+ valuesBuffer, 1024, "|",
+ exception);
+
+ if (FIFTYONE_DEGREES_EXCEPTION_FAILED || found <= 0)
+ found = 0;
+ else
+ chunk_appendf(temp, "%s", valuesBuffer);
+#endif
if (!found)
chunk_appendf(temp, "%s", no_data);
@@ -414,16 +654,19 @@
static int _51d_fetch(const struct arg *args, struct sample *smp, const char *kw, void *private)
{
+ struct channel *chn;
+ struct htx *htx;
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
fiftyoneDegreesWorkset* ws; /* workset for detection */
struct lru64 *lru = NULL;
#endif
#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED
fiftyoneDegreesDeviceOffsets *offsets; /* Offsets for detection */
-
#endif
- struct channel *chn;
- struct htx *htx;
+#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
+ fiftyoneDegreesEvidenceKeyValuePairArray *evidence = NULL;
+ FIFTYONE_DEGREES_EXCEPTION_CREATE;
+#endif
chn = (smp->strm ? &smp->strm->req : NULL);
htx = smp_prefetch_htx(smp, chn, NULL, 1);
@@ -496,6 +739,21 @@
_51d_insert_cache_entry(smp, lru, (void*)args);
#endif
+#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
+ evidence = _51d_get_evidence(smp);
+ if (!evidence)
+ return 0;
+
+ fiftyoneDegreesResultsHashFromEvidence(
+ _51d_results, evidence, exception);
+ fiftyoneDegreesEvidenceFree(evidence);
+
+ if (FIFTYONE_DEGREES_EXCEPTION_FAILED)
+ return 0;
+
+ _51d_process_match(args, smp);
+#endif
+
_51d_set_smp(smp);
return 1;
}
@@ -509,6 +767,9 @@
#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED
fiftyoneDegreesDeviceOffsets *offsets; /* Offsets for detection */
#endif
+#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
+ FIFTYONE_DEGREES_EXCEPTION_CREATE;
+#endif
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
@@ -540,6 +801,7 @@
smp->data.u.str.area[smp->data.u.str.data] = '\0';
/* Perform detection. */
+#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
fiftyoneDegreesMatch(ws, smp->data.u.str.area);
_51d_process_match(args, smp, ws);
@@ -570,6 +832,15 @@
#endif
#endif
+#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
+ fiftyoneDegreesResultsHashFromUserAgent(_51d_results, smp->data.u.str.area,
+ smp->data.u.str.data, exception);
+ if (FIFTYONE_DEGREES_EXCEPTION_FAILED)
+ return 0;
+
+ _51d_process_match(args, smp);
+#endif
+
_51d_set_smp(smp);
return 1;
}
@@ -619,10 +890,18 @@
static int init_51degrees(void)
{
int i = 0;
- struct buffer *temp;
struct _51d_property_names *name;
char **_51d_property_list = NULL;
+#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
+ struct buffer *temp;
fiftyoneDegreesDataSetInitStatus _51d_dataset_status = DATA_SET_INIT_STATUS_NOT_SET;
+#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
+ fiftyoneDegreesConfigHash config = fiftyoneDegreesHashInMemoryConfig;
+ fiftyoneDegreesPropertiesRequired properties = fiftyoneDegreesPropertiesDefault;
+ fiftyoneDegreesMemoryReader reader;
+ fiftyoneDegreesStatusCode status;
+ FIFTYONE_DEGREES_EXCEPTION_CREATE;
+#endif
if (!global_51degrees.data_file_path)
return ERR_NONE;
@@ -643,6 +922,7 @@
_51d_property_list[i++] = name->name;
}
+#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
_51d_dataset_status = fiftyoneDegreesInitWithPropertyArray(global_51degrees.data_file_path, &global_51degrees.data_set, (const char**)_51d_property_list, i);
temp = get_trash_chunk();
@@ -708,6 +988,63 @@
}
#endif
+#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
+ config.b.b.freeData = true;
+
+ if (global_51degrees.use_perf_graph != -1)
+ config.usePerformanceGraph = global_51degrees.use_perf_graph;
+ if (global_51degrees.use_pred_graph != -1)
+ config.usePredictiveGraph = global_51degrees.use_pred_graph;
+
+ if (global_51degrees.drift > 0)
+ config.drift = global_51degrees.drift;
+ if (global_51degrees.difference > 0)
+ config.difference = global_51degrees.difference;
+
+ if (global_51degrees.allow_unmatched != -1)
+ config.b.allowUnmatched = global_51degrees.allow_unmatched;
+
+ config.strings.concurrency =
+ config.properties.concurrency =
+ config.values.concurrency =
+ config.profiles.concurrency =
+ config.nodes.concurrency =
+ config.profileOffsets.concurrency =
+ config.maps.concurrency =
+ config.components.concurrency =
+ config.rootNodes.concurrency = global.nbthread;
+
+ properties.array = (const char **)_51d_property_list;
+ properties.count = i;
+
+ status = fiftyoneDegreesFileReadToByteArray(global_51degrees.data_file_path, &reader);
+ if (status == FIFTYONE_DEGREES_STATUS_SUCCESS && !FIFTYONE_DEGREES_EXCEPTION_FAILED) {
+ FIFTYONE_DEGREES_EXCEPTION_CLEAR;
+
+ status = fiftyoneDegreesHashInitManagerFromMemory(
+ &global_51degrees.manager,
+ &config,
+ &properties,
+ reader.startByte,
+ reader.length,
+ exception);
+ }
+
+ free(_51d_property_list);
+ _51d_property_list = NULL;
+ i = 0;
+
+ if (status != FIFTYONE_DEGREES_STATUS_SUCCESS || FIFTYONE_DEGREES_EXCEPTION_FAILED) {
+ const char *message = fiftyoneDegreesStatusGetMessage(status, global_51degrees.data_file_path);
+ if (message)
+ ha_alert("51Degrees Setup - Error reading 51Degrees data file. %s\n",
+ message);
+ else
+ ha_alert("51Degrees Setup - Error reading 51Degrees data file.\n");
+ return ERR_ALERT | ERR_FATAL;
+ }
+#endif
+
return ERR_NONE;
}
@@ -715,6 +1052,7 @@
{
struct _51d_property_names *_51d_prop_name, *_51d_prop_nameb;
+#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
free(global_51degrees.header_names);
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
if (global_51degrees.pool)
@@ -727,6 +1065,7 @@
free(global_51degrees.header_offsets);
#endif
fiftyoneDegreesDataSetFree(&global_51degrees.data_set);
+#endif
ha_free(&global_51degrees.data_file_path);
list_for_each_entry_safe(_51d_prop_name, _51d_prop_nameb, &global_51degrees.property_names, list) {
@@ -738,12 +1077,61 @@
while (lru64_destroy(_51d_lru_tree));
#endif
}
+
+#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
+static int init_51degrees_per_thread()
+{
+ if (!global_51degrees.data_file_path) {
+ /* noop */
+ return 1;
+ }
+
+ _51d_headers.max = global.tune.max_http_hdr;
+ _51d_headers.buf = calloc(_51d_headers.max, sizeof(*_51d_headers.buf));
+ _51d_headers.count = 0;
+
+ if (!_51d_headers.buf)
+ return 0;
+
+ if (!_51d_init_internal())
+ return 0;
+
+ return 1;
+}
+
+static void deinit_51degrees_per_thread()
+{
+ int i;
+
+ if (_51d_results) {
+ fiftyoneDegreesResultsHashFree(_51d_results);
+ _51d_results = NULL;
+ }
+
+ if (_51d_headers.buf) {
+ for (i = 0; i < _51d_headers.max; i++)
+ free(_51d_headers.buf[i]);
+ free(_51d_headers.buf);
+ _51d_headers.buf = NULL;
+ }
+
+ _51d_headers.max = 0;
+ _51d_headers.count = 0;
+}
+#endif
static struct cfg_kw_list _51dcfg_kws = {{ }, {
{ CFG_GLOBAL, "51degrees-data-file", _51d_data_file },
{ CFG_GLOBAL, "51degrees-property-name-list", _51d_property_name_list },
{ CFG_GLOBAL, "51degrees-property-separator", _51d_property_separator },
{ CFG_GLOBAL, "51degrees-cache-size", _51d_cache_size },
+#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
+ { CFG_GLOBAL, "51degrees-use-performance-graph", _51d_use_perf_graph },
+ { CFG_GLOBAL, "51degrees-use-predictive-graph", _51d_use_pred_graph },
+ { CFG_GLOBAL, "51degrees-drift", _51d_drift },
+ { CFG_GLOBAL, "51degrees-difference", _51d_difference },
+ { CFG_GLOBAL, "51degrees-allow-unmatched", _51d_allow_unmatched },
+#endif
{ 0, NULL, NULL },
}};
@@ -780,4 +1168,12 @@
#else
REGISTER_BUILD_OPTS("Built with 51Degrees Trie support (dummy library).");
#endif
+#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
+ REGISTER_PER_THREAD_INIT(init_51degrees_per_thread);
+ REGISTER_PER_THREAD_DEINIT(deinit_51degrees_per_thread);
+#ifndef FIFTYONEDEGREES_DUMMY_LIB
+ REGISTER_BUILD_OPTS("Built with 51Degrees V4 Hash support.");
+#else
+ REGISTER_BUILD_OPTS("Built with 51Degrees V4 Hash support (dummy library).");
+#endif
#endif