blob: 121c59fa12ed1e61bd83f5f482dc9d572e7d1c7b [file] [log] [blame]
Willy Tarreauc2186022009-10-26 19:48:54 +01001/*
2 * Elastic Binary Trees - macros and structures for Multi-Byte data nodes.
Willy Tarreauf3bfede2011-07-25 11:38:17 +02003 * Version 6.0.6
Willy Tarreau414c4b22011-01-04 13:21:06 +01004 * (C) 2002-2011 - Willy Tarreau <w@1wt.eu>
Willy Tarreauc2186022009-10-26 19:48:54 +01005 *
Willy Tarreauf3bfede2011-07-25 11:38:17 +02006 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation, version 2.1
9 * exclusively.
Willy Tarreauc2186022009-10-26 19:48:54 +010010 *
Willy Tarreauf3bfede2011-07-25 11:38:17 +020011 * This library is distributed in the hope that it will be useful,
Willy Tarreauc2186022009-10-26 19:48:54 +010012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Willy Tarreauf3bfede2011-07-25 11:38:17 +020013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
Willy Tarreauc2186022009-10-26 19:48:54 +010015 *
Willy Tarreauf3bfede2011-07-25 11:38:17 +020016 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Willy Tarreauc2186022009-10-26 19:48:54 +010019 */
20
Willy Tarreauead63a02009-11-02 14:41:23 +010021#ifndef _EBMBTREE_H
22#define _EBMBTREE_H
23
Willy Tarreauc2186022009-10-26 19:48:54 +010024#include <string.h>
25#include "ebtree.h"
26
27/* Return the structure of type <type> whose member <member> points to <ptr> */
28#define ebmb_entry(ptr, type, member) container_of(ptr, type, member)
29
30#define EBMB_ROOT EB_ROOT
31#define EBMB_TREE_HEAD EB_TREE_HEAD
32
33/* This structure carries a node, a leaf, and a key. It must start with the
34 * eb_node so that it can be cast into an eb_node. We could also have put some
35 * sort of transparent union here to reduce the indirection level, but the fact
36 * is, the end user is not meant to manipulate internals, so this is pointless.
37 * The 'node.bit' value here works differently from scalar types, as it contains
38 * the number of identical bits between the two branches.
39 */
40struct ebmb_node {
41 struct eb_node node; /* the tree node, must be at the beginning */
42 unsigned char key[0]; /* the key, its size depends on the application */
43};
44
45/*
46 * Exported functions and macros.
47 * Many of them are always inlined because they are extremely small, and
48 * are generally called at most once or twice in a program.
49 */
50
51/* Return leftmost node in the tree, or NULL if none */
52static forceinline struct ebmb_node *ebmb_first(struct eb_root *root)
53{
54 return ebmb_entry(eb_first(root), struct ebmb_node, node);
55}
56
57/* Return rightmost node in the tree, or NULL if none */
58static forceinline struct ebmb_node *ebmb_last(struct eb_root *root)
59{
60 return ebmb_entry(eb_last(root), struct ebmb_node, node);
61}
62
63/* Return next node in the tree, or NULL if none */
64static forceinline struct ebmb_node *ebmb_next(struct ebmb_node *ebmb)
65{
66 return ebmb_entry(eb_next(&ebmb->node), struct ebmb_node, node);
67}
68
69/* Return previous node in the tree, or NULL if none */
70static forceinline struct ebmb_node *ebmb_prev(struct ebmb_node *ebmb)
71{
72 return ebmb_entry(eb_prev(&ebmb->node), struct ebmb_node, node);
73}
74
75/* Return next node in the tree, skipping duplicates, or NULL if none */
76static forceinline struct ebmb_node *ebmb_next_unique(struct ebmb_node *ebmb)
77{
78 return ebmb_entry(eb_next_unique(&ebmb->node), struct ebmb_node, node);
79}
80
81/* Return previous node in the tree, skipping duplicates, or NULL if none */
82static forceinline struct ebmb_node *ebmb_prev_unique(struct ebmb_node *ebmb)
83{
84 return ebmb_entry(eb_prev_unique(&ebmb->node), struct ebmb_node, node);
85}
86
87/* Delete node from the tree if it was linked in. Mark the node unused. Note
88 * that this function relies on a non-inlined generic function: eb_delete.
89 */
90static forceinline void ebmb_delete(struct ebmb_node *ebmb)
91{
92 eb_delete(&ebmb->node);
93}
94
95/* The following functions are not inlined by default. They are declared
96 * in ebmbtree.c, which simply relies on their inline version.
97 */
98REGPRM3 struct ebmb_node *ebmb_lookup(struct eb_root *root, const void *x, unsigned int len);
99REGPRM3 struct ebmb_node *ebmb_insert(struct eb_root *root, struct ebmb_node *new, unsigned int len);
Willy Tarreau3a932442010-05-09 19:29:23 +0200100REGPRM2 struct ebmb_node *ebmb_lookup_longest(struct eb_root *root, const void *x);
101REGPRM3 struct ebmb_node *ebmb_lookup_prefix(struct eb_root *root, const void *x, unsigned int pfx);
102REGPRM3 struct ebmb_node *ebmb_insert_prefix(struct eb_root *root, struct ebmb_node *new, unsigned int len);
Willy Tarreauc2186022009-10-26 19:48:54 +0100103
104/* The following functions are less likely to be used directly, because their
105 * code is larger. The non-inlined version is preferred.
106 */
107
108/* Delete node from the tree if it was linked in. Mark the node unused. */
109static forceinline void __ebmb_delete(struct ebmb_node *ebmb)
110{
111 __eb_delete(&ebmb->node);
112}
113
Willy Tarreau414c4b22011-01-04 13:21:06 +0100114/* Find the first occurence of a key of a least <len> bytes matching <x> in the
115 * tree <root>. The caller is responsible for ensuring that <len> will not exceed
116 * the common parts between the tree's keys and <x>. In case of multiple matches,
117 * the leftmost node is returned. This means that this function can be used to
118 * lookup string keys by prefix if all keys in the tree are zero-terminated. If
119 * no match is found, NULL is returned. Returns first node if <len> is zero.
Willy Tarreauc2186022009-10-26 19:48:54 +0100120 */
121static forceinline struct ebmb_node *__ebmb_lookup(struct eb_root *root, const void *x, unsigned int len)
122{
123 struct ebmb_node *node;
124 eb_troot_t *troot;
Willy Tarreau3a932442010-05-09 19:29:23 +0200125 int pos, side;
126 int node_bit;
Willy Tarreauc2186022009-10-26 19:48:54 +0100127
128 troot = root->b[EB_LEFT];
129 if (unlikely(troot == NULL))
Willy Tarreauce3d44a2011-01-04 14:07:36 +0100130 goto ret_null;
Willy Tarreauc2186022009-10-26 19:48:54 +0100131
Willy Tarreau414c4b22011-01-04 13:21:06 +0100132 if (unlikely(len == 0))
133 goto walk_down;
134
Willy Tarreau3a932442010-05-09 19:29:23 +0200135 pos = 0;
Willy Tarreauc2186022009-10-26 19:48:54 +0100136 while (1) {
Willy Tarreau3a932442010-05-09 19:29:23 +0200137 if (eb_gettag(troot) == EB_LEAF) {
Willy Tarreauc2186022009-10-26 19:48:54 +0100138 node = container_of(eb_untag(troot, EB_LEAF),
139 struct ebmb_node, node.branches);
Willy Tarreau414c4b22011-01-04 13:21:06 +0100140 if (memcmp(node->key + pos, x, len) != 0)
Willy Tarreauce3d44a2011-01-04 14:07:36 +0100141 goto ret_null;
Willy Tarreau3a932442010-05-09 19:29:23 +0200142 else
Willy Tarreauce3d44a2011-01-04 14:07:36 +0100143 goto ret_node;
Willy Tarreauc2186022009-10-26 19:48:54 +0100144 }
145 node = container_of(eb_untag(troot, EB_NODE),
146 struct ebmb_node, node.branches);
147
Willy Tarreau3a932442010-05-09 19:29:23 +0200148 node_bit = node->node.bit;
149 if (node_bit < 0) {
Willy Tarreauc2186022009-10-26 19:48:54 +0100150 /* We have a dup tree now. Either it's for the same
151 * value, and we walk down left, or it's a different
152 * one and we don't have our key.
153 */
Willy Tarreau414c4b22011-01-04 13:21:06 +0100154 if (memcmp(node->key + pos, x, len) != 0)
Willy Tarreauce3d44a2011-01-04 14:07:36 +0100155 goto ret_null;
156 else
157 goto walk_left;
Willy Tarreauc2186022009-10-26 19:48:54 +0100158 }
159
Willy Tarreau3a932442010-05-09 19:29:23 +0200160 /* OK, normal data node, let's walk down. We check if all full
161 * bytes are equal, and we start from the last one we did not
162 * completely check. We stop as soon as we reach the last byte,
163 * because we must decide to go left/right or abort.
164 */
165 node_bit = ~node_bit + (pos << 3) + 8; // = (pos<<3) + (7 - node_bit)
166 if (node_bit < 0) {
167 /* This surprizing construction gives better performance
168 * because gcc does not try to reorder the loop. Tested to
169 * be fine with 2.95 to 4.2.
170 */
171 while (1) {
Willy Tarreau414c4b22011-01-04 13:21:06 +0100172 if (node->key[pos++] ^ *(unsigned char*)(x++))
Willy Tarreauce3d44a2011-01-04 14:07:36 +0100173 goto ret_null; /* more than one full byte is different */
Willy Tarreau414c4b22011-01-04 13:21:06 +0100174 if (--len == 0)
175 goto walk_left; /* return first node if all bytes matched */
Willy Tarreau3a932442010-05-09 19:29:23 +0200176 node_bit += 8;
177 if (node_bit >= 0)
178 break;
179 }
180 }
Willy Tarreauc2186022009-10-26 19:48:54 +0100181
Willy Tarreau3a932442010-05-09 19:29:23 +0200182 /* here we know that only the last byte differs, so node_bit < 8.
183 * We have 2 possibilities :
184 * - more than the last bit differs => return NULL
185 * - walk down on side = (x[pos] >> node_bit) & 1
186 */
187 side = *(unsigned char *)x >> node_bit;
188 if (((node->key[pos] >> node_bit) ^ side) > 1)
Willy Tarreauce3d44a2011-01-04 14:07:36 +0100189 goto ret_null;
Willy Tarreau3a932442010-05-09 19:29:23 +0200190 side &= 1;
191 troot = node->node.branches.b[side];
Willy Tarreauc2186022009-10-26 19:48:54 +0100192 }
Willy Tarreauce3d44a2011-01-04 14:07:36 +0100193 walk_left:
194 troot = node->node.branches.b[EB_LEFT];
195 walk_down:
196 while (eb_gettag(troot) != EB_LEAF)
197 troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT];
198 node = container_of(eb_untag(troot, EB_LEAF),
199 struct ebmb_node, node.branches);
200 ret_node:
201 return node;
202 ret_null:
203 return NULL;
Willy Tarreauc2186022009-10-26 19:48:54 +0100204}
205
206/* Insert ebmb_node <new> into subtree starting at node root <root>.
207 * Only new->key needs be set with the key. The ebmb_node is returned.
208 * If root->b[EB_RGHT]==1, the tree may only contain unique keys. The
Willy Tarreau414c4b22011-01-04 13:21:06 +0100209 * len is specified in bytes. It is absolutely mandatory that this length
210 * is the same for all keys in the tree. This function cannot be used to
211 * insert strings.
Willy Tarreauc2186022009-10-26 19:48:54 +0100212 */
213static forceinline struct ebmb_node *
214__ebmb_insert(struct eb_root *root, struct ebmb_node *new, unsigned int len)
215{
216 struct ebmb_node *old;
217 unsigned int side;
Willy Tarreau3a932442010-05-09 19:29:23 +0200218 eb_troot_t *troot, **up_ptr;
Willy Tarreau6258f7b2011-09-19 20:48:00 +0200219 eb_troot_t *root_right;
Willy Tarreauc2186022009-10-26 19:48:54 +0100220 int diff;
221 int bit;
Willy Tarreau3a932442010-05-09 19:29:23 +0200222 eb_troot_t *new_left, *new_rght;
223 eb_troot_t *new_leaf;
224 int old_node_bit;
Willy Tarreauc2186022009-10-26 19:48:54 +0100225
226 side = EB_LEFT;
227 troot = root->b[EB_LEFT];
228 root_right = root->b[EB_RGHT];
229 if (unlikely(troot == NULL)) {
230 /* Tree is empty, insert the leaf part below the left branch */
231 root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF);
232 new->node.leaf_p = eb_dotag(root, EB_LEFT);
233 new->node.node_p = NULL; /* node part unused */
234 return new;
235 }
236
Willy Tarreauc2186022009-10-26 19:48:54 +0100237 /* The tree descent is fairly easy :
238 * - first, check if we have reached a leaf node
239 * - second, check if we have gone too far
240 * - third, reiterate
241 * Everywhere, we use <new> for the node node we are inserting, <root>
242 * for the node we attach it to, and <old> for the node we are
243 * displacing below <new>. <troot> will always point to the future node
244 * (tagged with its type). <side> carries the side the node <new> is
245 * attached to below its parent, which is also where previous node
246 * was attached.
247 */
248
249 bit = 0;
250 while (1) {
251 if (unlikely(eb_gettag(troot) == EB_LEAF)) {
Willy Tarreau3a932442010-05-09 19:29:23 +0200252 /* insert above a leaf */
Willy Tarreauc2186022009-10-26 19:48:54 +0100253 old = container_of(eb_untag(troot, EB_LEAF),
254 struct ebmb_node, node.branches);
Willy Tarreauc2186022009-10-26 19:48:54 +0100255 new->node.node_p = old->node.leaf_p;
Willy Tarreau3a932442010-05-09 19:29:23 +0200256 up_ptr = &old->node.leaf_p;
257 goto check_bit_and_break;
Willy Tarreauc2186022009-10-26 19:48:54 +0100258 }
259
260 /* OK we're walking down this link */
261 old = container_of(eb_untag(troot, EB_NODE),
262 struct ebmb_node, node.branches);
Willy Tarreau3a932442010-05-09 19:29:23 +0200263 old_node_bit = old->node.bit;
264
265 if (unlikely(old->node.bit < 0)) {
266 /* We're above a duplicate tree, so we must compare the whole value */
267 new->node.node_p = old->node.node_p;
268 up_ptr = &old->node.node_p;
269 check_bit_and_break:
270 bit = equal_bits(new->key, old->key, bit, len << 3);
271 break;
272 }
Willy Tarreauc2186022009-10-26 19:48:54 +0100273
274 /* Stop going down when we don't have common bits anymore. We
275 * also stop in front of a duplicates tree because it means we
276 * have to insert above. Note: we can compare more bits than
277 * the current node's because as long as they are identical, we
278 * know we descend along the correct side.
279 */
Willy Tarreau3a932442010-05-09 19:29:23 +0200280
281 bit = equal_bits(new->key, old->key, bit, old_node_bit);
282 if (unlikely(bit < old_node_bit)) {
283 /* The tree did not contain the key, so we insert <new> before the
284 * node <old>, and set ->bit to designate the lowest bit position in
285 * <new> which applies to ->branches.b[].
286 */
287 new->node.node_p = old->node.node_p;
288 up_ptr = &old->node.node_p;
289 break;
Willy Tarreauc2186022009-10-26 19:48:54 +0100290 }
Willy Tarreau3a932442010-05-09 19:29:23 +0200291 /* we don't want to skip bits for further comparisons, so we must limit <bit>.
292 * However, since we're going down around <old_node_bit>, we know it will be
293 * properly matched, so we can skip this bit.
294 */
295 bit = old_node_bit + 1;
296
297 /* walk down */
298 root = &old->node.branches;
299 side = old_node_bit & 7;
300 side ^= 7;
301 side = (new->key[old_node_bit >> 3] >> side) & 1;
302 troot = root->b[side];
303 }
304
305 new_left = eb_dotag(&new->node.branches, EB_LEFT);
306 new_rght = eb_dotag(&new->node.branches, EB_RGHT);
307 new_leaf = eb_dotag(&new->node.branches, EB_LEAF);
308
309 /* Note: we can compare more bits than
310 * the current node's because as long as they are identical, we
311 * know we descend along the correct side.
312 */
313 new->node.bit = bit;
314 diff = cmp_bits(new->key, old->key, bit);
315 if (diff == 0) {
316 new->node.bit = -1; /* mark as new dup tree, just in case */
Willy Tarreauc2186022009-10-26 19:48:54 +0100317
Willy Tarreau3a932442010-05-09 19:29:23 +0200318 if (likely(eb_gettag(root_right))) {
319 /* we refuse to duplicate this key if the tree is
320 * tagged as containing only unique keys.
Willy Tarreauc2186022009-10-26 19:48:54 +0100321 */
Willy Tarreau3a932442010-05-09 19:29:23 +0200322 return old;
323 }
Willy Tarreauc2186022009-10-26 19:48:54 +0100324
Willy Tarreau3a932442010-05-09 19:29:23 +0200325 if (eb_gettag(troot) != EB_LEAF) {
326 /* there was already a dup tree below */
327 struct eb_node *ret;
328 ret = eb_insert_dup(&old->node, &new->node);
329 return container_of(ret, struct ebmb_node, node);
330 }
331 /* otherwise fall through */
332 }
Willy Tarreauc2186022009-10-26 19:48:54 +0100333
Willy Tarreau3a932442010-05-09 19:29:23 +0200334 if (diff >= 0) {
335 new->node.branches.b[EB_LEFT] = troot;
336 new->node.branches.b[EB_RGHT] = new_leaf;
337 new->node.leaf_p = new_rght;
338 *up_ptr = new_left;
339 }
340 else if (diff < 0) {
341 new->node.branches.b[EB_LEFT] = new_leaf;
342 new->node.branches.b[EB_RGHT] = troot;
343 new->node.leaf_p = new_left;
344 *up_ptr = new_rght;
345 }
Willy Tarreauc2186022009-10-26 19:48:54 +0100346
Willy Tarreau3a932442010-05-09 19:29:23 +0200347 /* Ok, now we are inserting <new> between <root> and <old>. <old>'s
348 * parent is already set to <new>, and the <root>'s branch is still in
349 * <side>. Update the root's leaf till we have it. Note that we can also
350 * find the side by checking the side of new->node.node_p.
351 */
352
353 root->b[side] = eb_dotag(&new->node.branches, EB_NODE);
354 return new;
355}
356
357
358/* Find the first occurence of the longest prefix matching a key <x> in the
359 * tree <root>. It's the caller's responsibility to ensure that key <x> is at
360 * least as long as the keys in the tree. If none can be found, return NULL.
361 */
362static forceinline struct ebmb_node *__ebmb_lookup_longest(struct eb_root *root, const void *x)
363{
364 struct ebmb_node *node;
365 eb_troot_t *troot, *cover;
366 int pos, side;
367 int node_bit;
368
369 troot = root->b[EB_LEFT];
370 if (unlikely(troot == NULL))
371 return NULL;
372
373 cover = NULL;
374 pos = 0;
375 while (1) {
376 if ((eb_gettag(troot) == EB_LEAF)) {
377 node = container_of(eb_untag(troot, EB_LEAF),
378 struct ebmb_node, node.branches);
379 if (check_bits(x - pos, node->key, pos, node->node.pfx))
380 goto not_found;
381
382 return node;
383 }
384 node = container_of(eb_untag(troot, EB_NODE),
385 struct ebmb_node, node.branches);
386
387 node_bit = node->node.bit;
388 if (node_bit < 0) {
389 /* We have a dup tree now. Either it's for the same
390 * value, and we walk down left, or it's a different
391 * one and we don't have our key.
392 */
393 if (check_bits(x - pos, node->key, pos, node->node.pfx))
394 goto not_found;
395
396 troot = node->node.branches.b[EB_LEFT];
397 while (eb_gettag(troot) != EB_LEAF)
398 troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT];
399 node = container_of(eb_untag(troot, EB_LEAF),
400 struct ebmb_node, node.branches);
401 return node;
402 }
403
404 node_bit >>= 1; /* strip cover bit */
405 node_bit = ~node_bit + (pos << 3) + 8; // = (pos<<3) + (7 - node_bit)
406 if (node_bit < 0) {
407 /* This uncommon construction gives better performance
408 * because gcc does not try to reorder the loop. Tested to
409 * be fine with 2.95 to 4.2.
410 */
411 while (1) {
412 x++; pos++;
413 if (node->key[pos-1] ^ *(unsigned char*)(x-1))
414 goto not_found; /* more than one full byte is different */
415 node_bit += 8;
416 if (node_bit >= 0)
417 break;
Willy Tarreauc2186022009-10-26 19:48:54 +0100418 }
Willy Tarreau3a932442010-05-09 19:29:23 +0200419 }
420
421 /* here we know that only the last byte differs, so 0 <= node_bit <= 7.
422 * We have 2 possibilities :
423 * - more than the last bit differs => data does not match
424 * - walk down on side = (x[pos] >> node_bit) & 1
425 */
426 side = *(unsigned char *)x >> node_bit;
427 if (((node->key[pos] >> node_bit) ^ side) > 1)
428 goto not_found;
429
430 if (!(node->node.bit & 1)) {
431 /* This is a cover node, let's keep a reference to it
432 * for later. The covering subtree is on the left, and
433 * the covered subtree is on the right, so we have to
434 * walk down right.
435 */
436 cover = node->node.branches.b[EB_LEFT];
437 troot = node->node.branches.b[EB_RGHT];
438 continue;
439 }
440 side &= 1;
441 troot = node->node.branches.b[side];
442 }
443
444 not_found:
445 /* Walk down last cover tre if it exists. It does not matter if cover is NULL */
446 return ebmb_entry(eb_walk_down(cover, EB_LEFT), struct ebmb_node, node);
447}
448
449
450/* Find the first occurence of a prefix matching a key <x> of <pfx> BITS in the
Willy Tarreau414c4b22011-01-04 13:21:06 +0100451 * tree <root>. It's the caller's responsibility to ensure that key <x> is at
452 * least as long as the keys in the tree. If none can be found, return NULL.
Willy Tarreau3a932442010-05-09 19:29:23 +0200453 */
454static forceinline struct ebmb_node *__ebmb_lookup_prefix(struct eb_root *root, const void *x, unsigned int pfx)
455{
456 struct ebmb_node *node;
457 eb_troot_t *troot;
458 int pos, side;
459 int node_bit;
460
461 troot = root->b[EB_LEFT];
462 if (unlikely(troot == NULL))
463 return NULL;
464
465 pos = 0;
466 while (1) {
467 if ((eb_gettag(troot) == EB_LEAF)) {
468 node = container_of(eb_untag(troot, EB_LEAF),
469 struct ebmb_node, node.branches);
470 if (node->node.pfx != pfx)
471 return NULL;
472 if (check_bits(x - pos, node->key, pos, node->node.pfx))
473 return NULL;
474 return node;
475 }
476 node = container_of(eb_untag(troot, EB_NODE),
477 struct ebmb_node, node.branches);
478
479 node_bit = node->node.bit;
480 if (node_bit < 0) {
481 /* We have a dup tree now. Either it's for the same
482 * value, and we walk down left, or it's a different
483 * one and we don't have our key.
484 */
485 if (node->node.pfx != pfx)
486 return NULL;
487 if (check_bits(x - pos, node->key, pos, node->node.pfx))
488 return NULL;
489
490 troot = node->node.branches.b[EB_LEFT];
491 while (eb_gettag(troot) != EB_LEAF)
492 troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT];
493 node = container_of(eb_untag(troot, EB_LEAF),
494 struct ebmb_node, node.branches);
495 return node;
496 }
497
498 node_bit >>= 1; /* strip cover bit */
499 node_bit = ~node_bit + (pos << 3) + 8; // = (pos<<3) + (7 - node_bit)
500 if (node_bit < 0) {
501 /* This uncommon construction gives better performance
502 * because gcc does not try to reorder the loop. Tested to
503 * be fine with 2.95 to 4.2.
504 */
505 while (1) {
506 x++; pos++;
507 if (node->key[pos-1] ^ *(unsigned char*)(x-1))
508 return NULL; /* more than one full byte is different */
509 node_bit += 8;
510 if (node_bit >= 0)
511 break;
Willy Tarreauc2186022009-10-26 19:48:54 +0100512 }
Willy Tarreau3a932442010-05-09 19:29:23 +0200513 }
514
515 /* here we know that only the last byte differs, so 0 <= node_bit <= 7.
516 * We have 2 possibilities :
517 * - more than the last bit differs => data does not match
518 * - walk down on side = (x[pos] >> node_bit) & 1
519 */
520 side = *(unsigned char *)x >> node_bit;
521 if (((node->key[pos] >> node_bit) ^ side) > 1)
522 return NULL;
523
524 if (!(node->node.bit & 1)) {
525 /* This is a cover node, it may be the entry we're
526 * looking for. We already know that it matches all the
527 * bits, let's compare prefixes and descend the cover
528 * subtree if they match.
529 */
Willy Tarreau22c0a932011-07-25 12:22:44 +0200530 if ((unsigned short)node->node.bit >> 1 == pfx)
Willy Tarreau3a932442010-05-09 19:29:23 +0200531 troot = node->node.branches.b[EB_LEFT];
532 else
533 troot = node->node.branches.b[EB_RGHT];
534 continue;
535 }
536 side &= 1;
537 troot = node->node.branches.b[side];
538 }
539}
540
541
542/* Insert ebmb_node <new> into a prefix subtree starting at node root <root>.
543 * Only new->key and new->pfx need be set with the key and its prefix length.
544 * Note that bits between <pfx> and <len> are theorically ignored and should be
545 * zero, as it is not certain yet that they will always be ignored everywhere
546 * (eg in bit compare functions).
547 * The ebmb_node is returned.
548 * If root->b[EB_RGHT]==1, the tree may only contain unique keys. The
549 * len is specified in bytes.
550 */
551static forceinline struct ebmb_node *
552__ebmb_insert_prefix(struct eb_root *root, struct ebmb_node *new, unsigned int len)
553{
554 struct ebmb_node *old;
555 unsigned int side;
556 eb_troot_t *troot, **up_ptr;
Willy Tarreau6258f7b2011-09-19 20:48:00 +0200557 eb_troot_t *root_right;
Willy Tarreau3a932442010-05-09 19:29:23 +0200558 int diff;
559 int bit;
560 eb_troot_t *new_left, *new_rght;
561 eb_troot_t *new_leaf;
562 int old_node_bit;
563
564 side = EB_LEFT;
565 troot = root->b[EB_LEFT];
566 root_right = root->b[EB_RGHT];
567 if (unlikely(troot == NULL)) {
568 /* Tree is empty, insert the leaf part below the left branch */
569 root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF);
570 new->node.leaf_p = eb_dotag(root, EB_LEFT);
571 new->node.node_p = NULL; /* node part unused */
572 return new;
573 }
574
575 len <<= 3;
576 if (len > new->node.pfx)
577 len = new->node.pfx;
578
579 /* The tree descent is fairly easy :
580 * - first, check if we have reached a leaf node
581 * - second, check if we have gone too far
582 * - third, reiterate
583 * Everywhere, we use <new> for the node node we are inserting, <root>
584 * for the node we attach it to, and <old> for the node we are
585 * displacing below <new>. <troot> will always point to the future node
586 * (tagged with its type). <side> carries the side the node <new> is
587 * attached to below its parent, which is also where previous node
588 * was attached.
589 */
590
591 bit = 0;
592 while (1) {
593 if (unlikely(eb_gettag(troot) == EB_LEAF)) {
594 /* Insert above a leaf. Note that this leaf could very
595 * well be part of a cover node.
596 */
597 old = container_of(eb_untag(troot, EB_LEAF),
598 struct ebmb_node, node.branches);
599 new->node.node_p = old->node.leaf_p;
600 up_ptr = &old->node.leaf_p;
601 goto check_bit_and_break;
602 }
603
604 /* OK we're walking down this link */
605 old = container_of(eb_untag(troot, EB_NODE),
606 struct ebmb_node, node.branches);
607 old_node_bit = old->node.bit;
608 /* Note that old_node_bit can be :
609 * < 0 : dup tree
610 * = 2N : cover node for N bits
611 * = 2N+1 : normal node at N bits
612 */
613
614 if (unlikely(old_node_bit < 0)) {
615 /* We're above a duplicate tree, so we must compare the whole value */
616 new->node.node_p = old->node.node_p;
617 up_ptr = &old->node.node_p;
618 check_bit_and_break:
619 /* No need to compare everything if the leaves are shorter than the new one. */
620 if (len > old->node.pfx)
621 len = old->node.pfx;
622 bit = equal_bits(new->key, old->key, bit, len);
Willy Tarreauc2186022009-10-26 19:48:54 +0100623 break;
624 }
625
Willy Tarreau3a932442010-05-09 19:29:23 +0200626 /* WARNING: for the two blocks below, <bit> is counted in half-bits */
627
628 bit = equal_bits(new->key, old->key, bit, old_node_bit >> 1);
629 bit = (bit << 1) + 1; // assume comparisons with normal nodes
Willy Tarreau3a932442010-05-09 19:29:23 +0200630
631 /* we must always check that our prefix is larger than the nodes
632 * we visit, otherwise we have to stop going down. The following
633 * test is able to stop before both normal and cover nodes.
634 */
635 if (bit >= (new->node.pfx << 1) && (new->node.pfx << 1) < old_node_bit) {
636 /* insert cover node here on the left */
637 new->node.node_p = old->node.node_p;
638 up_ptr = &old->node.node_p;
639 new->node.bit = new->node.pfx << 1;
640 diff = -1;
Willy Tarreau3a932442010-05-09 19:29:23 +0200641 goto insert_above;
642 }
643
644 if (unlikely(bit < old_node_bit)) {
645 /* The tree did not contain the key, so we insert <new> before the
646 * node <old>, and set ->bit to designate the lowest bit position in
647 * <new> which applies to ->branches.b[]. We know that the bit is not
648 * greater than the prefix length thanks to the test above.
649 */
650 new->node.node_p = old->node.node_p;
651 up_ptr = &old->node.node_p;
652 new->node.bit = bit;
653 diff = cmp_bits(new->key, old->key, bit >> 1);
Willy Tarreau3a932442010-05-09 19:29:23 +0200654 goto insert_above;
655 }
656
657 if (!(old_node_bit & 1)) {
658 /* if we encounter a cover node with our exact prefix length, it's
659 * necessarily the same value, so we insert there as a duplicate on
660 * the left. For that, we go down on the left and the leaf detection
661 * code will finish the job.
662 */
663 if ((new->node.pfx << 1) == old_node_bit) {
664 root = &old->node.branches;
665 side = EB_LEFT;
666 troot = root->b[side];
Willy Tarreau3a932442010-05-09 19:29:23 +0200667 continue;
668 }
669
670 /* cover nodes are always walked through on the right */
671 side = EB_RGHT;
672 bit = old_node_bit >> 1; /* recheck that bit */
673 root = &old->node.branches;
674 troot = root->b[side];
Willy Tarreau3a932442010-05-09 19:29:23 +0200675 continue;
676 }
677
678 /* we don't want to skip bits for further comparisons, so we must limit <bit>.
679 * However, since we're going down around <old_node_bit>, we know it will be
680 * properly matched, so we can skip this bit.
681 */
682 old_node_bit >>= 1;
683 bit = old_node_bit + 1;
684
Willy Tarreauc2186022009-10-26 19:48:54 +0100685 /* walk down */
686 root = &old->node.branches;
Willy Tarreau3a932442010-05-09 19:29:23 +0200687 side = old_node_bit & 7;
688 side ^= 7;
689 side = (new->key[old_node_bit >> 3] >> side) & 1;
Willy Tarreauc2186022009-10-26 19:48:54 +0100690 troot = root->b[side];
691 }
692
Willy Tarreau3a932442010-05-09 19:29:23 +0200693 /* Right here, we have 4 possibilities :
694 * - the tree does not contain any leaf matching the
695 * key, and we have new->key < old->key. We insert
696 * new above old, on the left ;
697 *
698 * - the tree does not contain any leaf matching the
699 * key, and we have new->key > old->key. We insert
700 * new above old, on the right ;
701 *
702 * - the tree does contain the key with the same prefix
703 * length. We add the new key next to it as a first
704 * duplicate (since it was alone).
705 *
706 * The last two cases can easily be partially merged.
707 *
708 * - the tree contains a leaf matching the key, we have
709 * to insert above it as a cover node. The leaf with
710 * the shortest prefix becomes the left subtree and
711 * the leaf with the longest prefix becomes the right
712 * one. The cover node gets the min of both prefixes
713 * as its new bit.
Willy Tarreauc2186022009-10-26 19:48:54 +0100714 */
715
Willy Tarreau3a932442010-05-09 19:29:23 +0200716 /* first we want to ensure that we compare the correct bit, which means
717 * the largest common to both nodes.
Willy Tarreauc2186022009-10-26 19:48:54 +0100718 */
Willy Tarreau3a932442010-05-09 19:29:23 +0200719 if (bit > new->node.pfx)
720 bit = new->node.pfx;
721 if (bit > old->node.pfx)
722 bit = old->node.pfx;
723
Willy Tarreau3a932442010-05-09 19:29:23 +0200724 new->node.bit = (bit << 1) + 1; /* assume normal node by default */
725
726 /* if one prefix is included in the second one, we don't compare bits
727 * because they won't necessarily match, we just proceed with a cover
728 * node insertion.
729 */
730 diff = 0;
731 if (bit < old->node.pfx && bit < new->node.pfx)
732 diff = cmp_bits(new->key, old->key, bit);
733
734 if (diff == 0) {
735 /* Both keys match. Either it's a duplicate entry or we have to
736 * put the shortest prefix left and the largest one right below
737 * a new cover node. By default, diff==0 means we'll be inserted
738 * on the right.
739 */
740 new->node.bit--; /* anticipate cover node insertion */
741 if (new->node.pfx == old->node.pfx) {
Willy Tarreau3a932442010-05-09 19:29:23 +0200742 new->node.bit = -1; /* mark as new dup tree, just in case */
743
744 if (unlikely(eb_gettag(root_right))) {
745 /* we refuse to duplicate this key if the tree is
746 * tagged as containing only unique keys.
747 */
748 return old;
749 }
750
751 if (eb_gettag(troot) != EB_LEAF) {
752 /* there was already a dup tree below */
753 struct eb_node *ret;
754 ret = eb_insert_dup(&old->node, &new->node);
755 return container_of(ret, struct ebmb_node, node);
756 }
757 /* otherwise fall through to insert first duplicate */
758 }
759 /* otherwise we just rely on the tests below to select the right side */
760 else if (new->node.pfx < old->node.pfx)
761 diff = -1; /* force insertion to left side */
762 }
763
764 insert_above:
765 new_left = eb_dotag(&new->node.branches, EB_LEFT);
766 new_rght = eb_dotag(&new->node.branches, EB_RGHT);
767 new_leaf = eb_dotag(&new->node.branches, EB_LEAF);
768
769 if (diff >= 0) {
Willy Tarreau3a932442010-05-09 19:29:23 +0200770 new->node.branches.b[EB_LEFT] = troot;
771 new->node.branches.b[EB_RGHT] = new_leaf;
772 new->node.leaf_p = new_rght;
773 *up_ptr = new_left;
774 }
775 else {
Willy Tarreau3a932442010-05-09 19:29:23 +0200776 new->node.branches.b[EB_LEFT] = new_leaf;
777 new->node.branches.b[EB_RGHT] = troot;
778 new->node.leaf_p = new_left;
779 *up_ptr = new_rght;
780 }
781
Willy Tarreauc2186022009-10-26 19:48:54 +0100782 root->b[side] = eb_dotag(&new->node.branches, EB_NODE);
783 return new;
784}
785
Willy Tarreau3a932442010-05-09 19:29:23 +0200786
787
Willy Tarreauead63a02009-11-02 14:41:23 +0100788#endif /* _EBMBTREE_H */
789