Hash optimize patch (1.9.3-p0)

hash_optimize.patch

diff --git a/common.mk b/common.mk
index ea244cc..de09d9a 100644
--- a/common.mk
+++ b/common.mk
@@ -692,7 +692,8 @@ signal.$(OBJEXT): {$(VPATH)}signal.c $(RUBY_H_INCLUDES) \
   $(VM_CORE_H_INCLUDES) {$(VPATH)}debug.h
 sprintf.$(OBJEXT): {$(VPATH)}sprintf.c $(RUBY_H_INCLUDES) {$(VPATH)}re.h \
   {$(VPATH)}regex.h {$(VPATH)}vsnprintf.c $(ENCODING_H_INCLUDES)
-st.$(OBJEXT): {$(VPATH)}st.c $(RUBY_H_INCLUDES)
+st.$(OBJEXT): {$(VPATH)}st.c $(RUBY_H_INCLUDES) {$(VPATH)}pool_alloc.inc.h \
+  {$(VPATH)}internal.h
 strftime.$(OBJEXT): {$(VPATH)}strftime.c $(RUBY_H_INCLUDES) \
   {$(VPATH)}timev.h
 string.$(OBJEXT): {$(VPATH)}string.c $(RUBY_H_INCLUDES) {$(VPATH)}re.h \
diff --git a/gc.c b/gc.c
index 3238d65..3b3af5c 100644
--- a/gc.c
+++ b/gc.c
@@ -766,6 +766,15 @@ vm_xmalloc(rb_objspace_t *objspace, size_t size)
 }
 
 static void *
+vm_xmalloc_only(rb_objspace_t *objspace, size_t size)
+{
+    void *mem;
+
+    TRY_WITH_GC(mem = malloc(size));
+    return vm_malloc_fixup(objspace, mem, size);
+}
+
+static void *
 vm_xrealloc(rb_objspace_t *objspace, void *ptr, size_t size)
 {
     void *mem;
@@ -827,6 +836,18 @@ ruby_xmalloc(size_t size)
     return vm_xmalloc(&rb_objspace, size);
 }
 
+size_t
+ruby_gcprepare(size_t size)
+{
+    return vm_malloc_prepare(&rb_objspace, size);
+}
+
+void *
+ruby_xmalloc_prepared(size_t size)
+{
+    return vm_xmalloc_only(&rb_objspace, size);
+}
+
 static inline size_t
 xmalloc2_size(size_t n, size_t size)
 {
diff --git a/internal.h b/internal.h
index 172e7f4..e238707 100644
--- a/internal.h
+++ b/internal.h
@@ -92,6 +92,10 @@ void Init_File(void);
 
 /* gc.c */
 void Init_heap(void);
+#define xgc_prepare ruby_gcprepare
+#define xmalloc_prepared ruby_xmalloc_prepared
+size_t xgc_prepare(size_t);
+void *xmalloc_prepared(size_t);
 
 /* inits.c */
 void rb_call_inits(void);
diff --git a/pool_alloc.inc.h b/pool_alloc.inc.h
new file mode 100644
index 0000000..b72ccd8
--- /dev/null
+++ b/pool_alloc.inc.h
@@ -0,0 +1,153 @@
+/*
+ * this is generic pool allocator
+ * you should define following macroses:
+ * ITEM_NAME - unique identifier, which allows to hold functions in a namespace
+ * ITEM_TYPEDEF(name) - passed to typedef to localize item type
+ * free_entry - desired name of function for free entry
+ * alloc_entry - defired name of function for allocate entry
+ */
+
+#define NAME_(prefix, kind) sta_##prefix##_##kind
+#define NAME(prefix, kind) NAME_(prefix, kind)
+
+#define holder_typename NAME(holder, ITEM_NAME)
+#define entry_typename NAME(entry, ITEM_NAME)
+#define list_typename NAME(list, ITEM_NAME)
+#define union_typename NAME(union, ITEM_NAME)
+#define item_type NAME(item, ITEM_NAME)
+
+typedef ITEM_TYPEDEF(item_type);
+typedef struct holder_typename holder_typename;
+typedef struct entry_typename entry_typename;
+
+typedef struct list_typename {
+    entry_typename *fore, *back;
+} list_typename;
+
+typedef union union_typename {
+    list_typename l;
+    item_type item;
+} union_typename;
+
+struct entry_typename {
+    union_typename p;
+    holder_typename *holder;
+};
+
+#define HOLDER_SIZE ((4096 - sizeof(void*) * 3 - sizeof(int)) / sizeof(entry_typename) )
+struct holder_typename {
+    unsigned int free;
+    entry_typename items[HOLDER_SIZE];
+};
+
+#define free_entry_p NAME(free_pointer, ITEM_NAME)
+#define free_entry_count NAME(count, ITEM_NAME)
+static entry_typename *free_entry_p = NULL;
+static unsigned long free_entry_count = 0;
+
+#define entry_chain NAME(chain, ITEM_NAME)
+#define holder_alloc NAME(holder_alloc, ITEM_NAME)
+#define holder_free NAME(holder_free, ITEM_NAME)
+#define fore p.l.fore
+#define back p.l.back
+
+static inline void
+entry_chain(entry_typename *entry)
+{
+    entry->fore = free_entry_p;
+    entry->back = NULL;
+    if (free_entry_p) {
+	free_entry_p->back = entry;
+    }
+    free_entry_p = entry;
+}
+
+static void
+holder_alloc()
+{
+    holder_typename *holder;
+    unsigned int i;
+    register entry_typename *ptr;
+#ifdef xgc_prepare
+    size_t sz = xgc_prepare(sizeof(holder_typename));
+    if (free_entry_p) return;
+    holder = (holder_typename*)xmalloc_prepared(sz);
+#else
+    holder = alloc(holder_typename);
+#endif 
+    ptr = holder->items;
+    holder->free = HOLDER_SIZE;
+    for(i = HOLDER_SIZE - 1; i; ptr++, i-- ) {
+        ptr->holder = holder;
+        ptr->fore = ptr + 1;
+        (ptr + 1)->back = ptr;
+    }
+    holder->items[0].back = NULL;
+    holder->items[HOLDER_SIZE - 1].holder = holder;
+    holder->items[HOLDER_SIZE - 1].fore = free_entry_p;
+    free_entry_p = &holder->items[0];
+    free_entry_count+= HOLDER_SIZE;
+}
+
+static void
+holder_free(holder_typename *holder)
+{
+    unsigned int i;
+    entry_typename *ptr = holder->items;
+    for(i = HOLDER_SIZE; i; i--, ptr++) {
+	if (ptr->fore) {
+	    ptr->fore->back = ptr->back;
+	}
+	if (ptr->back) {
+	    ptr->back->fore = ptr->fore;
+	} else {
+	    free_entry_p = ptr->fore;
+	}
+    }
+    free_entry_count-= HOLDER_SIZE;
+    free(holder);
+}
+
+static void
+free_entry(item_type *entry)
+{
+    holder_typename *holder = ((entry_typename *)entry)->holder;
+    entry_chain((entry_typename *)entry);
+    holder->free++;
+    free_entry_count++;
+    if (holder->free == HOLDER_SIZE && free_entry_count > HOLDER_SIZE * 16) {
+	holder_free(holder);
+    }
+}
+
+static item_type *
+alloc_entry()
+{
+    entry_typename *result;
+    if (!free_entry_p) {
+	holder_alloc();
+    }
+    result = free_entry_p;
+    free_entry_p = result->fore;
+    result->holder->free--;
+    free_entry_count--;
+    return (item_type *)result;
+}
+
+
+
+#undef NAME_
+#undef NAME
+#undef holder_typename
+#undef entry_typename
+#undef list_typename
+#undef union_typename
+#undef item_type
+#undef free_entry_p
+#undef free_entry_count
+#undef HOLDER_SIZE
+#undef entry_chain
+#undef holder_alloc
+#undef holdef_free
+#undef fore
+#undef back
diff --git a/st.c b/st.c
index ba21b31..6cfc163 100644
--- a/st.c
+++ b/st.c
@@ -7,6 +7,7 @@
 #include "st.h"
 #else
 #include "ruby/ruby.h"
+#include "internal.h"
 #endif
 
 #include <stdio.h>
@@ -25,8 +26,10 @@ struct st_table_entry {
     st_table_entry *fore, *back;
 };
 
-#define ST_DEFAULT_MAX_DENSITY 5
+#define ST_DEFAULT_MAX_DENSITY 3
 #define ST_DEFAULT_INIT_TABLE_SIZE 11
+#define ST_DEFAULT_PACKED_TABLE_SIZE 19
+#define MAX_PACKED_HASH (ST_DEFAULT_PACKED_TABLE_SIZE / 3)
 
     /*
      * DEFAULT_MAX_DENSITY is the default for the largest we allow the
@@ -73,7 +76,129 @@ static void rehash(st_table *);
 
 /* remove cast to unsigned int in the future */
 #define do_hash(key,table) (unsigned int)(st_index_t)(*(table)->type->hash)((key))
-#define do_hash_bin(key,table) (do_hash((key), (table))%(table)->num_bins)
+#define PKEY_POS(i, num_bins) ((num_bins)-(i)*2-2) //((num_bins)/3 + (i)*2)
+#define PVAL_POS(i, num_bins) ((num_bins)-(i)*2-1) //((num_bins)/3 + (i)*2 + 1)
+#define PHASH_POS(i, num_bins) (i) //((num_bins)-(i)*3-1) //(i)
+#define PKEY(table, i) (st_data_t)(table)->bins[PKEY_POS(i, (table)->num_bins)]
+#define PVAL(table, i) (st_data_t)(table)->bins[PVAL_POS(i, (table)->num_bins)]
+#define PHASH(table, i) (st_data_t)(table)->bins[PHASH_POS(i, (table)->num_bins)]
+#define PKEY_SET(table, i, v) do{ (table)->bins[PKEY_POS(i, (table)->num_bins)] = (st_table_entry *)(v); } while(0)
+#define PVAL_SET(table, i, v) do{ (table)->bins[PVAL_POS(i, (table)->num_bins)] = (st_table_entry *)(v); } while(0)
+#define PHASH_SET(table, i, v) do{ (table)->bins[PHASH_POS(i, (table)->num_bins)] = (st_table_entry *)(v); } while(0)
+//#define ST_PACKED_REFERENCE
+#ifdef ST_PACKED_REFERENCE
+static inline void
+remove_packed_entry(st_table *table, st_index_t i)
+{
+    table->num_entries--;
+    for(;i < table->num_entries; i++) {
+        PVAL_SET(table, i, PVAL(table, i+1));
+        PKEY_SET(table, i, PKEY(table, i+1));
+        PHASH_SET(table, i, PHASH(table, i+1));
+    }
+}
+#else
+static inline void
+remove_packed_entry(st_table *table, st_index_t i)
+{
+    table->num_entries--;
+    if (i < table->num_entries) {
+        st_index_t mv = table->num_entries - i, upto = table->num_bins - 2*table->num_entries;
+        memmove(table->bins + i, table->bins + i + 1, sizeof(st_table_entry *) * mv);
+        memmove(table->bins + upto, table->bins + upto - 2,
+                sizeof(st_table_entry *) * mv * 2);
+    }
+}
+#endif
+
+#define ST_USE_POOLED_ALLOCATOR
+#ifdef ST_USE_POOLED_ALLOCATOR
+
+#define ITEM_NAME entry
+#define ITEM_TYPEDEF(name) st_table_entry name
+#define free_entry st_free_entry
+#define alloc_entry st_alloc_entry
+#include "pool_alloc.inc.h"
+#undef ITEM_NAME
+#undef ITEM_TYPEDEF
+#undef free_entry
+#undef alloc_entry
+
+typedef st_table_entry *st_table_entry_p;
+#define ITEM_NAME bins11
+#define ITEM_TYPEDEF(name) st_table_entry_p name[ST_DEFAULT_INIT_TABLE_SIZE]
+#define free_entry st_free_bins11
+#define alloc_entry st_alloc_bins11
+#include "pool_alloc.inc.h"
+#undef ITEM_NAME
+#undef ITEM_TYPEDEF
+#undef free_entry
+#undef alloc_entry
+
+#define ITEM_NAME bins19
+#define ITEM_TYPEDEF(name) st_table_entry_p name[ST_DEFAULT_PACKED_TABLE_SIZE]
+#define free_entry st_free_bins19
+#define alloc_entry st_alloc_bins19
+#include "pool_alloc.inc.h"
+#undef ITEM_NAME
+#undef ITEM_TYPEDEF
+#undef free_entry
+#undef alloc_entry
+
+#define ITEM_NAME table
+#define ITEM_TYPEDEF(name) st_table name
+#define free_entry st_dealloc_table
+#define alloc_entry st_alloc_table
+#include "pool_alloc.inc.h"
+#undef ITEM_NAME
+#undef ITEM_TYPEDEF
+#undef free_entry
+#undef alloc_entry
+
+static st_table_entry **
+st_alloc_bins(st_index_t num_bins)
+{
+    st_table_entry **result;
+    if (num_bins == ST_DEFAULT_PACKED_TABLE_SIZE) {
+	result = (st_table_entry **) st_alloc_bins19();
+    }
+    else
+    if (num_bins == ST_DEFAULT_INIT_TABLE_SIZE) {
+	result = (st_table_entry **) st_alloc_bins11();
+    }
+    else {
+	result = (st_table_entry **) malloc(num_bins * sizeof(st_table_entry *));
+    }
+    memset(result, 0, num_bins * sizeof(st_table_entry *));
+    return result;
+}
+
+static void
+st_free_bins(st_table_entry **bins, st_index_t num_bins)
+{
+    if (num_bins == ST_DEFAULT_PACKED_TABLE_SIZE) {
+	st_free_bins19(
+		(st_table_entry_p (*)[ST_DEFAULT_PACKED_TABLE_SIZE]) bins);
+    }
+    else
+    if (num_bins == ST_DEFAULT_INIT_TABLE_SIZE) {
+	st_free_bins11(
+		(st_table_entry_p (*)[ST_DEFAULT_INIT_TABLE_SIZE]) bins);
+    } else {
+	free(bins);
+    }
+}
+
+#else
+
+#define st_alloc_entry() alloc(st_table_entry)
+#define st_free_entry(entry) free(entry)
+#define st_alloc_table() alloc(st_table)
+#define st_dealloc_table(table) free(table)
+#define st_alloc_bins(size) (st_table_entry **)Calloc(size, sizeof(st_table_entry *))
+#define st_free_bins(bins, size) free(bins)
+
+#endif
 
 /*
  * MINSIZE is the minimum size of a dictionary.
@@ -85,8 +210,8 @@ static void rehash(st_table *);
 Table of prime numbers 2^n+a, 2<=n<=30.
 */
 static const unsigned int primes[] = {
-	8 + 3,
-	16 + 3,
+	ST_DEFAULT_INIT_TABLE_SIZE,
+	ST_DEFAULT_PACKED_TABLE_SIZE,
 	32 + 5,
 	64 + 3,
 	128 + 3,
@@ -161,8 +286,6 @@ stat_col(void)
 }
 #endif
 
-#define MAX_PACKED_NUMHASH (ST_DEFAULT_INIT_TABLE_SIZE/2)
-
 st_table*
 st_init_table_with_size(const struct st_hash_type *type, st_index_t size)
 {
@@ -181,14 +304,18 @@ st_init_table_with_size(const struct st_hash_type *type, st_index_t size)
     }
 #endif
 
-    size = new_size(size);	/* round up to prime number */
 
-    tbl = alloc(st_table);
+    tbl = st_alloc_table();
     tbl->type = type;
     tbl->num_entries = 0;
-    tbl->entries_packed = type == &type_numhash && size/2 <= MAX_PACKED_NUMHASH;
+    if ( (tbl->entries_packed = size <= MAX_PACKED_HASH) ) {
+	size = ST_DEFAULT_PACKED_TABLE_SIZE;
+    }
+    else {
+	size = new_size(size);	/* round up to prime number */
+    }
     tbl->num_bins = size;
-    tbl->bins = (st_table_entry **)Calloc(size, sizeof(st_table_entry*));
+    tbl->bins = st_alloc_bins(size);
     tbl->head = 0;
     tbl->tail = 0;
 
@@ -253,7 +380,7 @@ st_clear(st_table *table)
 	table->bins[i] = 0;
 	while (ptr != 0) {
 	    next = ptr->next;
-	    free(ptr);
+	    st_free_entry(ptr);
 	    ptr = next;
 	}
     }
@@ -266,8 +393,8 @@ void
 st_free_table(st_table *table)
 {
     st_clear(table);
-    free(table->bins);
-    free(table);
+    st_free_bins(table->bins, table->num_bins);
+    st_dealloc_table(table);
 }
 
 size_t
@@ -306,40 +433,54 @@ count_collision(const struct st_hash_type *type)
 #define FOUND_ENTRY
 #endif
 
-#define FIND_ENTRY(table, ptr, hash_val, bin_pos) do {\
-    (bin_pos) = (hash_val)%(table)->num_bins;\
-    (ptr) = (table)->bins[(bin_pos)];\
-    FOUND_ENTRY;\
-    if (PTR_NOT_EQUAL((table), (ptr), (hash_val), key)) {\
-	COLLISION;\
-	while (PTR_NOT_EQUAL((table), (ptr)->next, (hash_val), key)) {\
-	    (ptr) = (ptr)->next;\
-	}\
-	(ptr) = (ptr)->next;\
-    }\
-} while (0)
+static st_table_entry *
+find_entry(st_table *table, st_data_t key, st_index_t hash_val, st_index_t bin_pos)
+{
+    register st_table_entry *ptr = table->bins[bin_pos];
+    FOUND_ENTRY;
+    if (PTR_NOT_EQUAL(table, ptr, hash_val, key)) {
+	COLLISION;
+	while (PTR_NOT_EQUAL(table, ptr->next, hash_val, key)) {
+	    ptr = ptr->next;
+	}
+	ptr = ptr->next;
+    }
+    return ptr;
+}
+
+static inline st_index_t
+find_packed_index(st_table *table, st_index_t hash_val, st_data_t key)
+{
+    st_index_t i = 0;
+    for(;;) {
+        while (i < table->num_entries && PHASH(table, i) != hash_val) i++;
+        if (i == table->num_entries || EQUAL(table, key, PKEY(table, i)))
+            break;
+        i++;
+    }
+    return i;
+}
 
 #define collision_check 0
 
 int
 st_lookup(st_table *table, register st_data_t key, st_data_t *value)
 {
-    st_index_t hash_val, bin_pos;
+    st_index_t hash_val;
     register st_table_entry *ptr;
 
+    hash_val = do_hash(key, table);
+
     if (table->entries_packed) {
-        st_index_t i;
-        for (i = 0; i < table->num_entries; i++) {
-            if ((st_data_t)table->bins[i*2] == key) {
-                if (value !=0) *value = (st_data_t)table->bins[i*2+1];
-                return 1;
-            }
-        }
+	st_index_t i = find_packed_index(table, hash_val, key);
+	if (i < table->num_entries) {
+	    if (value != 0) *value = PVAL(table, i);
+	    return 1;
+	}
         return 0;
     }
 
-    hash_val = do_hash(key, table);
-    FIND_ENTRY(table, ptr, hash_val, bin_pos);
+    ptr = find_entry(table, key, hash_val, hash_val % table->num_bins);
 
     if (ptr == 0) {
 	return 0;
@@ -353,22 +494,21 @@ st_lookup(st_table *table, register st_data_t key, st_data_t *value)
 int
 st_get_key(st_table *table, register st_data_t key, st_data_t *result)
 {
-    st_index_t hash_val, bin_pos;
+    st_index_t hash_val;
     register st_table_entry *ptr;
 
+    hash_val = do_hash(key, table);
+
     if (table->entries_packed) {
-        st_index_t i;
-        for (i = 0; i < table->num_entries; i++) {
-            if ((st_data_t)table->bins[i*2] == key) {
-                if (result !=0) *result = (st_data_t)table->bins[i*2];
-                return 1;
-            }
-        }
+	st_index_t i = find_packed_index(table, hash_val, key);
+	if (i < table->num_entries) {
+	    if (result != 0) *result = PKEY(table, i);
+	    return 1;
+	}
         return 0;
     }
 
-    hash_val = do_hash(key, table);
-    FIND_ENTRY(table, ptr, hash_val, bin_pos);
+    ptr = find_entry(table, key, hash_val, hash_val % table->num_bins);
 
     if (ptr == 0) {
 	return 0;
@@ -382,85 +522,92 @@ st_get_key(st_table *table, register st_data_t key, st_data_t *result)
 #undef collision_check
 #define collision_check 1
 
-#define MORE_PACKABLE_P(table) \
-    ((st_index_t)((table)->num_entries+1) * 2 <= (table)->num_bins && \
-     (table)->num_entries+1 <= MAX_PACKED_NUMHASH)
-
-#define ADD_DIRECT(table, key, value, hash_val, bin_pos)\
-do {\
-    st_table_entry *entry;\
-    if ((table)->num_entries > ST_DEFAULT_MAX_DENSITY * (table)->num_bins) {\
-	rehash(table);\
-        (bin_pos) = (hash_val) % (table)->num_bins;\
-    }\
-    \
-    entry = alloc(st_table_entry);\
-    \
-    entry->hash = (hash_val);\
-    entry->key = (key);\
-    entry->record = (value);\
-    entry->next = (table)->bins[(bin_pos)];\
-    if ((table)->head != 0) {\
-	entry->fore = 0;\
-	(entry->back = (table)->tail)->fore = entry;\
-	(table)->tail = entry;\
-    }\
-    else {\
-	(table)->head = (table)->tail = entry;\
-	entry->fore = entry->back = 0;\
-    }\
-    (table)->bins[(bin_pos)] = entry;\
-    (table)->num_entries++;\
-} while (0)
+static void
+add_direct(st_table * table, st_data_t key, st_data_t value,
+	st_index_t hash_val, st_index_t bin_pos)
+{
+    st_table_entry *entry;
+    if (table->num_entries > ST_DEFAULT_MAX_DENSITY * table->num_bins) {
+	rehash(table);
+	bin_pos = hash_val % table->num_bins;
+    }
+
+    entry = st_alloc_entry();
+
+    entry->hash = hash_val;
+    entry->key = key;
+    entry->record = value;
+    if (table->head != 0) {
+	entry->fore = 0;
+	(entry->back = table->tail)->fore = entry;
+	table->tail = entry;
+    }
+    else {
+	table->head = table->tail = entry;
+	entry->fore = entry->back = 0;
+    }
+    entry->next = table->bins[bin_pos];
+    table->bins[bin_pos] = entry;
+    table->num_entries++;
+}
 
 static void
 unpack_entries(register st_table *table)
 {
     st_index_t i;
-    struct st_table_entry *packed_bins[MAX_PACKED_NUMHASH*2];
-    st_table tmp_table = *table;
-
-    memcpy(packed_bins, table->bins, sizeof(struct st_table_entry *) * table->num_entries*2);
-    table->bins = packed_bins;
-    tmp_table.entries_packed = 0;
-    tmp_table.num_entries = 0;
-    memset(tmp_table.bins, 0, sizeof(struct st_table_entry *) * tmp_table.num_bins);
+    st_table tmp_table = {table->type, 0, 0, 0, 0, 0, 0};
+
+    tmp_table.bins = st_alloc_bins(ST_DEFAULT_INIT_TABLE_SIZE);
+    tmp_table.num_bins = ST_DEFAULT_INIT_TABLE_SIZE;
     for (i = 0; i < table->num_entries; i++) {
-        st_insert(&tmp_table, (st_data_t)packed_bins[i*2], (st_data_t)packed_bins[i*2+1]);
+	add_direct(&tmp_table, PKEY(table, i), PVAL(table, i), PHASH(table, i), PHASH(table, i)%tmp_table.num_bins);
     }
+    st_free_bins(table->bins, table->num_bins);
     *table = tmp_table;
 }
 
+static int
+add_packed_direct(st_table *table, st_data_t key, st_data_t value, st_index_t hash_val)
+{
+    int res = 1;
+    if (table->num_entries < MAX_PACKED_HASH ) {
+	st_index_t i = table->num_entries;
+	PKEY_SET(table, i, key);
+	PVAL_SET(table, i, value);
+	PHASH_SET(table, i, hash_val);
+        table->num_entries++;
+    }
+    else {
+	unpack_entries(table);
+	res = 0;
+    }
+    return res;
+}
+
 int
 st_insert(register st_table *table, register st_data_t key, st_data_t value)
 {
     st_index_t hash_val, bin_pos;
     register st_table_entry *ptr;
 
+    hash_val = do_hash(key, table);
+
     if (table->entries_packed) {
-        st_index_t i;
-        for (i = 0; i < table->num_entries; i++) {
-            if ((st_data_t)table->bins[i*2] == key) {
-                table->bins[i*2+1] = (struct st_table_entry*)value;
-                return 1;
-            }
-        }
-        if (MORE_PACKABLE_P(table)) {
-            i = table->num_entries++;
-            table->bins[i*2] = (struct st_table_entry*)key;
-            table->bins[i*2+1] = (struct st_table_entry*)value;
-            return 0;
-        }
-        else {
-            unpack_entries(table);
-        }
+	st_index_t i = find_packed_index(table, hash_val, key);
+	if (i < table->num_entries) {
+	    PVAL_SET(table, i, value);
+	    return 1;
+	}
+	if (add_packed_direct(table, key, value, hash_val)) {
+	    return 0;
+	}
     }
 
-    hash_val = do_hash(key, table);
-    FIND_ENTRY(table, ptr, hash_val, bin_pos);
+    bin_pos = hash_val % table->num_bins;
+    ptr = find_entry(table, key, hash_val, bin_pos);
 
     if (ptr == 0) {
-	ADD_DIRECT(table, key, value, hash_val, bin_pos);
+	add_direct(table, key, value, hash_val, bin_pos);
 	return 0;
     }
     else {
@@ -476,31 +623,25 @@ st_insert2(register st_table *table, register st_data_t key, st_data_t value,
     st_index_t hash_val, bin_pos;
     register st_table_entry *ptr;
 
+    hash_val = do_hash(key, table);
+
     if (table->entries_packed) {
-        st_index_t i;
-        for (i = 0; i < table->num_entries; i++) {
-            if ((st_data_t)table->bins[i*2] == key) {
-                table->bins[i*2+1] = (struct st_table_entry*)value;
-                return 1;
-            }
-        }
-        if (MORE_PACKABLE_P(table)) {
-            i = table->num_entries++;
-            table->bins[i*2] = (struct st_table_entry*)key;
-            table->bins[i*2+1] = (struct st_table_entry*)value;
-            return 0;
-        }
-        else {
-            unpack_entries(table);
-        }
+	st_index_t i = find_packed_index(table, hash_val, key);
+	if (i < table->num_entries) {
+	    PVAL_SET(table, i, value);
+	    return 1;
+	}
+	if (add_packed_direct(table, key, value, hash_val)) {
+	    return 0;
+	}
     }
 
-    hash_val = do_hash(key, table);
-    FIND_ENTRY(table, ptr, hash_val, bin_pos);
+    bin_pos = hash_val % table->num_bins;
+    ptr = find_entry(table, key, hash_val, bin_pos);
 
     if (ptr == 0) {
 	key = (*func)(key);
-	ADD_DIRECT(table, key, value, hash_val, bin_pos);
+	add_direct(table, key, value, hash_val, bin_pos);
 	return 0;
     }
     else {
@@ -512,24 +653,16 @@ st_insert2(register st_table *table, register st_data_t key, st_data_t value,
 void
 st_add_direct(st_table *table, st_data_t key, st_data_t value)
 {
-    st_index_t hash_val, bin_pos;
+    st_index_t hash_val;
 
-    if (table->entries_packed) {
-        int i;
-        if (MORE_PACKABLE_P(table)) {
-            i = table->num_entries++;
-            table->bins[i*2] = (struct st_table_entry*)key;
-            table->bins[i*2+1] = (struct st_table_entry*)value;
-            return;
-        }
-        else {
-            unpack_entries(table);
-        }
+    hash_val = do_hash(key, table);
+    if (table->entries_packed ) {
+	if (add_packed_direct(table, key, value, hash_val)) {
+	    return;
+	}
     }
 
-    hash_val = do_hash(key, table);
-    bin_pos = hash_val % table->num_bins;
-    ADD_DIRECT(table, key, value, hash_val, bin_pos);
+    add_direct(table, key, value, hash_val, hash_val % table->num_bins);
 }
 
 static void
@@ -539,9 +672,8 @@ rehash(register st_table *table)
     st_index_t i, new_num_bins, hash_val;
 
     new_num_bins = new_size(table->num_bins+1);
-    new_bins = (st_table_entry**)
-	xrealloc(table->bins, new_num_bins * sizeof(st_table_entry*));
-    for (i = 0; i < new_num_bins; ++i) new_bins[i] = 0;
+    st_free_bins(table->bins, table->num_bins);
+    new_bins = st_alloc_bins(new_num_bins);
     table->num_bins = new_num_bins;
     table->bins = new_bins;
 
@@ -562,17 +694,16 @@ st_copy(st_table *old_table)
     st_index_t num_bins = old_table->num_bins;
     st_index_t hash_val;
 
-    new_table = alloc(st_table);
+    new_table = st_alloc_table();
     if (new_table == 0) {
 	return 0;
     }
 
     *new_table = *old_table;
-    new_table->bins = (st_table_entry**)
-	Calloc((unsigned)num_bins, sizeof(st_table_entry*));
+    new_table->bins = st_alloc_bins(num_bins);
 
     if (new_table->bins == 0) {
-	free(new_table);
+	st_dealloc_table(new_table);
 	return 0;
     }
 
@@ -585,9 +716,9 @@ st_copy(st_table *old_table)
 	prev = 0;
 	tail = &new_table->head;
 	do {
-	    entry = alloc(st_table_entry);
+	    entry = st_alloc_entry();
 	    if (entry == 0) {
-		st_free_table(new_table);
+		st_dealloc_table(new_table);
 		return 0;
 	    }
 	    *entry = *ptr;
@@ -604,21 +735,22 @@ st_copy(st_table *old_table)
     return new_table;
 }
 
-#define REMOVE_ENTRY(table, ptr) do					\
-    {									\
-	if ((ptr)->fore == 0 && (ptr)->back == 0) {			\
-	    (table)->head = 0;						\
-	    (table)->tail = 0;						\
-	}								\
-	else {								\
-	    st_table_entry *fore = (ptr)->fore, *back = (ptr)->back;	\
-	    if (fore) fore->back = back;				\
-	    if (back) back->fore = fore;				\
-	    if ((ptr) == (table)->head) (table)->head = fore;		\
-	    if ((ptr) == (table)->tail) (table)->tail = back;		\
-	}								\
-	(table)->num_entries--;						\
-    } while (0)
+static inline void
+remove_entry(st_table *table, st_table_entry *ptr)
+{
+    if (ptr->fore == 0 && ptr->back == 0) {
+	table->head = 0;
+	table->tail = 0;
+    }
+    else {
+	st_table_entry *fore = ptr->fore, *back = ptr->back;
+	if (fore) fore->back = back;
+	if (back) back->fore = fore;
+	if (ptr == table->head) table->head = fore;
+	if (ptr == table->tail) table->tail = back;
+    }
+    table->num_entries--;
+}
 
 int
 st_delete(register st_table *table, register st_data_t *key, st_data_t *value)
@@ -627,30 +759,26 @@ st_delete(register st_table *table, register st_data_t *key, st_data_t *value)
     st_table_entry **prev;
     register st_table_entry *ptr;
 
+    hash_val = do_hash(*key, table);
+
     if (table->entries_packed) {
-        st_index_t i;
-        for (i = 0; i < table->num_entries; i++) {
-            if ((st_data_t)table->bins[i*2] == *key) {
-                if (value != 0) *value = (st_data_t)table->bins[i*2+1];
-                table->num_entries--;
-                memmove(&table->bins[i*2], &table->bins[(i+1)*2],
-                        sizeof(struct st_table_entry*) * 2*(table->num_entries-i));
-                return 1;
-            }
+	st_index_t i = find_packed_index(table, hash_val, *key);
+	if (i < table->num_entries) {
+	    if (value != 0) *value = PVAL(table, i);
+            remove_packed_entry(table, i);
+	    return 1;
         }
         if (value != 0) *value = 0;
         return 0;
     }
 
-    hash_val = do_hash_bin(*key, table);
-
-    for (prev = &table->bins[hash_val]; (ptr = *prev) != 0; prev = &ptr->next) {
+    for (prev = &table->bins[hash_val % table->num_bins]; (ptr = *prev) != 0; prev = &ptr->next) {
 	if (EQUAL(table, *key, ptr->key)) {
 	    *prev = ptr->next;
-	    REMOVE_ENTRY(table, ptr);
+	    remove_entry(table, ptr);
 	    if (value != 0) *value = ptr->record;
 	    *key = ptr->key;
-	    free(ptr);
+	    st_free_entry(ptr);
 	    return 1;
 	}
     }
@@ -665,25 +793,25 @@ st_delete_safe(register st_table *table, register st_data_t *key, st_data_t *val
     st_index_t hash_val;
     register st_table_entry *ptr;
 
+    hash_val = do_hash(*key, table);
+
     if (table->entries_packed) {
-	st_index_t i;
-	for (i = 0; i < table->num_entries; i++) {
-	    if ((st_data_t)table->bins[i*2] == *key) {
-		if (value != 0) *value = (st_data_t)table->bins[i*2+1];
-		table->bins[i*2] = (void *)never;
-		return 1;
-	    }
+	st_index_t i = find_packed_index(table, hash_val, *key);
+	if (i < table->num_entries) {
+	    if (value != 0) *value = PVAL(table, i);
+	    PKEY_SET(table, i, never);
+	    PHASH_SET(table, i,  0);
+	    return 1;
 	}
 	if (value != 0) *value = 0;
 	return 0;
     }
 
-    hash_val = do_hash_bin(*key, table);
-    ptr = table->bins[hash_val];
+    ptr = table->bins[hash_val % table->num_bins];
 
     for (; ptr != 0; ptr = ptr->next) {
 	if ((ptr->key != never) && EQUAL(table, ptr->key, *key)) {
-	    REMOVE_ENTRY(table, ptr);
+	    remove_entry(table, ptr);
 	    *key = ptr->key;
 	    if (value != 0) *value = ptr->record;
 	    ptr->key = ptr->record = never;
@@ -703,13 +831,14 @@ st_cleanup_safe(st_table *table, st_data_t never)
 
     if (table->entries_packed) {
 	st_index_t i = 0, j = 0;
-	while ((st_data_t)table->bins[i*2] != never) {
+	while (PKEY(table, i) != never) {
 	    if (i++ == table->num_entries) return;
 	}
 	for (j = i; ++i < table->num_entries;) {
-	    if ((st_data_t)table->bins[i*2] == never) continue;
-	    table->bins[j*2] = table->bins[i*2];
-	    table->bins[j*2+1] = table->bins[i*2+1];
+	    if (PKEY(table, i) == never) continue;
+	    PKEY_SET(table, j,  PKEY(table, i));
+	    PVAL_SET(table, j,  PVAL(table, i));
+	    PHASH_SET(table, j,  PHASH(table, i));
 	    j++;
 	}
 	table->num_entries = j;
@@ -722,7 +851,7 @@ st_cleanup_safe(st_table *table, st_data_t never)
 	    if (ptr->key == never) {
 		tmp = ptr;
 		*last = ptr = ptr->next;
-		free(tmp);
+		st_free_entry(tmp);
 	    }
 	    else {
 		ptr = *(last = &ptr->next);
@@ -740,19 +869,22 @@ st_foreach(st_table *table, int (*func)(ANYARGS), st_data_t arg)
 
     if (table->entries_packed) {
         for (i = 0; i < table->num_entries; i++) {
-            st_index_t j;
+            st_index_t hash;
             st_data_t key, val;
-            key = (st_data_t)table->bins[i*2];
-            val = (st_data_t)table->bins[i*2+1];
+            key = PKEY(table, i);
+            val = PVAL(table, i);
+            hash = PHASH(table,i);
             retval = (*func)(key, val, arg);
 	    if (!table->entries_packed) goto unpacked;
             switch (retval) {
 	      case ST_CHECK:	/* check if hash is modified during iteration */
-                for (j = 0; j < table->num_entries; j++) {
-                    if ((st_data_t)table->bins[j*2] == key)
-                        break;
-                }
-                if (j == table->num_entries) {
+		/* work around uncomforming befaviour of hash */
+		if (PKEY(table, i) == Qundef && PHASH(table, i) == 0)
+		    break;
+		else if (i < table->num_entries &&
+			PHASH(table, i) == hash && EQUAL(table, key, PKEY(table, i)))
+		    break;
+                if (find_packed_index(table, hash, key) == table->num_entries) {
                     /* call func with error notice */
                     retval = (*func)(0, 0, arg, 1);
                     return 1;
@@ -763,9 +895,7 @@ st_foreach(st_table *table, int (*func)(ANYARGS), st_data_t arg)
 	      case ST_STOP:
 		return 0;
 	      case ST_DELETE:
-                table->num_entries--;
-                memmove(&table->bins[i*2], &table->bins[(i+1)*2],
-                        sizeof(struct st_table_entry*) * 2*(table->num_entries-i));
+                remove_packed_entry(table, i);
                 i--;
                 break;
             }
@@ -806,8 +936,8 @@ st_foreach(st_table *table, int (*func)(ANYARGS), st_data_t arg)
 		    if (ptr == tmp) {
 			tmp = ptr->fore;
 			*last = ptr->next;
-			REMOVE_ENTRY(table, ptr);
-			free(ptr);
+			remove_entry(table, ptr);
+			st_free_entry(ptr);
 			if (ptr == tmp) return 0;
 			ptr = tmp;
 			break;
@@ -886,7 +1016,7 @@ st_reverse_foreach(st_table *table, int (*func)(ANYARGS), st_data_t arg)
 		    if (ptr == tmp) {
 			tmp = ptr->back;
 			*last = ptr->next;
-			REMOVE_ENTRY(table, ptr);
+			remove_entry(table, ptr);
 			free(ptr);
 			ptr = tmp;
 			break;