Hi

I'm using db4 for a memory sensative application and encounter some
memory problem.

I've wrote a small test program
the program opens a file with 1,000,000 20 bytes keys and stores them
using the key as the data as well(for simplicity).

runing it I'd expect the process to use 20byte*2(for key and
data)*1000000 bytes
i.e. 38-39 Mb of memory.

I'm runing it with cache set to 50 Mb and hash size set to 1000000
and stat every 10000 records.

at around the 780,000 record the db on the file system(untouched till
now) is sudenly written at 81Mb and RSS(memory) is 59 Mb

This points to two problems
1. memory consumption is much more the caculated.
2. once cache is not enough the file system starts to work with
unreasonable sizes

db_stat:
# db_stat -d test.db
61561 Hash magic number.
8 Hash version number.
Flags: little-endian
16384 Underlying database page size.
0 Specified fill factor.
1000000 Number of keys in the database.
1000000 Number of data items in the database.
3061 Number of hash buckets.
10M Number of bytes free on bucket pages (79% ff).
0 Number of overflow pages.
0 Number of bytes free in overflow pages (0% ff).
1035 Number of bucket overflow pages.
10M Number of bytes free in bucket overflow pages (37% ff).
0 Number of duplicate pages.
0 Number of bytes free in duplicate pages (0% ff).
1 Number of pages on the free list.

The program:
#include <db_cxx.h>
#include <stdio.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <unistd.h>

#define CACHE_SIZE 50*1024*1024
#define NUM_CACHES 1
#define HASH_SIZE 1000000

typedef unsigned long u32;
typedef unsigned char u8;

Db* m_db;
bool mem_init = false;

void print_mem()
{
static char get_mem[1024];
static char* get_file = "ls -lh test.db";

if ( ! mem_init ) {
mem_init = true;
sprintf(get_mem, "cat /proc/%d/status | egrep 'VmSize|VmRSS'",
getpid());
}
system(get_mem);
system(get_file);
}

int init_db()
{
int rv;
u32 oFlags = DB_CREATE | DB_THREAD;


m_db = new Db(NULL, DB_CXX_NO_EXCEPTIONS);

rv = m_db->set_h_nelem(HASH_SIZE);
if ( 0 != rv ) {
fprintf(stderr, "ERROR: set_h_nelem failed %s", db_strerror(rv));
return -1;
}
rv = m_db->set_cachesize(0, CACHE_SIZE, NUM_CACHES);
if ( 0 != rv ) {
fprintf(stderr, "ERROR: set_cachesize failed %s", db_strerror(rv));
return -1;
}

rv = m_db->open(NULL, // Transaction pointer
"test.db", // Database file name
NULL, // Optional logical database name
DB_HASH, // Database access method
oFlags, // Open flags
0); // File mode (using defaults)
if ( 0 != rv ) {
fprintf(stderr, "ERROR: db open failed %s", db_strerror(rv));
return -1;
}



return 0;
}

int insert(u8* key, int key_size, void* data, int data_size)
{
Dbt l_key, l_data;
int rv;

l_key.set_data((char*)key);
l_key.set_size(key_size);
l_data.set_data(data);
l_data.set_size(data_size);

rv = m_db->put(NULL, &l_key, &l_data, 0);
if ( 0 != rv ) {
fprintf(stderr, "ERROR: db put failed %s", db_strerror(rv));
return rv;
}
return 0;
}

int run_all(char* test_file)
{
int rv;
FILE* fp;
char line[100];
int j;
char str_key[42];
u8 key[23];
u32 tot = 0;
u32 suc = 0;

print_mem();


rv = init_db();
if ( 0 != rv ) {
return rv;
}

fp = fopen(test_file, "r");
if (NULL == fp) {
fprintf(stderr, "Failed open test file\n");
return -1;
}

while ( fgets(line, 100, fp) ) {
sscanf(line, "%s", str_key);
for (j=0; j<20; j++) {
sscanf(str_key+j*2, "%02X", &key[j]);
}
rv = insert(key, 20, key, 20);
if (0 != rv) {
fprintf(stderr, "Insert failed(%d)\n", rv);
}
else {
suc++;
}
tot++;
if ( tot%10000 == 0 ) {
struct rusage usage;
fprintf(stderr, "%u\n", tot);
print_mem();
}
}

fprintf(stderr, "Got %u sucess: %u\n", tot, suc);

fclose(fp);
m_db->close(0);
print_mem();
return 0;
}


main(int argc, char* argv[])
{
if ( argc < 2 ) {
fprintf(stderr, "Please Enter test file\n");
}
run_all(argv[1]);

}