* Bugfix: 64bit box can not specify memory pool size larger than 2G with '-m' option

  * new option '-E' to automatically remove log files that are no longer needed, but PLEASE NOTICE:
    - "Automatic log file removal is likely to make catastrophic recovery impossible."
    - "Replication applications will rarely want to configure automatic log file removal as it increases the likelihood a master will be unable to satisfy a client's request for a recent log record." So this option will be disabled when replication is on regardless of whether you specify it.
  * Bugfix: slight security issue when client specify a negative value length


git-svn-id: http://memcachedb.googlecode.com/svn/trunk@94 8a849e15-1439-0410-a037-f546adf1399c

Author: stvchu

ChangeLog

@@ -1,3 +1,10 @@
+2009-02-26 Steve Chu <[email protected]>
+  * Bugfix: 64bit box can not specify memory pool size larger than 2G with '-m' option
+  * new option '-E' to automatically remove log files that are no longer needed, but PLEASE NOTICE:
+    - "Automatic log file removal is likely to make catastrophic recovery impossible."
+    - "Replication applications will rarely want to configure automatic log file removal as it increases the likelihood a master will be unable to satisfy a client's request for a recent log record." So this option will be disabled when replication is on regardless of whether you specify it.
+  * Bugfix: slight security issue when client specify a negative value length
+
 2009-01-13 Steve Chu <[email protected]>
   * some default settings policy changed(because most of us is using powerful box):
    max simultaneous connections ('-c'): 1024 -> 4096

bdb.c

@@ -44,6 +44,7 @@ void bdb_settings_init(void)
     bdb_settings.page_size = 4096;  /* default is 4K */
     bdb_settings.db_type = DB_BTREE;
     bdb_settings.txn_nosync = 0; /* default DB_TXN_NOSYNC is off */
+    bdb_settings.log_auto_remove = 0; /* default DB_LOG_AUTO_REMOVE is off */
     bdb_settings.dldetect_val = 100 * 1000; /* default is 100 millisecond */
     bdb_settings.chkpoint_val = 60 * 5;
     bdb_settings.memp_trickle_val = 30;
@@ -125,10 +126,16 @@ void bdb_env_init(void){
             exit(EXIT_FAILURE);
         }
     }
-
+    
     /* set MPOOL size */
-    env->set_cachesize(env, 0, bdb_settings.cache_size, 0);
-
+    if (sizeof(void *) == 4 && bdb_settings.cache_size > (1024uLL * 1024uLL * 1024uLL * 2uLL)) {
+        fprintf(stderr, "32bit box only max 2GB memory pool allowed\n");
+        exit(EXIT_FAILURE);
+    }
+    env->set_cachesize(env, (u_int32_t) (bdb_settings.cache_size / (1024uLL * 1024uLL * 1024uLL)), 
+                            (u_int32_t) (bdb_settings.cache_size % (1024uLL * 1024uLL * 1024uLL)), 
+                            (int) (bdb_settings.cache_size / (1024uLL * 1024uLL * 1024uLL * 4uLL) + 1uLL) );
+    
     /* set DB_TXN_NOSYNC flag */
     if (bdb_settings.txn_nosync){
         env->set_flags(env, DB_TXN_NOSYNC, 1);
@@ -145,6 +152,18 @@ void bdb_env_init(void){
     /* set transaction log buffer */
     env->set_lg_bsize(env, bdb_settings.txn_lg_bsize);
 
+    /* NOTICE:
+    If set, Berkeley DB will automatically remove log files that are no longer needed.
+    Automatic log file removal is likely to make catastrophic recovery impossible.
+    Replication applications will rarely want to configure automatic log file removal as it
+    increases the likelihood a master will be unable to satisfy a client's request 
+    for a recent log record.
+     */
+    if (!bdb_settings.is_replicated && bdb_settings.log_auto_remove){
+        fprintf(stderr, "log_auto_remove\n");        
+        env->log_set_config(env, DB_LOG_AUTO_REMOVE, 1);
+    }
+    
     /* if no home dir existed, we create it */
     if (0 != access(bdb_settings.env_home, F_OK)) {
         if (0 != mkdir(bdb_settings.env_home, 0750)) {

memcachedb.c

@@ -1260,7 +1260,8 @@ static void process_update_command(conn *c, token_t *tokens, const size_t ntoken
     exptime = strtol(tokens[3].value, NULL, 10);
     vlen = strtol(tokens[4].value, NULL, 10);
 
-    if(errno == ERANGE || ((flags == 0 || exptime == 0) && errno == EINVAL)) {
+    if(errno == ERANGE || ((flags == 0 || exptime == 0) && errno == EINVAL)
+       || vlen < 0) {
         out_string(c, "CLIENT_ERROR bad command line format");
         return;
     }
@@ -2348,6 +2349,7 @@ static void usage(void) {
     printf("-e <num>      percent of the pages in the cache that should be clean, default is 60%%\n");
     printf("-D <num>      do deadlock detecting every <num> millisecond, 0 for disable, default is 100ms\n");
     printf("-N            enable DB_TXN_NOSYNC to gain big performance improved, default is off\n");
+    printf("-E            automatically remove log files that are no longer needed\n");
     printf("-X            allocate region memory from the heap, default is off\n");
     printf("--------------------Replication Options-------------------------------\n");
     printf("-R            identifies the host and port used by this site (required).\n");
@@ -2564,7 +2566,7 @@ int main (int argc, char **argv) {
     setbuf(stderr, NULL);
 
     /* process arguments */
-    while ((c = getopt(argc, argv, "a:U:p:s:c:hivl:dru:P:t:b:f:H:B:m:A:L:C:T:e:D:NXMSR:O:n:")) != -1) {
+    while ((c = getopt(argc, argv, "a:U:p:s:c:hivl:dru:P:t:b:f:H:B:m:A:L:C:T:e:D:NEXMSR:O:n:")) != -1) {
         switch (c) {
         case 'a':
             /* access for unix domain socket, as octal mask (like chmod)*/
@@ -2643,7 +2645,7 @@ int main (int argc, char **argv) {
             }
             break;
         case 'm':
-            bdb_settings.cache_size = atoi(optarg) * 1024 * 1024;
+            bdb_settings.cache_size = atoi(optarg) * 1024uLL * 1024uLL;
             break;
         case 'A':
             bdb_settings.page_size = atoi(optarg);
@@ -2671,6 +2673,9 @@ int main (int argc, char **argv) {
         case 'N':
             bdb_settings.txn_nosync = 1;
             break;
+        case 'E':
+            bdb_settings.log_auto_remove = 1;
+            break;
         case 'X':
             bdb_settings.env_flags |= DB_PRIVATE;
             break;

memcachedb.h

@@ -124,11 +124,12 @@ enum mdb_rep_role { MDB_MASTER, MDB_CLIENT, MDB_UNKNOWN };
 struct bdb_settings {
     char *db_file;    /* db filename, where dbfile located. */
     char *env_home;    /* db env home dir path */
-    u_int32_t cache_size; /* cache size */
+    u_int64_t cache_size; /* cache size */
     u_int32_t txn_lg_bsize; /* transaction log buffer size */
     u_int32_t page_size;    /* underlying database pagesize*/
     DBTYPE db_type;
     int txn_nosync;    /* DB_TXN_NOSYNC flag, if 1 will lose transaction's durability for performance */
+    int log_auto_remove;    /* DB_LOG_AUTO_REMOVE flag, if 1 will make catastrophic recovery impossible. */
     int dldetect_val; /* do deadlock detect every *db_lock_detect_val* millisecond, 0 for disable */
     int chkpoint_val;  /* do checkpoint every *db_chkpoint_val* second, 0 for disable */
     int memp_trickle_val;  /* do memp_trickle every *memp_trickle_val* second, 0 for disable */

run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-#./memcachedb -p21201 -d -r -H ./testenv -N -v >log 2>&1
-./memcachedb -p21201 -d -r -H ./testenv0 -N -R 127.0.0.1:31201 -M -n 2 -v >log_m 2>&1
-sleep 5
-./memcachedb -p21202 -d -r -H ./testenv1 -N -R 127.0.0.1:31202 -O 127.0.0.1:31201 -S -n 2 -v >log_s 2>&1
+./memcachedb -p21201 -d -r -H ./testenv -N -E -v >log 2>&1
+#./memcachedb -p21201 -d -r -H ./testenv0 -N -R 127.0.0.1:31201 -M -n 2 -v >log_m 2>&1
+#sleep 5
+#./memcachedb -p21202 -d -r -H ./testenv1 -N -R 127.0.0.1:31202 -O 127.0.0.1:31201 -S -n 2 -v >log_s 2>&1

stats.c

@@ -23,6 +23,9 @@
 void stats_bdb(char *temp){
     char *pos = temp;
     int ret;
+    u_int32_t gbytes = 0;
+    u_int32_t bytes = 0;
+    int ncache = 0;
     /* get bdb version */
     pos += sprintf(pos, "STAT db_ver %d.%d.%d\r\n", bdb_version.majver, 
                                                     bdb_version.minver, 
@@ -40,9 +43,15 @@ void stats_bdb(char *temp){
             pos += sprintf(pos, "STAT db_type hash\r\n");
         }
     }
-    pos += sprintf(pos, "STAT cache_size %u\r\n", bdb_settings.cache_size);
+    
+    /* get cache size */
+    if((ret = env->get_cachesize(env, &gbytes, &bytes, &ncache)) == 0){
+        pos += sprintf(pos, "STAT cache_size %lu/%lu/%d\r\n", gbytes, bytes, ncache);
+    }
+    
     pos += sprintf(pos, "STAT txn_lg_bsize %u\r\n", bdb_settings.txn_lg_bsize);
     pos += sprintf(pos, "STAT txn_nosync %d\r\n", bdb_settings.txn_nosync);
+    pos += sprintf(pos, "STAT log_auto_remove %d\r\n", bdb_settings.log_auto_remove);
     pos += sprintf(pos, "STAT dldetect_val %d\r\n", bdb_settings.dldetect_val);
     pos += sprintf(pos, "STAT chkpoint_val %d\r\n", bdb_settings.chkpoint_val);
     pos += sprintf(pos, "STAT memp_trickle_val %d\r\n", bdb_settings.memp_trickle_val);