introducing automatic purging of expired pastes, triggered by default at least 5 minutes apart, deleting a maximum of 10 pastes - resolves #3

lib/configuration.php

@@ -70,6 +70,11 @@ class configuration
             'header' => null,
             'dir' => 'data',
         ),
+        'purge' => array(
+            'limit' => 300,
+            'batchsize' => 10,
+            'dir' => 'data',
+        ),
         'model' => array(
             'class' => 'privatebin_data',
         ),

lib/model.php

@@ -35,6 +35,7 @@ class model
      * Factory constructor.
      *
      * @param configuration $conf
+     * @return void
      */
     public function __construct(configuration $conf)
     {
@@ -54,8 +55,24 @@ class model
         return $paste;
     }
 
+    /**
+     * Checks if a purge is necessary and triggers it if yes.
+     *
+     * @return void
+     */
+    public function purge()
+    {
+        purgelimiter::setConfiguration($this->_conf);
+        if (purgelimiter::canPurge())
+        {
+            $this->_getStore()->purge($this->_conf->getKey('batchsize', 'purge'));
+        }
+    }
+
     /**
      * Gets, and creates if neccessary, a store object
+     *
+     * @return privatebin_abstract
      */
     private function _getStore()
     {

lib/privatebin.php

@@ -264,6 +264,7 @@ class privatebin
         // The user posts a standard paste.
         else
         {
+            $this->_model->purge();
             $paste = $this->_model->getPaste();
             try {
                 $paste->setData($data);

lib/privatebin/abstract.php

@@ -123,6 +123,35 @@ abstract class privatebin_abstract
      */
     abstract public function existsComment($pasteid, $parentid, $commentid);
 
+    /**
+     * Returns up to batch size number of paste ids that have expired
+     *
+     * @access protected
+     * @param  int $batchsize
+     * @return array
+     */
+    abstract protected function _getExpiredPastes($batchsize);
+
+    /**
+     * Perform a purge of old pastes, at most the given batchsize is deleted.
+     *
+     * @access public
+     * @param  int $batchsize
+     * @return void
+     */
+    public function purge($batchsize)
+    {
+        if ($batchsize < 1) return;
+        $pastes = $this->_getExpiredPastes($batchsize);
+        if (count($pastes))
+        {
+            foreach ($pastes as $pasteid)
+            {
+                $this->delete($pasteid);
+            }
+        }
+    }
+
     /**
      * Get next free slot for comment from postdate.
      *

lib/privatebin/data.php

@@ -210,6 +210,67 @@ class privatebin_data extends privatebin_abstract
         );
     }
 
+    /**
+     * Returns up to batch size number of paste ids that have expired
+     *
+     * @access private
+     * @param  int $batchsize
+     * @return array
+     */
+    protected function _getExpiredPastes($batchsize)
+    {
+        $pastes = array();
+        $firstLevel = array_filter(
+            scandir(self::$_dir),
+            array('self', '_isFirstLevelDir')
+        );
+        if (count($firstLevel) > 0)
+        {
+            // try at most 10 times the $batchsize pastes before giving up
+            for ($i = 0, $max = $batchsize * 10; $i < $max; ++$i)
+            {
+                $firstKey = array_rand($firstLevel);
+                $secondLevel = array_filter(
+                    scandir(self::$_dir . $firstLevel[$firstKey]),
+                    array('self', '_isSecondLevelDir')
+                );
+
+                // skip this folder in the next checks if it is empty
+                if (count($secondLevel) == 0)
+                {
+                    unset($firstLevel[$firstKey]);
+                    continue;
+                }
+
+                $secondKey = array_rand($secondLevel);
+                $path = self::$_dir . $firstLevel[$firstKey] . '/' . $secondLevel[$secondKey];
+                if (!is_dir($path)) continue;
+                $thirdLevel = array_filter(
+                    scandir($path),
+                    array('model_paste', 'isValidId')
+                );
+                if (count($thirdLevel) == 0) continue;
+                $thirdKey = array_rand($thirdLevel);
+                $pasteid = $thirdLevel[$thirdKey];
+                if (in_array($pasteid, $pastes)) continue;
+
+                if ($this->exists($pasteid))
+                {
+                    $data = $this->read($pasteid);
+                    if (
+                        property_exists($data->meta, 'expire_date') &&
+                        $data->meta->expire_date < time()
+                    )
+                    {
+                        $pastes[] = $pasteid;
+                        if (count($pastes) >= $batchsize) break;
+                    }
+                }
+            }
+        }
+        return $pastes;
+    }
+
     /**
      * initialize privatebin
      *
@@ -266,4 +327,30 @@ class privatebin_data extends privatebin_abstract
     {
         return self::_dataid2path($dataid) . $dataid . '.discussion/';
     }
+
+    /**
+     * Check that the given element is a valid first level directory.
+     *
+     * @access private
+     * @static
+     * @param  string $element
+     * @return bool
+     */
+    private static function _isFirstLevelDir($element)
+    {
+        return self::_isSecondLevelDir($element) && is_dir(self::$_dir . '/' . $element);
+    }
+
+    /**
+     * Check that the given element is a valid second level directory.
+     *
+     * @access private
+     * @static
+     * @param  string $element
+     * @return bool
+     */
+    private static function _isSecondLevelDir($element)
+    {
+        return (bool) preg_match('/^[a-f0-9]{2}$/', $element);
+    }
 }

lib/privatebin/db.php

@@ -302,7 +302,7 @@ class privatebin_db extends privatebin_abstract
      * Test if a paste exists.
      *
      * @access public
-     * @param  string $dataid
+     * @param  string $pasteid
      * @return void
      */
     public function exists($pasteid)
@@ -381,7 +381,7 @@ class privatebin_db extends privatebin_abstract
      * Test if a comment exists.
      *
      * @access public
-     * @param  string $dataid
+     * @param  string $pasteid
      * @param  string $parentid
      * @param  string $commentid
      * @return void
@@ -395,6 +395,30 @@ class privatebin_db extends privatebin_abstract
         );
     }
 
+    /**
+     * Returns up to batch size number of paste ids that have expired
+     *
+     * @access private
+     * @param  int $batchsize
+     * @return array
+     */
+    protected function _getExpiredPastes($batchsize)
+    {
+        $pastes = array();
+        $rows = self::_select(
+            'SELECT dataid FROM ' . self::_sanitizeIdentifier('paste') .
+            ' WHERE expiredate < ? LIMIT ?', array(time(), $batchsize)
+        );
+        if (count($rows))
+        {
+            foreach ($rows as $row)
+            {
+                $pastes[] = $row['dataid'];
+            }
+        }
+        return $pastes;
+    }
+
     /**
      * execute a statement
      *

lib/purgelimiter.php

@@ -0,0 +1,99 @@
+<?php
+/**
+ * PrivateBin
+ *
+ * a zero-knowledge paste bin
+ *
+ * @link      https://github.com/PrivateBin/PrivateBin
+ * @copyright 2012 Sébastien SAUVAGE (sebsauvage.net)
+ * @license   http://www.opensource.org/licenses/zlib-license.php The zlib/libpng License
+ * @version   0.22
+ */
+
+/**
+ * purgelimiter
+ *
+ * Handles purge limiting, so purging is not triggered to often.
+ */
+class purgelimiter extends persistence
+{
+    /**
+     * time limit in seconds, defaults to 300s
+     *
+     * @access private
+     * @static
+     * @var    int
+     */
+    private static $_limit = 300;
+
+    /**
+     * set the time limit in seconds
+     *
+     * @access public
+     * @static
+     * @param  int $limit
+     * @return void
+     */
+    public static function setLimit($limit)
+    {
+        self::$_limit = $limit;
+    }
+
+    /**
+     * set configuration options of the traffic limiter
+     *
+     * @access public
+     * @static
+     * @param configuration $conf
+     * @return void
+     */
+    public static function setConfiguration(configuration $conf)
+    {
+        self::setLimit($conf->getKey('limit', 'purge'));
+        self::setPath($conf->getKey('dir', 'purge'));
+    }
+
+    /**
+     * check if the purge can be performed
+     *
+     * @access public
+     * @static
+     * @throws Exception
+     * @return bool
+     */
+    public static function canPurge()
+    {
+        // disable limits if set to less then 1
+        if (self::$_limit < 1) return true;
+
+        $file = 'purge_limiter.php';
+        $now = time();
+        if (!self::_exists($file))
+        {
+            self::_store(
+                $file,
+                '<?php' . PHP_EOL .
+                '$GLOBALS[\'purge_limiter\'] = ' . $now . ';' . PHP_EOL
+            );
+        }
+
+        $path = self::getPath($file);
+        require $path;
+        $pl = $GLOBALS['purge_limiter'];
+
+        if ($pl + self::$_limit >= $now)
+        {
+            $result = false;
+        }
+        else
+        {
+            $result = true;
+            self::_store(
+                $file,
+                '<?php' . PHP_EOL .
+                '$GLOBALS[\'purge_limiter\'] = ' . $now . ';' . PHP_EOL
+            );
+        }
+        return $result;
+    }
+}

tst/configuration.php

@@ -12,6 +12,7 @@ class configurationTest extends PHPUnit_Framework_TestCase
         $this->_options = configuration::getDefaults();
         $this->_options['model_options']['dir'] = PATH . $this->_options['model_options']['dir'];
         $this->_options['traffic']['dir'] = PATH . $this->_options['traffic']['dir'];
+        $this->_options['purge']['dir'] = PATH . $this->_options['purge']['dir'];
         $this->_minimalConfig = '[main]' . PHP_EOL . '[model]' . PHP_EOL . '[model_options]';
     }
 

tst/privatebin.php

@@ -184,7 +184,11 @@ class privatebinTest extends PHPUnit_Framework_TestCase
     public function testCreateInvalidTimelimit()
     {
         $this->reset();
-        $_POST = helper::getPaste();
+        $options = parse_ini_file(CONF, true);
+        $options['traffic']['limit'] = 0;
+        helper::confBackup();
+        helper::createIniFile(CONF, $options);
+        $_POST = helper::getPaste(array('expire' => 25));
         $_SERVER['HTTP_X_REQUESTED_WITH'] = 'JSONHttpRequest';
         $_SERVER['REQUEST_METHOD'] = 'POST';
         $_SERVER['REMOTE_ADDR'] = '::1';
@@ -193,8 +197,14 @@ class privatebinTest extends PHPUnit_Framework_TestCase
         new privatebin;
         $content = ob_get_contents();
         $response = json_decode($content, true);
-        $this->assertEquals(1, $response['status'], 'outputs error status');
+        $this->assertEquals(0, $response['status'], 'outputs status');
-        $this->assertFalse($this->_model->exists(helper::getPasteId()), 'paste exists after posting data');
+        $this->assertTrue($this->_model->exists($response['id']), 'paste exists after posting data');
+        $paste = $this->_model->read($response['id']);
+        $this->assertEquals(
+            hash_hmac('sha256', $response['id'], $paste->meta->salt),
+            $response['deletetoken'],
+            'outputs valid delete token'
+        );
     }
 
     /**
@@ -228,11 +238,10 @@ class privatebinTest extends PHPUnit_Framework_TestCase
         $this->reset();
         $options = parse_ini_file(CONF, true);
         $options['traffic']['header'] = 'X_FORWARDED_FOR';
-        $options['traffic']['limit'] = 100;
         helper::confBackup();
         helper::createIniFile(CONF, $options);
         $_POST = helper::getPaste();
-        $_SERVER['HTTP_X_FORWARDED_FOR'] = '::1';
+        $_SERVER['HTTP_X_FORWARDED_FOR'] = '::2';
         $_SERVER['HTTP_X_REQUESTED_WITH'] = 'JSONHttpRequest';
         $_SERVER['REQUEST_METHOD'] = 'POST';
         $_SERVER['REMOTE_ADDR'] = '::1';
@@ -240,8 +249,14 @@ class privatebinTest extends PHPUnit_Framework_TestCase
         new privatebin;
         $content = ob_get_contents();
         $response = json_decode($content, true);
-        $this->assertEquals(1, $response['status'], 'outputs error status');
+        $this->assertEquals(0, $response['status'], 'outputs status');
-        $this->assertFalse($this->_model->exists(helper::getPasteId()), 'paste exists after posting data');
+        $this->assertTrue($this->_model->exists($response['id']), 'paste exists after posting data');
+        $paste = $this->_model->read($response['id']);
+        $this->assertEquals(
+            hash_hmac('sha256', $response['id'], $paste->meta->salt),
+            $response['deletetoken'],
+            'outputs valid delete token'
+        );
     }
 
     /**

tst/privatebin/data.php

@@ -63,4 +63,37 @@ class privatebin_dataTest extends PHPUnit_Framework_TestCase
         $this->assertEquals(json_decode(json_encode($original)), $this->_model->read(helper::getPasteId()));
     }
 
+    public function testPurge()
+    {
+        $expired = helper::getPaste(array('expire_date' => 1344803344));
+        $paste = helper::getPaste(array('expire_date' => time() + 3600));
+        $keys = array('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'x', 'y', 'z');
+        $ids = array();
+        foreach ($keys as $key)
+        {
+            $ids[$key] = substr(md5($key), 0, 16);
+            $this->assertFalse($this->_model->exists($ids[$key]), "paste $key does not yet exist");
+            if (in_array($key, array('x', 'y', 'z')))
+            {
+                $this->assertTrue($this->_model->create($ids[$key], $paste), "store $key paste");
+            }
+            else
+            {
+                $this->assertTrue($this->_model->create($ids[$key], $expired), "store $key paste");
+            }
+            $this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after storing it");
+        }
+        $this->_model->purge(10);
+        foreach ($ids as $key => $id)
+        {
+            if (in_array($key, array('x', 'y', 'z')))
+            {
+                $this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after purge");
+            }
+            else
+            {
+                $this->assertFalse($this->_model->exists($ids[$key]), "paste $key was purged");
+            }
+        }
+    }
 }

tst/privatebin/db.php

@@ -16,6 +16,12 @@ class privatebin_dbTest extends PHPUnit_Framework_TestCase
         $this->_model = privatebin_db::getInstance($this->_options);
     }
 
+    public function tearDown()
+    {
+        /* Tear Down Routine */
+        if (is_dir(PATH . 'data')) helper::rmdir(PATH . 'data');
+    }
+
     public function testDatabaseBasedDataStoreWorks()
     {
         $this->_model->delete(helper::getPasteId());
@@ -62,6 +68,41 @@ class privatebin_dbTest extends PHPUnit_Framework_TestCase
         $this->assertEquals(json_decode(json_encode($original)), $this->_model->read(helper::getPasteId()));
     }
 
+    public function testPurge()
+    {
+        $this->_model->delete(helper::getPasteId());
+        $expired = helper::getPaste(array('expire_date' => 1344803344));
+        $paste = helper::getPaste(array('expire_date' => time() + 3600));
+        $keys = array('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'x', 'y', 'z');
+        $ids = array();
+        foreach ($keys as $key)
+        {
+            $ids[$key] = substr(md5($key), 0, 16);
+            $this->assertFalse($this->_model->exists($ids[$key]), "paste $key does not yet exist");
+            if (in_array($key, array('x', 'y', 'z')))
+            {
+                $this->assertTrue($this->_model->create($ids[$key], $paste), "store $key paste");
+            }
+            else
+            {
+                $this->assertTrue($this->_model->create($ids[$key], $expired), "store $key paste");
+            }
+            $this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after storing it");
+        }
+        $this->_model->purge(10);
+        foreach ($ids as $key => $id)
+        {
+            if (in_array($key, array('x', 'y', 'z')))
+            {
+                $this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after purge");
+            }
+            else
+            {
+                $this->assertFalse($this->_model->exists($ids[$key]), "paste $key was purged");
+            }
+        }
+    }
+
     /**
      * @expectedException PDOException
      */
@@ -185,6 +226,7 @@ class privatebin_dbTest extends PHPUnit_Framework_TestCase
 
     public function testTableUpgrade()
     {
+        mkdir(PATH . 'data');
         $path = PATH . 'data/db-test.sq3';
         @unlink($path);
         $this->_options['dsn'] = 'sqlite:' . $path;

tst/privatebinWithDb.php

@@ -4,7 +4,6 @@ require_once 'privatebin.php';
 class privatebinWithDbTest extends privatebinTest
 {
     private $_options = array(
-        'dsn' => 'sqlite:../data/tst.sq3',
         'usr' => null,
         'pwd' => null,
         'opt' => array(
@@ -13,11 +12,15 @@ class privatebinWithDbTest extends privatebinTest
         ),
     );
 
+    private $_path;
+
     public function setUp()
     {
         /* Setup Routine */
+        $this->_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'privatebin_data';
+        if(!is_dir($this->_path)) mkdir($this->_path);
+        $this->_options['dsn'] = 'sqlite:' . $this->_path . '/tst.sq3';
         $this->_model = privatebin_db::getInstance($this->_options);
-        serversalt::setPath(PATH . 'data');
         $this->reset();
     }
 
@@ -25,7 +28,7 @@ class privatebinWithDbTest extends privatebinTest
     {
         /* Tear Down Routine */
         parent::tearDown();
-        @unlink('../data/tst.sq3');
+        helper::rmdir($this->_path);
     }
 
     public function reset()

tst/purgelimiter.php

@@ -0,0 +1,36 @@
+<?php
+class purgelimiterTest extends PHPUnit_Framework_TestCase
+{
+    private $_path;
+
+    public function setUp()
+    {
+        /* Setup Routine */
+        $this->_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'privatebin_data';
+        if(!is_dir($this->_path)) mkdir($this->_path);
+        purgelimiter::setPath($this->_path);
+    }
+
+    public function tearDown()
+    {
+        /* Tear Down Routine */
+        helper::rmdir($this->_path);
+    }
+
+    public function testLimit()
+    {
+        // initialize it
+        purgelimiter::canPurge();
+
+        // try setting it
+        purgelimiter::setLimit(1);
+        $this->assertEquals(false, purgelimiter::canPurge());
+        sleep(2);
+        $this->assertEquals(true, purgelimiter::canPurge());
+
+        // disable it
+        purgelimiter::setLimit(0);
+        purgelimiter::canPurge();
+        $this->assertEquals(true, purgelimiter::canPurge());
+    }
+}