Index: D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage.class =================================================================== --- D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage.class (revision 14982) +++ D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage.class (working copy) @@ -1334,9 +1334,10 @@ * * @param string $value the value * @param int $size (as a STORAGE_SIZE_XXX constant) + * @param bool $lengthInBytes (optional) whether to interpret the size in bytes or in characters * @return the truncated string */ - function _truncateString($value, $size) { + function _truncateString($value, $size, $lengthInBytes=false) { switch ($size) { case STORAGE_SIZE_SMALL: $size = 32; @@ -1350,7 +1351,11 @@ $size = 255; break; } - return GalleryUtilities::utf8Substring($value, 0, $size); + if ($lengthInBytes) { + return GalleryUtilities::utf8Strcut($value, 0, $size); + } else { + return GalleryUtilities::utf8Substring($value, 0, $size); + } } } @@ -1707,5 +1712,15 @@ return $extras->_getUniqueIdWithConnection($db); } + + /** + * Truncate UTF-8 strings either to given character or to byte length depending on the MySQL + * version. + * @see GalleryStorage::_truncateString + */ + function _truncateString($value, $size, $lengthInBytes=false) { + $lengthInBytes = $this->_serverInfo && version_compare($this->_serverInfo, '4.1.0', '<'); + return parent::_truncateString($value, $size, $lengthInBytes); + } } ?> Index: D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage/Db2Storage.class =================================================================== --- D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage/Db2Storage.class (revision 14982) +++ D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage/Db2Storage.class (working copy) @@ -278,5 +278,14 @@ return 'CALL ADMIN_CMD (\'RUNSTATS ON TABLE ' . $this->_username . '.%s WITH DISTRIBUTION ON ALL COLUMNS AND INDEXES ALL\')'; } + + /** + * Truncate UTF-8 strings to given byte length rather than character length since DB2 interprets + * string lengths in bytes. + * @see GalleryStorage::_truncateString + */ + function _truncateString($value, $size, $lengthInBytes=false) { + return parent::_truncateString($value, $size, true); + } } ?> Index: D:/eclipse_workspace/gallery2/modules/core/classes/GalleryUtilities.class =================================================================== --- D:/eclipse_workspace/gallery2/modules/core/classes/GalleryUtilities.class (revision 14982) +++ D:/eclipse_workspace/gallery2/modules/core/classes/GalleryUtilities.class (working copy) @@ -815,6 +815,80 @@ } /** + * A multibyte safe version of mb_strcut for UTF-8. + * @param string $string the input string containing raw UTF-8 + * @param int $start the byte-offset of the start position + * @param int $length the length in bytes (not in characters) + * @return string a multibyte safe substring of input value + */ + function utf8Strcut($string, $start, $length) { + static $hasMbStrCut; + if (!isset($hasMbStrCut)) { + $hasMbStrCut = function_exists('mb_strcut'); + } + + if ($hasMbStrCut) { + return (string) mb_strcut($string, $start, $length, 'UTF-8'); + } else { + return GalleryUtilities::_phpUtf8Strcut($string, $start, $length); + } + } + + /** + * A multibyte safe PHP implementation of mb_strcut for UTF-8. + * + * This is going to be a whole lot slower than mb_substr, so we should use that instead, + * whenever we can. + * + * @see utf8Strcut + */ + function _phpUtf8Strcut($string, $start, $length) { + $strLength = strlen($string); + $start = $start < 0 ? $strLength + $start : $start; + $start = $start < 0 ? 0 : $start; + $length = $length < 0 ? $strLength - $start + $length : $length; + $length = $length < 0 ? 0 : $length; + $length = ($start + $length) > $strLength ? $strLength - $start : $length; + + if ($start >= ($strLength - 1) || $strLength == 0 || $length == 0) { + return ''; + } + + /* 1. Jump to the $start byte and check if it's the start of a character. */ + if ($start != 0 && (ord($string[$start]) & 0xC0) == 0x80) { + /* It's the 2nd to 4th byte of a multibyte character, go back to the 1st byte of it */ + $j = 0; + /* Go back a maximum of 3 bytes and stop at the beginning of the string */ + while ($start > 0 && $j++ < 3 && (ord($string[$start]) & 0xC0) == 0x80) { + $start--; + } + } + + /* 2. Exract the multibyte unsafe substring starting at the modified $start */ + $result = substr($string, $start, $length); + /* 3. Keep the resulting string up to the last complete character */ + $end = $start + $length - 1; + if ($end >= ($strLength -1)) { + return $result; + } else if ((ord($string[$end + 1]) & 0xC0) != 0x80) { + /* The byte at $end is the last one in the character */ + return $result; + } else if (ord($string[$end]) & 0x80) { /* Is it a single-byte character? */ + /* It's an incomplete multibyte character, go back to the last character */ + $j = 0; + /* Go back a maximum of 3 bytes */ + while ($end > $start && $j++ < 3 && (ord($string[$end]) & 0xC0) == 0x80) { + $end--; + } + $end = $end > $start ? $end-- : $start; + + return substr($string, $start, $end - $start); + } + + return $result; + } + + /** * Convert a numerical unicode value to a multibyte UTF-8 string. Adapted from code found here: * http://us2.php.net/utf8_encode * Index: D:/eclipse_workspace/gallery2/modules/core/test/phpunit/StorageTest.class =================================================================== --- D:/eclipse_workspace/gallery2/modules/core/test/phpunit/StorageTest.class (revision 14982) +++ D:/eclipse_workspace/gallery2/modules/core/test/phpunit/StorageTest.class (working copy) @@ -40,6 +40,7 @@ $storage =& $gallery->getStorage(); $this->_db = $storage->_db; + $this->_serverInfo = isset($storage->_serverInfo) ? $storage->_serverInfo : null; $this->_isEmptyAllowedForNotNullColumn = $storage->_isEmptyAllowedForNotNullColumn; $storage->_db = new StorageTestDB($this); } @@ -49,6 +50,7 @@ $storage =& $gallery->getStorage(); $storage->_db = $this->_db; + $storage->_serverInfo = $this->_serverInfo; $storage->_isEmptyAllowedForNotNullColumn = $this->_isEmptyAllowedForNotNullColumn; parent::tearDown(); @@ -121,10 +123,6 @@ global $gallery; $storage =& $gallery->getStorage(); - /* - * We really should use a fake map, but it's hard to insert a fake map definition into - * GalleryStorage at the moment. - */ $ret = $storage->addMapEntry( 'ExternalIdMap', array('externalId' => str_repeat('x', 200), @@ -772,6 +770,59 @@ return $this->failWithStatus($ret); } } + + function testTruncateString() { + global $gallery; + $storage =& $gallery->getStorage(); + /* Use a real SQL connection for this test */ + $storage->_db = $this->_db; + $storage->_serverInfo = $this->_serverInfo; + + /* Test UTF-8 string with 5 characters / 10 bytes */ + $testString = "\xd0\xa0\xd1\x83\xd1\x81\xd1\x81\xd0\xba"; + /* Get a STRING-LARGE version of that string */ + $originalStringLarge = str_repeat($testString, 51); + $originalStringTooLarge = str_repeat($testString, 52); + $key = 'test' . rand(); + $itemId = -7; + $ret = GalleryCoreApi::addMapEntry('GalleryDerivativePreferencesMap', + array('itemId' => $itemId, + 'order' => 1, + 'derivativeType' => 1, + 'derivativeOperations' => $originalStringLarge)); + if ($ret) { + return $this->failWithStatus($ret); + } + + list ($ret, $results) = GalleryCoreApi::getMapEntry('GalleryDerivativePreferencesMap', + array('derivativeOperations'), + array('itemId' => $itemId)); + if ($ret) { + return $this->failWithStatus($ret); + } + + /* Cleanup */ + $ret = GalleryCoreApi::removeMapEntry('GalleryDerivativePreferencesMap', + array('itemId' => $itemId)); + if ($ret) { + return $this->failWithStatus($ret); + } + /* Commit our DB transaction to release locks from the PluginParameterMap table */ + $ret = $storage->checkPoint(); + if ($ret) { + return $this->failWithStatus($ret); + } + + $expected = $originalStringLarge; + if ($storage->getType() == 'db2' + || ($storage->getType() == 'mysql' && $this->_serverInfo + && version_compare($this->_serverInfo, '4.1.0', '<'))) { + $expected = GalleryUtilities::utf8Strcut($originalStringLarge, 0, 255); + } + + $this->assertEquals(1, $results->resultCount(), 'no / wrong rows selected'); + $this->assertEquals(array($expected), $results->nextResult(), 'inserted string'); + } } class StorageTestDB { Index: D:/eclipse_workspace/gallery2/modules/core/test/phpunit/UtilitiesTest.class =================================================================== --- D:/eclipse_workspace/gallery2/modules/core/test/phpunit/UtilitiesTest.class (revision 14982) +++ D:/eclipse_workspace/gallery2/modules/core/test/phpunit/UtilitiesTest.class (working copy) @@ -943,6 +943,58 @@ 'php version'); } + function testAssertMbFunctionOverloadingDisabled() { + /* Assert that we're not accidentaly using mb_ functions in our fallback */ + $mbFuncOverload = (int) ini_get('mbstring.func_overload'); + $this->assert(!($mbFuncOverload & 2), 'mb function overload str functions. ' . + 'Please disable mbstring.func_overload to test PHP fallback.'); + } + + function testUtf8Strcut() { + /* Russian */ + $utf8 = ("\xd0\xa0\xd1\x83\xd1\x81\xd1\x81\xd0\xba\xd0" . + "\xb8\xd0\xb9\xd0\x97\xd0\xb4\xd1\x80\xd0\xb0" . + "\xd0\xb2\xd1\x81\xd1\x82\xd0\xb2\xd1\x83"); + $this->assertEquals("\xd0\xa0\xd1\x83\xd1\x81\xd1\x81\xd0\xba", + $result = GalleryUtilities::utf8Strcut($utf8, 0, 10), + 'standard version'); + $this->assertEquals(10, strlen($result), 'length with standard version'); + $this->assertEquals("\xd0\xa0\xd1\x83\xd1\x81\xd1\x81\xd0\xba", + GalleryUtilities::_phpUtf8Strcut($utf8, 0, 10), + 'PHP version'); + + /* With byte offset */ + $this->assertEquals("\xd1\x81\xd1\x82\xd0\xb2", + GalleryUtilities::utf8Strcut($utf8, 24, 7), + 'standard version with offset'); + $this->assertEquals("\xd1\x81\xd1\x82\xd0\xb2", + GalleryUtilities::_phpUtf8Strcut($utf8, 24, 7), + 'PHP version with offset'); + + /* With byte offset and length matching total length */ + $this->assertEquals("\xd1\x81\xd1\x82\xd0\xb2\xd1\x83", + GalleryUtilities::utf8Strcut($utf8, 24, 8), + 'standard version with offset up to end'); + $this->assertEquals("\xd1\x81\xd1\x82\xd0\xb2\xd1\x83", + GalleryUtilities::_phpUtf8Strcut($utf8, 24, 8), + 'PHP version with offset up to end'); + + /* Start position larger than string */ + $this->assertEquals('', GalleryUtilities::utf8Strcut("short", 20, 5), + 'standard version with too large start position'); + $this->assertEquals('', GalleryUtilities::_phpUtf8Strcut("short", 20, 5), + 'PHP version with too large start position'); + + /* Test some special characters */ + $utf8 = 'apo\'strophe.jpg "quotes"'; + $this->assertEquals('apo\'strophe.jpg "quote', + GalleryUtilities::utf8Substring($utf8, 0, 22), + 'standard version special characters'); + $this->assertEquals('apo\'strophe.jpg "quote', + GalleryUtilities::_phpUtf8Substring($utf8, 0, 22), + 'php version special characters'); + } + function testIsEmbedded() { $key = 'G2_EMBED'; $backupValue = null;