From f335b55bdf7f669f930f7c6d06193c021fd92edb Mon Sep 17 00:00:00 2001 From: Travis Keep Date: Wed, 20 Aug 2014 21:46:02 +0000 Subject: [PATCH] ICU-10802 Merge branch tkeep/10802andy into trunk. X-SVN-Rev: 36214 --- .gitattributes | 3 + icu4c/source/common/Makefile.in | 2 +- icu4c/source/common/common.vcxproj | 4 + icu4c/source/common/common.vcxproj.filters | 6 + icu4c/source/common/sharedobject.cpp | 28 +- icu4c/source/common/sharedobject.h | 46 ++- icu4c/source/common/ucln_cmn.h | 9 +- icu4c/source/common/unifiedcache.cpp | 375 ++++++++++++++++++ icu4c/source/common/unifiedcache.h | 320 +++++++++++++++ icu4c/source/i18n/coll.cpp | 18 +- icu4c/source/i18n/collationbuilder.cpp | 8 +- icu4c/source/i18n/collationroot.cpp | 19 +- icu4c/source/i18n/collationroot.h | 2 + icu4c/source/i18n/collationtailoring.cpp | 4 + icu4c/source/i18n/collationtailoring.h | 13 + icu4c/source/i18n/measfmt.cpp | 52 +-- icu4c/source/i18n/numfmt.cpp | 47 +-- icu4c/source/i18n/plurrule.cpp | 56 +-- icu4c/source/i18n/reldatefmt.cpp | 49 +-- icu4c/source/i18n/rulebasedcollator.cpp | 40 +- icu4c/source/i18n/ucln_in.h | 3 - icu4c/source/i18n/ucol_imp.h | 65 ++- icu4c/source/i18n/ucol_res.cpp | 350 ++++++++++++---- icu4c/source/i18n/unicode/tblcoll.h | 8 +- icu4c/source/test/intltest/Makefile.in | 2 +- icu4c/source/test/intltest/intltest.vcxproj | 5 +- .../test/intltest/intltest.vcxproj.filters | 5 +- icu4c/source/test/intltest/itutil.cpp | 9 + icu4c/source/test/intltest/measfmttest.cpp | 2 + icu4c/source/test/intltest/tsmthred.cpp | 109 ++++- icu4c/source/test/intltest/tsmthred.h | 1 + .../source/test/intltest/unifiedcachetest.cpp | 173 ++++++++ 32 files changed, 1525 insertions(+), 308 deletions(-) create mode 100644 icu4c/source/common/unifiedcache.cpp create mode 100644 icu4c/source/common/unifiedcache.h create mode 100644 icu4c/source/test/intltest/unifiedcachetest.cpp diff --git a/.gitattributes b/.gitattributes index 3157b41a370..e6f7aa7e4ca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -53,6 +53,8 @@ icu4c/source/aclocal.m4 -text icu4c/source/allinone/icucheck.bat -text icu4c/source/common/common.vcxproj -text icu4c/source/common/common.vcxproj.filters -text +icu4c/source/common/unifiedcache.cpp -text +icu4c/source/common/unifiedcache.h -text icu4c/source/data/curr/pool.res -text icu4c/source/data/in/coll/ucadata-implicithan.icu -text icu4c/source/data/in/coll/ucadata-unihan.icu -text @@ -145,6 +147,7 @@ icu4c/source/test/depstest/icu-dependencies-mode.el -text icu4c/source/test/intltest/intltest.vcxproj -text icu4c/source/test/intltest/intltest.vcxproj.filters -text icu4c/source/test/intltest/numfmtspectest.cpp -text +icu4c/source/test/intltest/unifiedcachetest.cpp -text icu4c/source/test/iotest/iotest.vcxproj -text icu4c/source/test/iotest/iotest.vcxproj.filters -text icu4c/source/test/letest/cletest.vcxproj -text diff --git a/icu4c/source/common/Makefile.in b/icu4c/source/common/Makefile.in index 98d23d6a444..09b1bdf47fc 100644 --- a/icu4c/source/common/Makefile.in +++ b/icu4c/source/common/Makefile.in @@ -105,7 +105,7 @@ serv.o servnotf.o servls.o servlk.o servlkf.o servrbf.o servslkf.o \ uidna.o usprep.o uts46.o punycode.o \ util.o util_props.o parsepos.o locbased.o cwchar.o wintz.o dtintrv.o ucnvsel.o propsvec.o \ ulist.o uloc_tag.o icudataver.o icuplug.o listformatter.o lrucache.o \ -sharedobject.o simplepatternformatter.o +sharedobject.o simplepatternformatter.o unifiedcache.o ## Header files to install HEADERS = $(srcdir)/unicode/*.h diff --git a/icu4c/source/common/common.vcxproj b/icu4c/source/common/common.vcxproj index 9a1a5d10b7e..506964ce13f 100644 --- a/icu4c/source/common/common.vcxproj +++ b/icu4c/source/common/common.vcxproj @@ -400,6 +400,9 @@ + + false + @@ -1128,6 +1131,7 @@ ..\..\include\unicode\%(Filename)%(Extension);%(Outputs) + diff --git a/icu4c/source/common/common.vcxproj.filters b/icu4c/source/common/common.vcxproj.filters index 5082ac19ea5..82c8f13ebff 100644 --- a/icu4c/source/common/common.vcxproj.filters +++ b/icu4c/source/common/common.vcxproj.filters @@ -124,6 +124,9 @@ collections + + collections + collections @@ -636,6 +639,9 @@ collections + + collections + collections diff --git a/icu4c/source/common/sharedobject.cpp b/icu4c/source/common/sharedobject.cpp index bad79801d7d..6affcd09cd5 100644 --- a/icu4c/source/common/sharedobject.cpp +++ b/icu4c/source/common/sharedobject.cpp @@ -12,19 +12,41 @@ SharedObject::~SharedObject() {} void SharedObject::addRef() const { - umtx_atomic_inc(&refCount); + umtx_atomic_inc(&totalRefCount); } void SharedObject::removeRef() const { - if(umtx_atomic_dec(&refCount) == 0) { + if(umtx_atomic_dec(&totalRefCount) == 0) { delete this; } } +void +SharedObject::addSoftRef() const { + addRef(); + umtx_atomic_inc(&softRefCount); +} + +void +SharedObject::removeSoftRef() const { + umtx_atomic_dec(&softRefCount); + removeRef(); +} + +UBool +SharedObject::allSoftReferences() const { + return umtx_loadAcquire(totalRefCount) == umtx_loadAcquire(softRefCount); +} + int32_t SharedObject::getRefCount() const { - return umtx_loadAcquire(refCount); + return umtx_loadAcquire(totalRefCount); +} + +int32_t +SharedObject::getSoftRefCount() const { + return umtx_loadAcquire(softRefCount); } void diff --git a/icu4c/source/common/sharedobject.h b/icu4c/source/common/sharedobject.h index ed79a6a73e3..432c79ba0b9 100644 --- a/icu4c/source/common/sharedobject.h +++ b/icu4c/source/common/sharedobject.h @@ -26,11 +26,15 @@ U_NAMESPACE_BEGIN */ class U_COMMON_API SharedObject : public UObject { public: - /** Initializes refCount to 0. */ - SharedObject() : refCount(0) {} + /** Initializes totalRefCount, softRefCount to 0. */ + SharedObject() : totalRefCount(0), softRefCount(0) {} + + /** Initializes totalRefCount, softRefCount to 0. */ + SharedObject(const SharedObject &other) + : UObject(other), + totalRefCount(0), + softRefCount(0) {} - /** Initializes refCount to 0. */ - SharedObject(const SharedObject &other) : UObject(other), refCount(0) {} virtual ~SharedObject(); /** @@ -39,16 +43,41 @@ public: void addRef() const; /** - * Decrements the number of references to this object, - * and auto-deletes "this" if the number becomes 0. Thread-safe. + * Increments the number of soft references to this object. Thread-safe. + */ + void addSoftRef() const; + + /** + * Decrements the number of references to this object. Thread-safe. */ void removeRef() const; /** - * Returns the reference counter. Uses a memory barrier. + * Decrements the number of soft references to this object. Thread-safe. + */ + void removeSoftRef() const; + + /** + * Returns the reference counter including soft references. + * Uses a memory barrier. */ int32_t getRefCount() const; + /** + * Returns the count of soft references only. Uses a memory barrier. + * Used for testing the cache. Regular clients won't need this. + */ + int32_t getSoftRefCount() const; + + /** + * If allSoftReferences() == TRUE then this object has only soft + * references. The converse is not necessarily true. + */ + UBool allSoftReferences() const; + + /** + * Deletes this object if it has no references or soft references. + */ void deleteIfZeroRefCount() const; /** @@ -103,7 +132,8 @@ public: } private: - mutable u_atomic_int32_t refCount; + mutable u_atomic_int32_t totalRefCount; + mutable u_atomic_int32_t softRefCount; }; U_NAMESPACE_END diff --git a/icu4c/source/common/ucln_cmn.h b/icu4c/source/common/ucln_cmn.h index 208bc3cc78c..0e2abc6a520 100644 --- a/icu4c/source/common/ucln_cmn.h +++ b/icu4c/source/common/ucln_cmn.h @@ -37,7 +37,6 @@ typedef enum ECleanupCommonType { UCLN_COMMON_BREAKITERATOR, UCLN_COMMON_BREAKITERATOR_DICT, UCLN_COMMON_SERVICE, - UCLN_COMMON_URES, UCLN_COMMON_LOCALE, UCLN_COMMON_LOCALE_AVAILABLE, UCLN_COMMON_ULOC, @@ -51,6 +50,14 @@ typedef enum ECleanupCommonType { UCLN_COMMON_PUTIL, UCLN_COMMON_LIST_FORMATTER, UCLN_COMMON_UINIT, + + /* + Unified caches caches collation stuff. Collation data structures + contain resource bundles which means that unified cache cleanup + must happen before resource bundle clean up. + */ + UCLN_COMMON_UNIFIED_CACHE, + UCLN_COMMON_URES, UCLN_COMMON_COUNT /* This must be last */ } ECleanupCommonType; diff --git a/icu4c/source/common/unifiedcache.cpp b/icu4c/source/common/unifiedcache.cpp new file mode 100644 index 00000000000..7475cb8789e --- /dev/null +++ b/icu4c/source/common/unifiedcache.cpp @@ -0,0 +1,375 @@ +/* +****************************************************************************** +* Copyright (C) 2014, International Business Machines Corporation and +* others. All Rights Reserved. +****************************************************************************** +* +* File UNIFIEDCACHE.CPP +****************************************************************************** +*/ + +#include "uhash.h" +#include "unifiedcache.h" +#include "umutex.h" +#include "mutex.h" +#include "uassert.h" +#include "ucln_cmn.h" + +static icu::UnifiedCache *gCache = NULL; +static icu::SharedObject *gNoValue = NULL; +static UMutex gCacheMutex = U_MUTEX_INITIALIZER; +static UConditionVar gInProgressValueAddedCond = U_CONDITION_INITIALIZER; +static icu::UInitOnce gCacheInitOnce = U_INITONCE_INITIALIZER; + +U_CDECL_BEGIN +static UBool U_CALLCONV unifiedcache_cleanup() { + gCacheInitOnce.reset(); + if (gCache) { + delete gCache; + gCache = NULL; + } + if (gNoValue) { + delete gNoValue; + gNoValue = NULL; + } + return TRUE; +} +U_CDECL_END + + +U_NAMESPACE_BEGIN + +U_CAPI int32_t U_EXPORT2 +ucache_hashKeys(const UHashTok key) { + const CacheKeyBase *ckey = (const CacheKeyBase *) key.pointer; + return ckey->hashCode(); +} + +U_CAPI UBool U_EXPORT2 +ucache_compareKeys(const UHashTok key1, const UHashTok key2) { + const CacheKeyBase *p1 = (const CacheKeyBase *) key1.pointer; + const CacheKeyBase *p2 = (const CacheKeyBase *) key2.pointer; + return *p1 == *p2; +} + +U_CAPI void U_EXPORT2 +ucache_deleteKey(void *obj) { + CacheKeyBase *p = (CacheKeyBase *) obj; + delete p; +} + +CacheKeyBase::~CacheKeyBase() { +} + +static void U_CALLCONV cacheInit(UErrorCode &status) { + U_ASSERT(gCache == NULL); + ucln_common_registerCleanup( + UCLN_COMMON_UNIFIED_CACHE, unifiedcache_cleanup); + + // gNoValue must be created first to avoid assertion error in + // cache constructor. + gNoValue = new SharedObject(); + gCache = new UnifiedCache(status); + if (gCache == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } + if (U_FAILURE(status)) { + delete gCache; + delete gNoValue; + gCache = NULL; + gNoValue = NULL; + return; + } + // We add a softref because we want hash elements with gNoValue to be + // elligible for purging but we don't ever want gNoValue to be deleted. + gNoValue->addSoftRef(); +} + +const UnifiedCache *UnifiedCache::getInstance(UErrorCode &status) { + umtx_initOnce(gCacheInitOnce, &cacheInit, status); + if (U_FAILURE(status)) { + return NULL; + } + U_ASSERT(gCache != NULL); + return gCache; +} + +UnifiedCache::UnifiedCache(UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + U_ASSERT(gNoValue != NULL); + fHashtable = uhash_open( + &ucache_hashKeys, + &ucache_compareKeys, + NULL, + &status); + if (U_FAILURE(status)) { + return; + } + uhash_setKeyDeleter(fHashtable, &ucache_deleteKey); +} + +int32_t UnifiedCache::keyCount() const { + Mutex lock(&gCacheMutex); + return uhash_count(fHashtable); +} + +void UnifiedCache::flush() const { + Mutex lock(&gCacheMutex); + + // Use a loop in case cache items that are flushed held hard references to + // other cache items making those additional cache items eligible for + // flushing. + while (_flush(FALSE)); + umtx_condBroadcast(&gInProgressValueAddedCond); +} + +void UnifiedCache::dump() { + UErrorCode status = U_ZERO_ERROR; + const UnifiedCache *cache = getInstance(status); + if (U_FAILURE(status)) { + fprintf(stderr, "Unified Cache: Error fetching cache.\n"); + return; + } + cache->dumpContents(); +} + +void UnifiedCache::dumpContents() const { + Mutex lock(&gCacheMutex); + _dumpContents(); +} + +// Dumps content of cache. +// On entry, gCacheMutex must be held. +// On exit, cache contents dumped to stderr. +void UnifiedCache::_dumpContents() const { + int32_t pos = -1; + const UHashElement *element = uhash_nextElement(fHashtable, &pos); + char buffer[256]; + int32_t cnt = 0; + for (; element != NULL; element = uhash_nextElement(fHashtable, &pos)) { + const SharedObject *sharedObject = + (const SharedObject *) element->value.pointer; + const CacheKeyBase *key = + (const CacheKeyBase *) element->key.pointer; + if (!sharedObject->allSoftReferences()) { + ++cnt; + fprintf( + stderr, + "Unified Cache: Key '%s', error %d, value %p, total refcount %d, soft refcount %d\n", + key->writeDescription(buffer, 256), + key->creationStatus, + sharedObject == gNoValue ? NULL :sharedObject, + sharedObject->getRefCount(), + sharedObject->getSoftRefCount()); + } + } + fprintf(stderr, "Unified Cache: %d out of a total of %d still have hard references\n", cnt, uhash_count(fHashtable)); +} + +UnifiedCache::~UnifiedCache() { + // Try our best to clean up first. + flush(); + { + // Now all that should be left in the cache are entries that refer to + // each other and entries with hard references from outside the cache. + // Nothing we can do about these so proceed to wipe out the cache. + Mutex lock(&gCacheMutex); + _flush(TRUE); + } + uhash_close(fHashtable); +} + +// Flushes the contents of the cache. If cache values hold references to other +// cache values then _flush should be called in a loop until it returns FALSE. +// On entry, gCacheMutex must be held. +// On exit, those values with only soft references are flushed. If all is true +// then every value is flushed even if hard references are held. +// Returns TRUE if any value in cache was flushed or FALSE otherwise. +UBool UnifiedCache::_flush(UBool all) const { + UBool result = FALSE; + int32_t pos = -1; + const UHashElement *element = uhash_nextElement(fHashtable, &pos); + for (; element != NULL; element = uhash_nextElement(fHashtable, &pos)) { + const SharedObject *sharedObject = + (const SharedObject *) element->value.pointer; + if (all || sharedObject->allSoftReferences()) { + uhash_removeElement(fHashtable, element); + sharedObject->removeSoftRef(); + result = TRUE; + } + } + return result; +} + +// Places a new value and creationStatus in the cache for the given key. +// On entry, gCacheMutex must be held. key must not exist in the cache. +// On exit, value and creation status placed under key. Soft reference added +// to value on successful add. On error sets status. +void UnifiedCache::_putNew( + const CacheKeyBase &key, + const SharedObject *value, + const UErrorCode creationStatus, + UErrorCode &status) const { + if (U_FAILURE(status)) { + return; + } + CacheKeyBase *keyToAdopt = key.clone(); + if (keyToAdopt == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + keyToAdopt->creationStatus = creationStatus; + uhash_put(fHashtable, keyToAdopt, (void *) value, &status); + if (U_SUCCESS(status)) { + value->addSoftRef(); + } +} + +// Places value and status at key if there is no value at key or if cache +// entry for key is in progress. Otherwise, it leaves the current value and +// status there. +// On entry. gCacheMutex must not be held. value must be +// included in the reference count of the object to which it points. +// On exit, value and status are changed to what was already in the cache if +// something was there and not in progress. Otherwise, value and status are left +// unchanged in which case they are placed in the cache on a best-effort basis. +// Caller must call removeRef() on value. +void UnifiedCache::_putIfAbsentAndGet( + const CacheKeyBase &key, + const SharedObject *&value, + UErrorCode &status) const { + Mutex lock(&gCacheMutex); + const UHashElement *element = uhash_find(fHashtable, &key); + if (element != NULL && !_inProgress(element)) { + _fetch(element, value, status); + return; + } + if (element == NULL) { + UErrorCode putError = U_ZERO_ERROR; + // best-effort basis only. + _putNew(key, value, status, putError); + return; + } + _put(element, value, status); +} + +// Attempts to fetch value and status for key from cache. +// On entry, gCacheMutex must not be held value must be NULL and status must +// be U_ZERO_ERROR. +// On exit, either returns FALSE (In this +// case caller should try to create the object) or returns TRUE with value +// pointing to the fetched value and status set to fetched status. When +// FALSE is returned status may be set to failure if an in progress hash +// entry could not be made but value will remain unchanged. When TRUE is +// returned, caler must call removeRef() on value. +UBool UnifiedCache::_poll( + const CacheKeyBase &key, + const SharedObject *&value, + UErrorCode &status) const { + U_ASSERT(value == NULL); + U_ASSERT(status == U_ZERO_ERROR); + Mutex lock(&gCacheMutex); + const UHashElement *element = uhash_find(fHashtable, &key); + while (element != NULL && _inProgress(element)) { + umtx_condWait(&gInProgressValueAddedCond, &gCacheMutex); + element = uhash_find(fHashtable, &key); + } + if (element != NULL) { + _fetch(element, value, status); + return TRUE; + } + _putNew(key, gNoValue, U_ZERO_ERROR, status); + return FALSE; +} + +// Gets value out of cache. +// On entry. gCacheMutex must not be held. value must be NULL. status +// must be U_ZERO_ERROR. +// On exit. value and status set to what is in cache at key or on cache +// miss the key's createObject() is called and value and status are set to +// the result of that. In this latter case, best effort is made to add the +// value and status to the cache. value will be set to NULL instead of +// gNoValue. Caller must call removeRef on value if non NULL. +void UnifiedCache::_get( + const CacheKeyBase &key, + const SharedObject *&value, + const void *creationContext, + UErrorCode &status) const { + U_ASSERT(value == NULL); + U_ASSERT(status == U_ZERO_ERROR); + if (_poll(key, value, status)) { + if (value == gNoValue) { + SharedObject::clearPtr(value); + } + return; + } + if (U_FAILURE(status)) { + return; + } + value = key.createObject(creationContext, status); + U_ASSERT(value == NULL || !value->allSoftReferences()); + U_ASSERT(value != NULL || status != U_ZERO_ERROR); + if (value == NULL) { + SharedObject::copyPtr(gNoValue, value); + } + _putIfAbsentAndGet(key, value, status); + if (value == gNoValue) { + SharedObject::clearPtr(value); + } +} + +// Store a value and error in given hash entry. +// On entry, gCacheMutex must be held. Hash entry element must be in progress. +// value must be non NULL. +// On Exit, soft reference added to value. value and status stored in hash +// entry. Soft reference removed from previous stored value. Waiting +// threads notified. +void UnifiedCache::_put( + const UHashElement *element, + const SharedObject *value, + const UErrorCode status) { + U_ASSERT(_inProgress(element)); + const CacheKeyBase *theKey = (const CacheKeyBase *) element->key.pointer; + const SharedObject *oldValue = (const SharedObject *) element->value.pointer; + theKey->creationStatus = status; + value->addSoftRef(); + UHashElement *ptr = const_cast(element); + ptr->value.pointer = (void *) value; + oldValue->removeSoftRef(); + + // Tell waiting threads that we replace in-progress status with + // an error. + umtx_condBroadcast(&gInProgressValueAddedCond); +} + +// Fetch value and error code from a particular hash entry. +// On entry, gCacheMutex must be held. value must be either NULL or must be +// included in the ref count of the object to which it points. +// On exit, value and status set to what is in the hash entry. Caller must +// eventually call removeRef on value. +// If hash entry is in progress, value will be set to gNoValue and status will +// be set to U_ZERO_ERROR. +void UnifiedCache::_fetch( + const UHashElement *element, + const SharedObject *&value, + UErrorCode &status) { + const CacheKeyBase *theKey = (const CacheKeyBase *) element->key.pointer; + status = theKey->creationStatus; + SharedObject::copyPtr( + (const SharedObject *) element->value.pointer, value); +} + +// Determine if given hash entry is in progress. +// On entry, gCacheMutex must be held. +UBool UnifiedCache::_inProgress(const UHashElement *element) { + const SharedObject *value = NULL; + UErrorCode status = U_ZERO_ERROR; + _fetch(element, value, status); + UBool result = (value == gNoValue && status == U_ZERO_ERROR); + SharedObject::clearPtr(value); + return result; +} + +U_NAMESPACE_END diff --git a/icu4c/source/common/unifiedcache.h b/icu4c/source/common/unifiedcache.h new file mode 100644 index 00000000000..61609cf125f --- /dev/null +++ b/icu4c/source/common/unifiedcache.h @@ -0,0 +1,320 @@ +/* +****************************************************************************** +* Copyright (C) 2014, International Business Machines Corporation and +* others. All Rights Reserved. +****************************************************************************** +* +* File UNIFIEDCACHE.H - The ICU Unified cache. +****************************************************************************** +*/ + +#ifndef __UNIFIED_CACHE_H__ +#define __UNIFIED_CACHE_H__ + +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/uobject.h" +#include "unicode/locid.h" +#include "sharedobject.h" +#include "unicode/unistr.h" +#include "cstring.h" +#include "ustr_imp.h" + +struct UHashtable; +struct UHashElement; + +U_NAMESPACE_BEGIN + +class UnifiedCache; + +/** + * A base class for all cache keys + */ +class U_COMMON_API CacheKeyBase : public UObject { + public: + CacheKeyBase() : creationStatus(U_ZERO_ERROR) {} + + /** + * Copy constructor. Needed to support cloning. + */ + CacheKeyBase(const CacheKeyBase &other) + : creationStatus(other.creationStatus) { } + virtual ~CacheKeyBase(); + + /** + * Returns the hash code for this object. + */ + virtual int32_t hashCode() const = 0; + + /** + * Clones this object polymorphically. Caller owns returned value. + */ + virtual CacheKeyBase *clone() const = 0; + + /** + * Equality operator. + */ + virtual UBool operator == (const CacheKeyBase &other) const = 0; + + /** + * Create a new object for this key. Called by cache on cache miss. + * createObject must add a reference to the object it returns. Note + * that getting an object from the cache and returning it without calling + * removeRef on it satisfies this requirement. It can also return NULL + * and set status to an error. + * + * @param creationContext the context in which the object is being + * created. May be NULL. + * @param status Implementations can return a failure here. + * In addition, implementations may return a + * non NULL object and set a warning status. + */ + virtual const SharedObject *createObject( + const void *creationContext, UErrorCode &status) const = 0; + + /** + * Writes a description of this key to buffer and returns buffer. Written + * description is NULL terminated. + */ + virtual char *writeDescription(char *buffer, int32_t bufSize) const = 0; + + /** + * Inequality operator. + */ + UBool operator != (const CacheKeyBase &other) const { + return !(*this == other); + } + private: + mutable UErrorCode creationStatus; + friend class UnifiedCache; +}; + + + +/** + * Templated version of CacheKeyBase. + * A key of type LocaleCacheKey maps to a value of type T. + */ +template +class CacheKey : public CacheKeyBase { + public: + virtual ~CacheKey() { } + /** + * The template parameter, T, determines the hash code returned. + */ + virtual int32_t hashCode() const { + const char *s = typeid(T).name(); + return ustr_hashCharsN(s, uprv_strlen(s)); + } + + /** + * Use the value type, T, as the description. + */ + virtual char *writeDescription(char *buffer, int32_t bufLen) const { + const char *s = typeid(T).name(); + uprv_strncpy(buffer, s, bufLen); + buffer[bufLen - 1] = 0; + return buffer; + } + + /** + * Two objects are equal if they are of the same type. + */ + virtual UBool operator == (const CacheKeyBase &other) const { + return typeid(*this) == typeid(other); + } +}; + +/** + * Cache key based on locale. + * A key of type LocaleCacheKey maps to a value of type T. + */ +template +class LocaleCacheKey : public CacheKey { + protected: + Locale fLoc; + public: + LocaleCacheKey(const Locale &loc) : fLoc(loc) {}; + virtual ~LocaleCacheKey() { } + virtual int32_t hashCode() const { + return 37 *CacheKey::hashCode() + fLoc.hashCode(); + } + virtual UBool operator == (const CacheKeyBase &other) const { + // reflexive + if (this == &other) { + return TRUE; + } + if (!CacheKey::operator == (other)) { + return FALSE; + } + // We know this and other are of same class because operator== on + // CacheKey returned true. + const LocaleCacheKey *fOther = + static_cast *>(&other); + return fLoc == fOther->fLoc; + } + virtual CacheKeyBase *clone() const { + return new LocaleCacheKey(*this); + } + virtual const T *createObject( + const void *creationContext, UErrorCode &status) const; + /** + * Use the locale id as the description. + */ + virtual char *writeDescription(char *buffer, int32_t bufLen) const { + const char *s = fLoc.getName(); + uprv_strncpy(buffer, s, bufLen); + buffer[bufLen - 1] = 0; + return buffer; + } + +}; + +/** + * The unified cache. A singleton type. + */ +class U_COMMON_API UnifiedCache : public UObject { + public: + /** + * @internal + */ + UnifiedCache(UErrorCode &status); + + /** + * Returns the cache instance. + */ + static const UnifiedCache *getInstance(UErrorCode &status); + + /** + * Fetches a value from the cache by key. Equivalent to + * get(key, NULL, ptr, status); + */ + template + void get( + const CacheKey& key, + const T *&ptr, + UErrorCode &status) const { + get(key, NULL, ptr, status); + } + + /** + * Fetches value from the cache by key. + * + * @param key the cache key. + * @param creationContext passed verbatim to createObject method of key + * @param ptr On entry, ptr must be NULL or be included if + * the reference count of the object it points + * to. On exit, ptr points to the fetched object + * from the cache or is left unchanged on + * failure. Caller must call removeRef on ptr + * if set to a non NULL value. + * @param status Any error returned here. May be set to a + * warning value even if ptr is set. + */ + template + void get( + const CacheKey& key, + const void *creationContext, + const T *&ptr, + UErrorCode &status) const { + if (U_FAILURE(status)) { + return; + } + UErrorCode creationStatus = U_ZERO_ERROR; + const SharedObject *value = NULL; + _get(key, value, creationContext, creationStatus); + const T *tvalue = (const T *) value; + if (U_SUCCESS(creationStatus)) { + SharedObject::copyPtr(tvalue, ptr); + } + SharedObject::clearPtr(tvalue); + // Take care not to overwrite a warning status passed in with + // another warning or U_ZERO_ERROR. + if (status == U_ZERO_ERROR || U_FAILURE(creationStatus)) { + status = creationStatus; + } + } + + /** + * Dumps the contents of this cache to standard error. Used for testing of + * cache only. + */ + void dumpContents() const; + + /** + * Convenience method to get a value of type T from cache for a + * particular locale with creationContext == NULL. + * @param loc the locale + * @param ptr On entry, must be NULL or included in the ref count + * of the object to which it points. + * On exit, fetched value stored here or is left + * unchanged on failure. Caller must call removeRef on + * ptr if set to a non NULL value. + * @param status Any error returned here. May be set to a + * warning value even if ptr is set. + */ + template + static void getByLocale( + const Locale &loc, const T *&ptr, UErrorCode &status) { + const UnifiedCache *cache = getInstance(status); + if (U_FAILURE(status)) { + return; + } + cache->get(LocaleCacheKey(loc), ptr, status); + } + + /** + * Dumps the cache contents to stderr. For testing only. + */ + static void dump(); + + /** + * Returns the number of keys in this cache. For testing only. + */ + int32_t keyCount() const; + + /** + * Removes any values from cache that are not referenced outside + * the cache. + */ + void flush() const; + + virtual ~UnifiedCache(); + private: + UHashtable *fHashtable; + UnifiedCache(const UnifiedCache &other); + UnifiedCache &operator=(const UnifiedCache &other); + UBool _flush(UBool all) const; + void _get( + const CacheKeyBase &key, + const SharedObject *&value, + const void *creationContext, + UErrorCode &status) const; + UBool _poll( + const CacheKeyBase &key, + const SharedObject *&value, + UErrorCode &status) const; + void _putNew( + const CacheKeyBase &key, + const SharedObject *value, + const UErrorCode creationStatus, + UErrorCode &status) const; + void _putIfAbsentAndGet( + const CacheKeyBase &key, + const SharedObject *&value, + UErrorCode &status) const; + void _dumpContents() const; + static void _put( + const UHashElement *element, + const SharedObject *value, + const UErrorCode status); + static void _fetch( + const UHashElement *element, + const SharedObject *&value, + UErrorCode &status); + static UBool _inProgress(const UHashElement *element); +}; + +U_NAMESPACE_END + +#endif diff --git a/icu4c/source/i18n/coll.cpp b/icu4c/source/i18n/coll.cpp index 0edcd9a0696..b71bc962ef9 100644 --- a/icu4c/source/i18n/coll.cpp +++ b/icu4c/source/i18n/coll.cpp @@ -456,21 +456,21 @@ Collator* U_EXPORT2 Collator::createInstance(const Locale& desiredLocale, } -Collator* Collator::makeInstance(const Locale& desiredLocale, - UErrorCode& status) -{ - Locale validLocale(""); - const CollationTailoring *t = - CollationLoader::loadTailoring(desiredLocale, validLocale, status); +Collator* Collator::makeInstance(const Locale& desiredLocale, UErrorCode& status) { + const CollationCacheEntry *entry = CollationLoader::loadTailoring(desiredLocale, status); if (U_SUCCESS(status)) { - Collator *result = new RuleBasedCollator(t, validLocale); + Collator *result = new RuleBasedCollator(entry); if (result != NULL) { + // Both the unified cache's get() and the RBC constructor + // did addRef(). Undo one of them. + entry->removeRef(); return result; } status = U_MEMORY_ALLOCATION_ERROR; } - if (t != NULL) { - t->deleteIfZeroRefCount(); + if (entry != NULL) { + // Undo the addRef() from the cache.get(). + entry->removeRef(); } return NULL; } diff --git a/icu4c/source/i18n/collationbuilder.cpp b/icu4c/source/i18n/collationbuilder.cpp index 81f32770b8d..197496da735 100644 --- a/icu4c/source/i18n/collationbuilder.cpp +++ b/icu4c/source/i18n/collationbuilder.cpp @@ -86,6 +86,7 @@ RuleBasedCollator::RuleBasedCollator() : data(NULL), settings(NULL), tailoring(NULL), + cacheEntry(NULL), validLocale(""), explicitlySetAttributes(0), actualLocaleIsSameAsValid(FALSE) { @@ -95,6 +96,7 @@ RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, UErrorCode &err : data(NULL), settings(NULL), tailoring(NULL), + cacheEntry(NULL), validLocale(""), explicitlySetAttributes(0), actualLocaleIsSameAsValid(FALSE) { @@ -106,6 +108,7 @@ RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, ECollationStren : data(NULL), settings(NULL), tailoring(NULL), + cacheEntry(NULL), validLocale(""), explicitlySetAttributes(0), actualLocaleIsSameAsValid(FALSE) { @@ -118,6 +121,7 @@ RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, : data(NULL), settings(NULL), tailoring(NULL), + cacheEntry(NULL), validLocale(""), explicitlySetAttributes(0), actualLocaleIsSameAsValid(FALSE) { @@ -131,6 +135,7 @@ RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, : data(NULL), settings(NULL), tailoring(NULL), + cacheEntry(NULL), validLocale(""), explicitlySetAttributes(0), actualLocaleIsSameAsValid(FALSE) { @@ -143,6 +148,7 @@ RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, : data(NULL), settings(NULL), tailoring(NULL), + cacheEntry(NULL), validLocale(""), explicitlySetAttributes(0), actualLocaleIsSameAsValid(FALSE) { @@ -172,7 +178,7 @@ RuleBasedCollator::internalBuildTailoring(const UnicodeString &rules, return; } t->actualLocale.setToBogus(); - adoptTailoring(t.orphan()); + adoptTailoring(t.orphan(), errorCode); // Set attributes after building the collator, // to keep the default settings consistent with the rule string. if(strength != UCOL_DEFAULT) { diff --git a/icu4c/source/i18n/collationroot.cpp b/icu4c/source/i18n/collationroot.cpp index b5d8b49bcf3..749f05c51c3 100644 --- a/icu4c/source/i18n/collationroot.cpp +++ b/icu4c/source/i18n/collationroot.cpp @@ -30,7 +30,7 @@ U_NAMESPACE_BEGIN namespace { -static const CollationTailoring *rootSingleton = NULL; +static const CollationCacheEntry *rootSingleton = NULL; static UInitOnce initOnce = U_INITONCE_INITIALIZER; } // namespace @@ -61,15 +61,26 @@ CollationRoot::load(UErrorCode &errorCode) { CollationDataReader::read(NULL, inBytes, udata_getLength(t->memory), *t, errorCode); if(U_FAILURE(errorCode)) { return; } ucln_i18n_registerCleanup(UCLN_I18N_COLLATION_ROOT, uprv_collation_root_cleanup); - t->addRef(); // The rootSingleton takes ownership. - rootSingleton = t.orphan(); + CollationCacheEntry *entry = new CollationCacheEntry(Locale::getRoot(), t.getAlias()); + if(entry != NULL) { + t.orphan(); // The rootSingleton took ownership of the tailoring. + entry->addRef(); + rootSingleton = entry; + } +} + +const CollationCacheEntry * +CollationRoot::getRootCacheEntry(UErrorCode &errorCode) { + umtx_initOnce(initOnce, CollationRoot::load, errorCode); + if(U_FAILURE(errorCode)) { return NULL; } + return rootSingleton; } const CollationTailoring * CollationRoot::getRoot(UErrorCode &errorCode) { umtx_initOnce(initOnce, CollationRoot::load, errorCode); if(U_FAILURE(errorCode)) { return NULL; } - return rootSingleton; + return rootSingleton->tailoring; } const CollationData * diff --git a/icu4c/source/i18n/collationroot.h b/icu4c/source/i18n/collationroot.h index 576d62067fb..345fbe77eb4 100644 --- a/icu4c/source/i18n/collationroot.h +++ b/icu4c/source/i18n/collationroot.h @@ -18,6 +18,7 @@ U_NAMESPACE_BEGIN +struct CollationCacheEntry; struct CollationData; struct CollationSettings; struct CollationTailoring; @@ -27,6 +28,7 @@ struct CollationTailoring; */ class U_I18N_API CollationRoot { // purely static public: + static const CollationCacheEntry *getRootCacheEntry(UErrorCode &errorCode); static const CollationTailoring *getRoot(UErrorCode &errorCode); static const CollationData *getData(UErrorCode &errorCode); static const CollationSettings *getSettings(UErrorCode &errorCode); diff --git a/icu4c/source/i18n/collationtailoring.cpp b/icu4c/source/i18n/collationtailoring.cpp index 666b07b2d29..caa0c082d07 100644 --- a/icu4c/source/i18n/collationtailoring.cpp +++ b/icu4c/source/i18n/collationtailoring.cpp @@ -101,6 +101,10 @@ CollationTailoring::getUCAVersion() const { return ((int32_t)version[1] << 4) | (version[2] >> 6); } +CollationCacheEntry::~CollationCacheEntry() { + SharedObject::clearPtr(tailoring); +} + U_NAMESPACE_END #endif // !UCONFIG_NO_COLLATION diff --git a/icu4c/source/i18n/collationtailoring.h b/icu4c/source/i18n/collationtailoring.h index e81c4037f2f..2a9b3d566f6 100644 --- a/icu4c/source/i18n/collationtailoring.h +++ b/icu4c/source/i18n/collationtailoring.h @@ -90,6 +90,19 @@ private: CollationTailoring(const CollationTailoring &other); }; +struct CollationCacheEntry : public SharedObject { + CollationCacheEntry(const Locale &loc, const CollationTailoring *t) + : validLocale(loc), tailoring(t) { + if(t != NULL) { + t->addRef(); + } + } + ~CollationCacheEntry(); + + Locale validLocale; + const CollationTailoring *tailoring; +}; + U_NAMESPACE_END #endif // !UCONFIG_NO_COLLATION diff --git a/icu4c/source/i18n/measfmt.cpp b/icu4c/source/i18n/measfmt.cpp index 547f7c312e0..82d76e79964 100644 --- a/icu4c/source/i18n/measfmt.cpp +++ b/icu4c/source/i18n/measfmt.cpp @@ -20,7 +20,6 @@ #include "quantityformatter.h" #include "unicode/plurrule.h" #include "unicode/decimfmt.h" -#include "lrucache.h" #include "uresimp.h" #include "unicode/ures.h" #include "cstring.h" @@ -34,26 +33,12 @@ #include "sharednumberformat.h" #include "sharedpluralrules.h" +#include "unifiedcache.h" #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) #define MEAS_UNIT_COUNT 46 #define WIDTH_INDEX_COUNT (UMEASFMT_WIDTH_NARROW + 1) -static icu::LRUCache *gCache = NULL; -static UMutex gCacheMutex = U_MUTEX_INITIALIZER; -static icu::UInitOnce gCacheInitOnce = U_INITONCE_INITIALIZER; - -U_CDECL_BEGIN -static UBool U_CALLCONV measfmt_cleanup() { - gCacheInitOnce.reset(); - if (gCache) { - delete gCache; - gCache = NULL; - } - return TRUE; -} -U_CDECL_END - U_NAMESPACE_BEGIN UOBJECT_DEFINE_RTTI_IMPLEMENTATION(MeasureFormat) @@ -303,9 +288,10 @@ static NumericDateFormatters *loadNumericDateFormatters( return result; } -// Creates the MeasureFormatCacheData for a particular locale -static SharedObject *U_CALLCONV createData( - const char *localeId, UErrorCode &status) { +template<> U_I18N_API +const MeasureFormatCacheData *LocaleCacheKey::createObject( + const void * /*unused*/, UErrorCode &status) const { + const char *localeId = fLoc.getName(); LocalUResourceBundlePointer topLevel(ures_open(NULL, localeId, &status)); static UNumberFormatStyle currencyStyles[] = { UNUM_CURRENCY_PLURAL, UNUM_CURRENCY_ISO, UNUM_CURRENCY}; @@ -347,33 +333,10 @@ static SharedObject *U_CALLCONV createData( decfmt->setRoundingMode(DecimalFormat::kRoundDown); } result->adoptIntegerFormat(inf); + result->addRef(); return result.orphan(); } -static void U_CALLCONV cacheInit(UErrorCode &status) { - U_ASSERT(gCache == NULL); - U_ASSERT(MeasureUnit::getIndexCount() == MEAS_UNIT_COUNT); - ucln_i18n_registerCleanup(UCLN_I18N_MEASFMT, measfmt_cleanup); - gCache = new SimpleLRUCache(100, &createData, status); - if (U_FAILURE(status)) { - delete gCache; - gCache = NULL; - } -} - -static UBool getFromCache( - const char *locale, - const MeasureFormatCacheData *&ptr, - UErrorCode &status) { - umtx_initOnce(gCacheInitOnce, &cacheInit, status); - if (U_FAILURE(status)) { - return FALSE; - } - Mutex lock(&gCacheMutex); - gCache->get(locale, ptr, status); - return U_SUCCESS(status); -} - static UBool isTimeUnit(const MeasureUnit &mu, const char *tu) { return uprv_strcmp(mu.getType(), "duration") == 0 && uprv_strcmp(mu.getSubtype(), tu) == 0; @@ -637,7 +600,8 @@ void MeasureFormat::initMeasureFormat( const char *name = locale.getName(); setLocaleIDs(name, name); - if (!getFromCache(name, cache, status)) { + UnifiedCache::getByLocale(locale, cache, status); + if (U_FAILURE(status)) { return; } diff --git a/icu4c/source/i18n/numfmt.cpp b/icu4c/source/i18n/numfmt.cpp index ac3bc559da4..ed4239650cc 100644 --- a/icu4c/source/i18n/numfmt.cpp +++ b/icu4c/source/i18n/numfmt.cpp @@ -52,7 +52,7 @@ #include "digitlst.h" #include #include "sharednumberformat.h" -#include "lrucache.h" +#include "unifiedcache.h" //#define FMT_DEBUG @@ -148,10 +148,6 @@ static const char *gFormatKeys[UNUM_FORMAT_STYLE_COUNT] = { "currencyFormat" // UNUM_CASH_CURRENCY }; -static icu::LRUCache *gNumberFormatCache = NULL; -static UMutex gNumberFormatCacheMutex = U_MUTEX_INITIALIZER; -static icu::UInitOnce gNumberFormatCacheInitOnce = U_INITONCE_INITIALIZER; - // Static hashtable cache of NumberingSystem objects used by NumberFormat static UHashtable * NumberingSystem_cache = NULL; static UMutex nscacheMutex = U_MUTEX_INITIALIZER; @@ -185,11 +181,6 @@ static UBool U_CALLCONV numfmt_cleanup(void) { uhash_close(NumberingSystem_cache); NumberingSystem_cache = NULL; } - gNumberFormatCacheInitOnce.reset(); - if (gNumberFormatCache) { - delete gNumberFormatCache; - gNumberFormatCache = NULL; - } return TRUE; } U_CDECL_END @@ -1243,47 +1234,25 @@ static void U_CALLCONV nscacheInit() { uhash_setValueDeleter(NumberingSystem_cache, deleteNumberingSystem); } -static SharedObject *U_CALLCONV createSharedNumberFormat( - const char *localeId, UErrorCode &status) { - if (U_FAILURE(status)) { - return NULL; - } +template<> U_I18N_API +const SharedNumberFormat *LocaleCacheKey::createObject( + const void * /*unused*/, UErrorCode &status) const { + const char *localeId = fLoc.getName(); NumberFormat *nf = NumberFormat::internalCreateInstance( localeId, UNUM_DECIMAL, status); if (U_FAILURE(status)) { return NULL; } - SharedObject *result = new SharedNumberFormat(nf); + SharedNumberFormat *result = new SharedNumberFormat(nf); if (result == NULL) { status = U_MEMORY_ALLOCATION_ERROR; delete nf; return NULL; } + result->addRef(); return result; } -static void U_CALLCONV numberFormatCacheInit(UErrorCode &status) { - U_ASSERT(gNumberFormatCache == NULL); - ucln_i18n_registerCleanup(UCLN_I18N_NUMFMT, numfmt_cleanup); - gNumberFormatCache = new SimpleLRUCache(100, &createSharedNumberFormat, status); - if (U_FAILURE(status)) { - delete gNumberFormatCache; - gNumberFormatCache = NULL; - } -} - -static void getSharedNumberFormatFromCache( - const char *locale, - const SharedNumberFormat *&ptr, - UErrorCode &status) { - umtx_initOnce(gNumberFormatCacheInitOnce, &numberFormatCacheInit, status); - if (U_FAILURE(status)) { - return; - } - Mutex lock(&gNumberFormatCacheMutex); - gNumberFormatCache->get(locale, ptr, status); -} - const SharedNumberFormat* U_EXPORT2 NumberFormat::createSharedInstance(const Locale& loc, UNumberFormatStyle kind, UErrorCode& status) { if (U_FAILURE(status)) { @@ -1294,7 +1263,7 @@ NumberFormat::createSharedInstance(const Locale& loc, UNumberFormatStyle kind, U return NULL; } const SharedNumberFormat *result = NULL; - getSharedNumberFormatFromCache(loc.getName(), result, status); + UnifiedCache::getByLocale(loc, result, status); return result; } diff --git a/icu4c/source/i18n/plurrule.cpp b/icu4c/source/i18n/plurrule.cpp index 2984bc19f18..ab9fba45140 100644 --- a/icu4c/source/i18n/plurrule.cpp +++ b/icu4c/source/i18n/plurrule.cpp @@ -30,25 +30,10 @@ #include "uassert.h" #include "uvectr32.h" #include "sharedpluralrules.h" -#include "lrucache.h" +#include "unifiedcache.h" #if !UCONFIG_NO_FORMATTING -static icu::LRUCache *gPluralRulesCache = NULL; -static UMutex gPluralRulesCacheMutex = U_MUTEX_INITIALIZER; -static icu::UInitOnce gPluralRulesCacheInitOnce = U_INITONCE_INITIALIZER; - -U_CDECL_BEGIN -static UBool U_CALLCONV plurrules_cleanup(void) { - gPluralRulesCacheInitOnce.reset(); - if (gPluralRulesCache) { - delete gPluralRulesCache; - gPluralRulesCache = NULL; - } - return TRUE; -} -U_CDECL_END - U_NAMESPACE_BEGIN #define ARRAY_SIZE(array) (int32_t)(sizeof array / sizeof array[0]) @@ -155,50 +140,25 @@ PluralRules::createDefaultRules(UErrorCode& status) { /******************************************************************************/ /* Create PluralRules cache */ -static SharedObject *U_CALLCONV createSharedPluralRules( - const char *localeId, UErrorCode &status) { - if (U_FAILURE(status)) { - return NULL; - } +template<> U_I18N_API +const SharedPluralRules *LocaleCacheKey::createObject( + const void * /*unused*/, UErrorCode &status) const { + const char *localeId = fLoc.getName(); PluralRules *pr = PluralRules::internalForLocale( localeId, UPLURAL_TYPE_CARDINAL, status); if (U_FAILURE(status)) { return NULL; } - SharedObject *result = new SharedPluralRules(pr); + SharedPluralRules *result = new SharedPluralRules(pr); if (result == NULL) { status = U_MEMORY_ALLOCATION_ERROR; delete pr; return NULL; } + result->addRef(); return result; } -static void U_CALLCONV pluralRulesCacheInit(UErrorCode &status) { - U_ASSERT(gPluralRulesCache == NULL); - ucln_i18n_registerCleanup(UCLN_I18N_PLURAL_RULE, plurrules_cleanup); - gPluralRulesCache = new SimpleLRUCache(100, &createSharedPluralRules, status); - if (U_FAILURE(status)) { - delete gPluralRulesCache; - gPluralRulesCache = NULL; - } -} - -static void getSharedPluralRulesFromCache( - const char *locale, - const SharedPluralRules *&ptr, - UErrorCode &status) { - umtx_initOnce(gPluralRulesCacheInitOnce, &pluralRulesCacheInit, status); - if (U_FAILURE(status)) { - return; - } - Mutex lock(&gPluralRulesCacheMutex); - gPluralRulesCache->get(locale, ptr, status); -} - - - - /* end plural rules cache */ /******************************************************************************/ @@ -213,7 +173,7 @@ PluralRules::createSharedInstance( return NULL; } const SharedPluralRules *result = NULL; - getSharedPluralRulesFromCache(locale.getName(), result, status); + UnifiedCache::getByLocale(locale, result, status); return result; } diff --git a/icu4c/source/i18n/reldatefmt.cpp b/icu4c/source/i18n/reldatefmt.cpp index 0fdef039f67..369a6727ecc 100644 --- a/icu4c/source/i18n/reldatefmt.cpp +++ b/icu4c/source/i18n/reldatefmt.cpp @@ -19,7 +19,6 @@ #include "unicode/decimfmt.h" #include "unicode/numfmt.h" #include "unicode/brkiter.h" -#include "lrucache.h" #include "uresimp.h" #include "unicode/ures.h" #include "cstring.h" @@ -31,25 +30,12 @@ #include "sharedbreakiterator.h" #include "sharedpluralrules.h" #include "sharednumberformat.h" +#include "unifiedcache.h" // Copied from uscript_props.cpp #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) -static icu::LRUCache *gCache = NULL; -static UMutex gCacheMutex = U_MUTEX_INITIALIZER; static UMutex gBrkIterMutex = U_MUTEX_INITIALIZER; -static icu::UInitOnce gCacheInitOnce = U_INITONCE_INITIALIZER; - -U_CDECL_BEGIN -static UBool U_CALLCONV reldatefmt_cleanup() { - gCacheInitOnce.reset(); - if (gCache) { - delete gCache; - gCache = NULL; - } - return TRUE; -} -U_CDECL_END U_NAMESPACE_BEGIN @@ -592,9 +578,9 @@ static UBool getDateTimePattern( return getStringByIndex(topLevel.getAlias(), 8, result, status); } -// Creates RelativeDateTimeFormatter specific data for a given locale -static SharedObject *U_CALLCONV createData( - const char *localeId, UErrorCode &status) { +template<> U_I18N_API +const RelativeDateTimeCacheData *LocaleCacheKey::createObject(const void * /*unused*/, UErrorCode &status) const { + const char *localeId = fLoc.getName(); LocalUResourceBundlePointer topLevel(ures_open(NULL, localeId, &status)); if (U_FAILURE(status)) { return NULL; @@ -620,32 +606,10 @@ static SharedObject *U_CALLCONV createData( if (U_FAILURE(status)) { return NULL; } + result->addRef(); return result.orphan(); } -static void U_CALLCONV cacheInit(UErrorCode &status) { - U_ASSERT(gCache == NULL); - ucln_i18n_registerCleanup(UCLN_I18N_RELDATEFMT, reldatefmt_cleanup); - gCache = new SimpleLRUCache(100, &createData, status); - if (U_FAILURE(status)) { - delete gCache; - gCache = NULL; - } -} - -static UBool getFromCache( - const char *locale, - const RelativeDateTimeCacheData *&ptr, - UErrorCode &status) { - umtx_initOnce(gCacheInitOnce, &cacheInit, status); - if (U_FAILURE(status)) { - return FALSE; - } - Mutex lock(&gCacheMutex); - gCache->get(locale, ptr, status); - return U_SUCCESS(status); -} - RelativeDateTimeFormatter::RelativeDateTimeFormatter(UErrorCode& status) : fCache(NULL), fNumberFormat(NULL), @@ -851,7 +815,8 @@ void RelativeDateTimeFormatter::init( UErrorCode &status) { LocalPointer nf(nfToAdopt); LocalPointer bi(biToAdopt); - if (!getFromCache(fLocale.getName(), fCache, status)) { + UnifiedCache::getByLocale(fLocale, fCache, status); + if (U_FAILURE(status)) { return; } const SharedPluralRules *pr = PluralRules::createSharedInstance( diff --git a/icu4c/source/i18n/rulebasedcollator.cpp b/icu4c/source/i18n/rulebasedcollator.cpp index 710ba8fe954..538da7d56ce 100644 --- a/icu4c/source/i18n/rulebasedcollator.cpp +++ b/icu4c/source/i18n/rulebasedcollator.cpp @@ -142,11 +142,12 @@ RuleBasedCollator::RuleBasedCollator(const RuleBasedCollator &other) data(other.data), settings(other.settings), tailoring(other.tailoring), + cacheEntry(other.cacheEntry), validLocale(other.validLocale), explicitlySetAttributes(other.explicitlySetAttributes), actualLocaleIsSameAsValid(other.actualLocaleIsSameAsValid) { settings->addRef(); - tailoring->addRef(); + cacheEntry->addRef(); } RuleBasedCollator::RuleBasedCollator(const uint8_t *bin, int32_t length, @@ -154,6 +155,7 @@ RuleBasedCollator::RuleBasedCollator(const uint8_t *bin, int32_t length, : data(NULL), settings(NULL), tailoring(NULL), + cacheEntry(NULL), validLocale(""), explicitlySetAttributes(0), actualLocaleIsSameAsValid(FALSE) { @@ -176,33 +178,44 @@ RuleBasedCollator::RuleBasedCollator(const uint8_t *bin, int32_t length, CollationDataReader::read(base->tailoring, bin, length, *t, errorCode); if(U_FAILURE(errorCode)) { return; } t->actualLocale.setToBogus(); - adoptTailoring(t.orphan()); + adoptTailoring(t.orphan(), errorCode); } -RuleBasedCollator::RuleBasedCollator(const CollationTailoring *t, const Locale &vl) - : data(t->data), - settings(t->settings), - tailoring(t), - validLocale(vl), +RuleBasedCollator::RuleBasedCollator(const CollationCacheEntry *entry) + : data(entry->tailoring->data), + settings(entry->tailoring->settings), + tailoring(entry->tailoring), + cacheEntry(entry), + validLocale(entry->validLocale), explicitlySetAttributes(0), actualLocaleIsSameAsValid(FALSE) { settings->addRef(); - tailoring->addRef(); + cacheEntry->addRef(); } RuleBasedCollator::~RuleBasedCollator() { SharedObject::clearPtr(settings); - SharedObject::clearPtr(tailoring); + SharedObject::clearPtr(cacheEntry); } void -RuleBasedCollator::adoptTailoring(CollationTailoring *t) { - U_ASSERT(settings == NULL && data == NULL && tailoring == NULL); +RuleBasedCollator::adoptTailoring(CollationTailoring *t, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { + t->deleteIfZeroRefCount(); + return; + } + U_ASSERT(settings == NULL && data == NULL && tailoring == NULL && cacheEntry == NULL); + cacheEntry = new CollationCacheEntry(t->actualLocale, t); + if(cacheEntry == NULL) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + t->deleteIfZeroRefCount(); + return; + } data = t->data; settings = t->settings; settings->addRef(); - t->addRef(); tailoring = t; + cacheEntry->addRef(); validLocale = t->actualLocale; actualLocaleIsSameAsValid = FALSE; } @@ -215,7 +228,8 @@ RuleBasedCollator::clone() const { RuleBasedCollator &RuleBasedCollator::operator=(const RuleBasedCollator &other) { if(this == &other) { return *this; } SharedObject::copyPtr(other.settings, settings); - SharedObject::copyPtr(other.tailoring, tailoring); + tailoring = other.tailoring; + SharedObject::copyPtr(other.cacheEntry, cacheEntry); data = tailoring->data; validLocale = other.validLocale; explicitlySetAttributes = other.explicitlySetAttributes; diff --git a/icu4c/source/i18n/ucln_in.h b/icu4c/source/i18n/ucln_in.h index f63137659aa..e9685161fbc 100644 --- a/icu4c/source/i18n/ucln_in.h +++ b/icu4c/source/i18n/ucln_in.h @@ -40,12 +40,9 @@ typedef enum ECleanupI18NType { UCLN_I18N_TIMEZONENAMES, UCLN_I18N_ZONEMETA, UCLN_I18N_TIMEZONE, - UCLN_I18N_PLURAL_RULE, UCLN_I18N_CURRENCY, UCLN_I18N_DECFMT, UCLN_I18N_NUMFMT, - UCLN_I18N_RELDATEFMT, - UCLN_I18N_MEASFMT, UCLN_I18N_SMPDTFMT, UCLN_I18N_USEARCH, UCLN_I18N_COLLATOR, diff --git a/icu4c/source/i18n/ucol_imp.h b/icu4c/source/i18n/ucol_imp.h index dd1c85a0b0d..bfa2bb445af 100644 --- a/icu4c/source/i18n/ucol_imp.h +++ b/icu4c/source/i18n/ucol_imp.h @@ -30,6 +30,8 @@ #if !UCONFIG_NO_COLLATION +// This part needs to compile as plain C code, for cintltst. + #include "unicode/ucol.h" /** Check whether two collators are equal. Collators are considered equal if they @@ -50,12 +52,16 @@ ucol_equals(const UCollator *source, const UCollator *target); #ifdef __cplusplus +#include "unicode/locid.h" +#include "unicode/ures.h" + U_NAMESPACE_BEGIN -struct CollationTailoring; +struct CollationCacheEntry; class Locale; class UnicodeString; +class UnifiedCache; /** Implemented in ucol_res.cpp. */ class CollationLoader { @@ -63,12 +69,63 @@ public: static void appendRootRules(UnicodeString &s); static void loadRules(const char *localeID, const char *collationType, UnicodeString &rules, UErrorCode &errorCode); - static const CollationTailoring *loadTailoring(const Locale &locale, Locale &validLocale, - UErrorCode &errorCode); + // Adds a reference to returned value. + static const CollationCacheEntry *loadTailoring(const Locale &locale, UErrorCode &errorCode); + + // Cache callback. Adds a reference to returned value. + const CollationCacheEntry *createCacheEntry(UErrorCode &errorCode); private: - CollationLoader(); // not implemented, all methods are static static void loadRootRules(UErrorCode &errorCode); + + // The following members are used by loadTailoring() + // and the cache callback. + static const uint32_t TRIED_SEARCH = 1; + static const uint32_t TRIED_DEFAULT = 2; + static const uint32_t TRIED_STANDARD = 4; + + CollationLoader(const CollationCacheEntry *re, const Locale &requested, UErrorCode &errorCode); + ~CollationLoader(); + + // All loadFromXXX methods add a reference to the returned value. + const CollationCacheEntry *loadFromLocale(UErrorCode &errorCode); + const CollationCacheEntry *loadFromBundle(UErrorCode &errorCode); + const CollationCacheEntry *loadFromCollations(UErrorCode &errorCode); + const CollationCacheEntry *loadFromData(UErrorCode &errorCode); + + // Adds a reference to returned value. + const CollationCacheEntry *getCacheEntry(UErrorCode &errorCode); + + /** + * Returns the rootEntry (with one addRef()) if loc==root, + * or else returns a new cache entry with ref count 1 for the loc and + * the root tailoring. + */ + const CollationCacheEntry *makeCacheEntryFromRoot( + const Locale &loc, UErrorCode &errorCode) const; + + /** + * Returns the entryFromCache as is if loc==validLocale, + * or else returns a new cache entry with ref count 1 for the loc and + * the same tailoring. In the latter case, a ref count is removed from + * entryFromCache. + */ + static const CollationCacheEntry *makeCacheEntry( + const Locale &loc, + const CollationCacheEntry *entryFromCache, + UErrorCode &errorCode); + + const UnifiedCache *cache; + const CollationCacheEntry *rootEntry; + Locale validLocale; + Locale locale; + char type[16]; + char defaultType[16]; + uint32_t typesTried; + UBool typeFallback; + UResourceBundle *bundle; + UResourceBundle *collations; + UResourceBundle *data; }; U_NAMESPACE_END diff --git a/icu4c/source/i18n/ucol_res.cpp b/icu4c/source/i18n/ucol_res.cpp index d7fd26da478..2bd5a0152f9 100644 --- a/icu4c/source/i18n/ucol_res.cpp +++ b/icu4c/source/i18n/ucol_res.cpp @@ -46,6 +46,7 @@ #include "uenumimp.h" #include "ulist.h" #include "umutex.h" +#include "unifiedcache.h" #include "uresimp.h" #include "ustrenum.h" #include "utracimp.h" @@ -131,125 +132,265 @@ CollationLoader::loadRules(const char *localeID, const char *collationType, } } -const CollationTailoring * -CollationLoader::loadTailoring(const Locale &locale, Locale &validLocale, UErrorCode &errorCode) { - const CollationTailoring *root = CollationRoot::getRoot(errorCode); +template<> U_I18N_API +const CollationCacheEntry * +LocaleCacheKey::createObject(const void *creationContext, + UErrorCode &errorCode) const { + CollationLoader *loader = + reinterpret_cast( + const_cast(creationContext)); + return loader->createCacheEntry(errorCode); +} + +const CollationCacheEntry * +CollationLoader::loadTailoring(const Locale &locale, UErrorCode &errorCode) { + const CollationCacheEntry *rootEntry = CollationRoot::getRootCacheEntry(errorCode); if(U_FAILURE(errorCode)) { return NULL; } const char *name = locale.getName(); if(*name == 0 || uprv_strcmp(name, "root") == 0) { - validLocale = Locale::getRoot(); - return root; + + // Have to add a ref. + rootEntry->addRef(); + return rootEntry; + } + + // Clear warning codes before loading where they get cached. + errorCode = U_ZERO_ERROR; + CollationLoader loader(rootEntry, locale, errorCode); + + // getCacheEntry adds a ref for us. + return loader.getCacheEntry(errorCode); +} + +CollationLoader::CollationLoader(const CollationCacheEntry *re, const Locale &requested, + UErrorCode &errorCode) + : cache(UnifiedCache::getInstance(errorCode)), rootEntry(re), + validLocale(re->validLocale), locale(requested), + typesTried(0), typeFallback(FALSE), + bundle(NULL), collations(NULL), data(NULL) { + type[0] = 0; + defaultType[0] = 0; + if(U_FAILURE(errorCode)) { return; } + + // Canonicalize the locale ID: Ignore all irrelevant keywords. + const char *baseName = locale.getBaseName(); + if(uprv_strcmp(locale.getName(), baseName) != 0) { + locale = Locale(baseName); + + // Fetch the collation type from the locale ID. + int32_t typeLength = requested.getKeywordValue("collation", + type, LENGTHOF(type) - 1, errorCode); + if(U_FAILURE(errorCode)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + type[typeLength] = 0; // in case of U_NOT_TERMINATED_WARNING + if(typeLength == 0) { + // No collation type. + } else if(uprv_stricmp(type, "default") == 0) { + // Ignore "default" (case-insensitive). + type[0] = 0; + } else { + // Copy the collation type. + T_CString_toLowerCase(type); + locale.setKeywordValue("collation", type, errorCode); + } } +} - LocalUResourceBundlePointer bundle(ures_open(U_ICUDATA_COLL, name, &errorCode)); +CollationLoader::~CollationLoader() { + ures_close(data); + ures_close(collations); + ures_close(bundle); +} + +const CollationCacheEntry * +CollationLoader::createCacheEntry(UErrorCode &errorCode) { + // This is a linear lookup and fallback flow turned into a state machine. + // Most local variables have been turned into instance fields. + // In a cache miss, cache.get() calls CacheKey::createObject(), + // which means that we progress via recursion. + // loadFromCollations() will recurse to itself as well for collation type fallback. + if(bundle == NULL) { + return loadFromLocale(errorCode); + } else if(collations == NULL) { + return loadFromBundle(errorCode); + } else if(data == NULL) { + return loadFromCollations(errorCode); + } else { + return loadFromData(errorCode); + } +} + +const CollationCacheEntry * +CollationLoader::loadFromLocale(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return NULL; } + U_ASSERT(bundle == NULL); + bundle = ures_open(U_ICUDATA_COLL, locale.getBaseName(), &errorCode); if(errorCode == U_MISSING_RESOURCE_ERROR) { errorCode = U_USING_DEFAULT_WARNING; - validLocale = Locale::getRoot(); - return root; + + // Have to add that ref that we promise. + rootEntry->addRef(); + return rootEntry; } - const char *vLocale = ures_getLocaleByType(bundle.getAlias(), ULOC_ACTUAL_LOCALE, &errorCode); + Locale requestedLocale(locale); + const char *vLocale = ures_getLocaleByType(bundle, ULOC_ACTUAL_LOCALE, &errorCode); if(U_FAILURE(errorCode)) { return NULL; } - validLocale = Locale(vLocale); + locale = validLocale = Locale(vLocale); // no type until loadFromCollations() + if(type[0] != 0) { + locale.setKeywordValue("collation", type, errorCode); + } + if(locale != requestedLocale) { + return getCacheEntry(errorCode); + } else { + return loadFromBundle(errorCode); + } +} +const CollationCacheEntry * +CollationLoader::loadFromBundle(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return NULL; } + U_ASSERT(collations == NULL); // There are zero or more tailorings in the collations table. - LocalUResourceBundlePointer collations( - ures_getByKey(bundle.getAlias(), "collations", NULL, &errorCode)); + collations = ures_getByKey(bundle, "collations", NULL, &errorCode); if(errorCode == U_MISSING_RESOURCE_ERROR) { errorCode = U_USING_DEFAULT_WARNING; - return root; + // Return the root tailoring with the validLocale, without collation type. + return makeCacheEntryFromRoot(validLocale, errorCode); } if(U_FAILURE(errorCode)) { return NULL; } - // Fetch the collation type from the locale ID and the default type from the data. - char type[16]; - int32_t typeLength = locale.getKeywordValue("collation", type, LENGTHOF(type) - 1, errorCode); - if(U_FAILURE(errorCode)) { - errorCode = U_ILLEGAL_ARGUMENT_ERROR; - return NULL; - } - type[typeLength] = 0; // in case of U_NOT_TERMINATED_WARNING - char defaultType[16]; + // Fetch the default type from the data. { UErrorCode internalErrorCode = U_ZERO_ERROR; LocalUResourceBundlePointer def( - ures_getByKeyWithFallback(collations.getAlias(), "default", NULL, - &internalErrorCode)); + ures_getByKeyWithFallback(collations, "default", NULL, &internalErrorCode)); int32_t length; const UChar *s = ures_getString(def.getAlias(), &length, &internalErrorCode); - if(U_SUCCESS(internalErrorCode) && length < LENGTHOF(defaultType)) { + if(U_SUCCESS(internalErrorCode) && 0 < length && length < LENGTHOF(defaultType)) { u_UCharsToChars(s, defaultType, length + 1); } else { uprv_strcpy(defaultType, "standard"); } } - if(typeLength == 0 || uprv_strcmp(type, "default") == 0) { + + // Record which collation types we have looked for already, + // so that we do not deadlock in the cache. + // + // If there is no explicit type, then we look in the cache + // for the entry with the default type. + // If the explicit type is the default type, then we do not look in the cache + // for the entry with an empty type. + // Otherwise, two concurrent requests with opposite fallbacks would deadlock each other. + // Also, it is easier to always enter the next method with a non-empty type. + if(type[0] == 0) { uprv_strcpy(type, defaultType); + typesTried |= TRIED_DEFAULT; + if(uprv_strcmp(type, "search") == 0) { + typesTried |= TRIED_SEARCH; + } + if(uprv_strcmp(type, "standard") == 0) { + typesTried |= TRIED_STANDARD; + } + locale.setKeywordValue("collation", type, errorCode); + return getCacheEntry(errorCode); } else { - T_CString_toLowerCase(type); + if(uprv_strcmp(type, defaultType) == 0) { + typesTried |= TRIED_DEFAULT; + } + if(uprv_strcmp(type, "search") == 0) { + typesTried |= TRIED_SEARCH; + } + if(uprv_strcmp(type, "standard") == 0) { + typesTried |= TRIED_STANDARD; + } + return loadFromCollations(errorCode); } +} +const CollationCacheEntry * +CollationLoader::loadFromCollations(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return NULL; } + U_ASSERT(data == NULL); // Load the collations/type tailoring, with type fallback. - UBool typeFallback = FALSE; - LocalUResourceBundlePointer data( - ures_getByKeyWithFallback(collations.getAlias(), type, NULL, &errorCode)); - if(errorCode == U_MISSING_RESOURCE_ERROR && - typeLength > 6 && uprv_strncmp(type, "search", 6) == 0) { - // fall back from something like "searchjl" to "search" - typeFallback = TRUE; - type[6] = 0; - errorCode = U_ZERO_ERROR; - data.adoptInstead( - ures_getByKeyWithFallback(collations.getAlias(), type, NULL, &errorCode)); - } - if(errorCode == U_MISSING_RESOURCE_ERROR && uprv_strcmp(type, defaultType) != 0) { - // fall back to the default type - typeFallback = TRUE; - uprv_strcpy(type, defaultType); - errorCode = U_ZERO_ERROR; - data.adoptInstead( - ures_getByKeyWithFallback(collations.getAlias(), type, NULL, &errorCode)); - } - if(errorCode == U_MISSING_RESOURCE_ERROR && uprv_strcmp(type, "standard") != 0) { - // fall back to the "standard" type - typeFallback = TRUE; - uprv_strcpy(type, "standard"); - errorCode = U_ZERO_ERROR; - data.adoptInstead( - ures_getByKeyWithFallback(collations.getAlias(), type, NULL, &errorCode)); - } + LocalUResourceBundlePointer localData( + ures_getByKeyWithFallback(collations, type, NULL, &errorCode)); + int32_t typeLength = uprv_strlen(type); if(errorCode == U_MISSING_RESOURCE_ERROR) { errorCode = U_USING_DEFAULT_WARNING; - return root; + typeFallback = TRUE; + if((typesTried & TRIED_SEARCH) == 0 && + typeLength > 6 && uprv_strncmp(type, "search", 6) == 0) { + // fall back from something like "searchjl" to "search" + typesTried |= TRIED_SEARCH; + type[6] = 0; + } else if((typesTried & TRIED_DEFAULT) == 0) { + // fall back to the default type + typesTried |= TRIED_DEFAULT; + uprv_strcpy(type, defaultType); + } else if((typesTried & TRIED_STANDARD) == 0) { + // fall back to the "standard" type + typesTried |= TRIED_STANDARD; + uprv_strcpy(type, "standard"); + } else { + // Return the root tailoring with the validLocale, without collation type. + return makeCacheEntryFromRoot(validLocale, errorCode); + } + locale.setKeywordValue("collation", type, errorCode); + return getCacheEntry(errorCode); } if(U_FAILURE(errorCode)) { return NULL; } - LocalPointer t(new CollationTailoring(root->settings)); - if(t.isNull() || t->isBogus()) { - errorCode = U_MEMORY_ALLOCATION_ERROR; - return NULL; + data = localData.orphan(); + const char *actualLocale = ures_getLocaleByType(data, ULOC_ACTUAL_LOCALE, &errorCode); + if(U_FAILURE(errorCode)) { return NULL; } + const char *vLocale = validLocale.getBaseName(); + UBool actualAndValidLocalesAreDifferent = uprv_strcmp(actualLocale, vLocale) != 0; + + // Set the collation types on the informational locales, + // except when they match the default types (for brevity and backwards compatibility). + // For the valid locale, suppress the default type. + if(uprv_strcmp(type, defaultType) != 0) { + validLocale.setKeywordValue("collation", type, errorCode); + if(U_FAILURE(errorCode)) { return NULL; } } // Is this the same as the root collator? If so, then use that instead. - const char *actualLocale = ures_getLocaleByType(data.getAlias(), ULOC_ACTUAL_LOCALE, &errorCode); - if(U_FAILURE(errorCode)) { return NULL; } if((*actualLocale == 0 || uprv_strcmp(actualLocale, "root") == 0) && uprv_strcmp(type, "standard") == 0) { if(typeFallback) { errorCode = U_USING_DEFAULT_WARNING; } - return root; + return makeCacheEntryFromRoot(validLocale, errorCode); + } + + locale = Locale(actualLocale); + if(actualAndValidLocalesAreDifferent) { + locale.setKeywordValue("collation", type, errorCode); + const CollationCacheEntry *entry = getCacheEntry(errorCode); + return makeCacheEntry(validLocale, entry, errorCode); + } else { + return loadFromData(errorCode); + } +} + +const CollationCacheEntry * +CollationLoader::loadFromData(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return NULL; } + LocalPointer t(new CollationTailoring(rootEntry->tailoring->settings)); + if(t.isNull() || t->isBogus()) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return NULL; } - t->actualLocale = Locale(actualLocale); // deserialize - LocalUResourceBundlePointer binary( - ures_getByKey(data.getAlias(), "%%CollationBin", NULL, &errorCode)); + LocalUResourceBundlePointer binary(ures_getByKey(data, "%%CollationBin", NULL, &errorCode)); // Note: U_MISSING_RESOURCE_ERROR --> The old code built from rules if available // but that created undesirable dependencies. int32_t length; const uint8_t *inBytes = ures_getBinary(binary.getAlias(), &length, &errorCode); - if(U_FAILURE(errorCode)) { return NULL; } - CollationDataReader::read(root, inBytes, length, *t, errorCode); + CollationDataReader::read(rootEntry->tailoring, inBytes, length, *t, errorCode); // Note: U_COLLATOR_VERSION_MISMATCH --> The old code built from rules if available // but that created undesirable dependencies. if(U_FAILURE(errorCode)) { return NULL; } @@ -258,27 +399,23 @@ CollationLoader::loadTailoring(const Locale &locale, Locale &validLocale, UError { UErrorCode internalErrorCode = U_ZERO_ERROR; int32_t length; - const UChar *s = ures_getStringByKey(data.getAlias(), "Sequence", &length, + const UChar *s = ures_getStringByKey(data, "Sequence", &length, &internalErrorCode); - if(U_SUCCESS(errorCode)) { + if(U_SUCCESS(internalErrorCode)) { t->rules.setTo(TRUE, s, length); } } - // Set the collation types on the informational locales, - // except when they match the default types (for brevity and backwards compatibility). - // For the valid locale, suppress the default type. - if(uprv_strcmp(type, defaultType) != 0) { - validLocale.setKeywordValue("collation", type, errorCode); - if(U_FAILURE(errorCode)) { return NULL; } - } + const char *actualLocale = locale.getBaseName(); // without type + const char *vLocale = validLocale.getBaseName(); + UBool actualAndValidLocalesAreDifferent = uprv_strcmp(actualLocale, vLocale) != 0; // For the actual locale, suppress the default type *according to the actual locale*. // For example, zh has default=pinyin and contains all of the Chinese tailorings. // zh_Hant has default=stroke but has no other data. // For the valid locale "zh_Hant" we need to suppress stroke. // For the actual locale "zh" we need to suppress pinyin instead. - if(uprv_strcmp(actualLocale, vLocale) != 0) { + if(actualAndValidLocalesAreDifferent) { // Opening a bundle for the actual locale should always succeed. LocalUResourceBundlePointer actualBundle( ures_open(U_ICUDATA_COLL, actualLocale, &errorCode)); @@ -295,16 +432,67 @@ CollationLoader::loadTailoring(const Locale &locale, Locale &validLocale, UError uprv_strcpy(defaultType, "standard"); } } + t->actualLocale = locale; if(uprv_strcmp(type, defaultType) != 0) { t->actualLocale.setKeywordValue("collation", type, errorCode); - if(U_FAILURE(errorCode)) { return NULL; } + } else if(uprv_strcmp(locale.getName(), locale.getBaseName()) != 0) { + // Remove the collation keyword if it was set. + t->actualLocale.setKeywordValue("collation", NULL, errorCode); } + if(U_FAILURE(errorCode)) { return NULL; } if(typeFallback) { errorCode = U_USING_DEFAULT_WARNING; } - t->bundle = bundle.orphan(); - return t.orphan(); + t->bundle = bundle; + bundle = NULL; + const CollationCacheEntry *entry = new CollationCacheEntry(validLocale, t.getAlias()); + if(entry == NULL) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + } else { + t.orphan(); + } + // Have to add that reference that we promise. + entry->addRef(); + return entry; +} + +const CollationCacheEntry * +CollationLoader::getCacheEntry(UErrorCode &errorCode) { + LocaleCacheKey key(locale); + const CollationCacheEntry *entry = NULL; + cache->get(key, this, entry, errorCode); + return entry; +} + +const CollationCacheEntry * +CollationLoader::makeCacheEntryFromRoot( + const Locale &loc, + UErrorCode &errorCode) const { + if (U_FAILURE(errorCode)) { + return NULL; + } + rootEntry->addRef(); + return makeCacheEntry(validLocale, rootEntry, errorCode); +} + +const CollationCacheEntry * +CollationLoader::makeCacheEntry( + const Locale &loc, + const CollationCacheEntry *entryFromCache, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode) || loc == entryFromCache->validLocale) { + return entryFromCache; + } + CollationCacheEntry *entry = new CollationCacheEntry(loc, entryFromCache->tailoring); + if(entry == NULL) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + entryFromCache->removeRef(); + return NULL; + } + entry->addRef(); + entryFromCache->removeRef(); + return entry; } U_NAMESPACE_END diff --git a/icu4c/source/i18n/unicode/tblcoll.h b/icu4c/source/i18n/unicode/tblcoll.h index 4025b80b47c..878c1605af7 100644 --- a/icu4c/source/i18n/unicode/tblcoll.h +++ b/icu4c/source/i18n/unicode/tblcoll.h @@ -71,6 +71,7 @@ U_NAMESPACE_BEGIN +struct CollationCacheEntry; struct CollationData; struct CollationSettings; struct CollationTailoring; @@ -789,7 +790,7 @@ private: friend class CollationElementIterator; friend class Collator; - RuleBasedCollator(const CollationTailoring *t, const Locale &vl); + RuleBasedCollator(const CollationCacheEntry *entry); /** * Enumeration of attributes that are relevant for short definition strings @@ -801,7 +802,7 @@ private: ATTR_LIMIT }; - void adoptTailoring(CollationTailoring *t); + void adoptTailoring(CollationTailoring *t, UErrorCode &errorCode); // Both lengths must be <0 or else both must be >=0. UCollationResult doCompare(const UChar *left, int32_t leftLength, @@ -846,7 +847,8 @@ private: const CollationData *data; const CollationSettings *settings; // reference-counted - const CollationTailoring *tailoring; // reference-counted + const CollationTailoring *tailoring; // alias of cacheEntry->tailoring + const CollationCacheEntry *cacheEntry; // reference-counted Locale validLocale; uint32_t explicitlySetAttributes; diff --git a/icu4c/source/test/intltest/Makefile.in b/icu4c/source/test/intltest/Makefile.in index 4f2610f2a76..47cf33fa8bb 100644 --- a/icu4c/source/test/intltest/Makefile.in +++ b/icu4c/source/test/intltest/Makefile.in @@ -57,7 +57,7 @@ uobjtest.o idnaref.o idnaconf.o nptrans.o punyref.o testidn.o testidna.o uts46te incaltst.o calcasts.o v32test.o uvectest.o textfile.o tokiter.o utxttest.o \ windttst.o winnmtst.o winutil.o csdetest.o tzrulets.o tzoffloc.o tzfmttst.o ssearch.o dtifmtts.o \ tufmtts.o itspoof.o simplethread.o bidiconf.o locnmtst.o dcfmtest.o alphaindextst.o listformattertest.o genderinfotest.o compactdecimalformattest.o regiontst.o \ -reldatefmttest.o lrucachetest.o simplepatternformattertest.o measfmttest.o scientificformathelpertest.o numfmtspectest.o +reldatefmttest.o lrucachetest.o simplepatternformattertest.o measfmttest.o scientificformathelpertest.o numfmtspectest.o unifiedcachetest.o DEPS = $(OBJECTS:.o=.d) diff --git a/icu4c/source/test/intltest/intltest.vcxproj b/icu4c/source/test/intltest/intltest.vcxproj index 7402c5c4c04..5f6263ff1ee 100644 --- a/icu4c/source/test/intltest/intltest.vcxproj +++ b/icu4c/source/test/intltest/intltest.vcxproj @@ -347,6 +347,9 @@ false + + false + false false @@ -572,4 +575,4 @@ - \ No newline at end of file + diff --git a/icu4c/source/test/intltest/intltest.vcxproj.filters b/icu4c/source/test/intltest/intltest.vcxproj.filters index db1669ee3ba..3a92940a8b6 100644 --- a/icu4c/source/test/intltest/intltest.vcxproj.filters +++ b/icu4c/source/test/intltest/intltest.vcxproj.filters @@ -139,6 +139,9 @@ collections + + collections + collections @@ -842,4 +845,4 @@ formatting - \ No newline at end of file + diff --git a/icu4c/source/test/intltest/itutil.cpp b/icu4c/source/test/intltest/itutil.cpp index 8557ff92beb..7e0e75c5d1a 100644 --- a/icu4c/source/test/intltest/itutil.cpp +++ b/icu4c/source/test/intltest/itutil.cpp @@ -35,6 +35,7 @@ extern IntlTest *createUCharsTrieTest(); static IntlTest *createEnumSetTest(); extern IntlTest *createLRUCacheTest(); extern IntlTest *createSimplePatternFormatterTest(); +extern IntlTest *createUnifiedCacheTest(); #define CASE(id, test) case id: \ name = #test; \ @@ -113,6 +114,14 @@ void IntlTestUtilities::runIndexedTest( int32_t index, UBool exec, const char* & callTest(*test, par); } break; + case 22: + name = "UnifiedCacheTest"; + if (exec) { + logln("TestSuite UnifiedCacheTest---"); logln(); + LocalPointer test(createUnifiedCacheTest()); + callTest(*test, par); + } + break; default: name = ""; break; //needed to end loop } } diff --git a/icu4c/source/test/intltest/measfmttest.cpp b/icu4c/source/test/intltest/measfmttest.cpp index f41ff331de4..14f2715e4d6 100644 --- a/icu4c/source/test/intltest/measfmttest.cpp +++ b/icu4c/source/test/intltest/measfmttest.cpp @@ -20,7 +20,9 @@ #include "unicode/measure.h" #include "unicode/measunit.h" #include "unicode/tmunit.h" +#include "unicode/plurrule.h" #include "charstr.h" +#include "unicode/reldatefmt.h" #define LENGTHOF(array) (int32_t)(sizeof(array) / sizeof((array)[0])) diff --git a/icu4c/source/test/intltest/tsmthred.cpp b/icu4c/source/test/intltest/tsmthred.cpp index 54d1fd9caca..0e687cf6a62 100644 --- a/icu4c/source/test/intltest/tsmthred.cpp +++ b/icu4c/source/test/intltest/tsmthred.cpp @@ -28,6 +28,9 @@ #include "tsmthred.h" #include "unicode/ushape.h" #include "unicode/translit.h" +#include "sharedobject.h" +#include "unifiedcache.h" +#include "uassert.h" #if U_PLATFORM_USES_ONLY_WIN32_API /* Prefer native Windows APIs even if POSIX is implemented (i.e., on Cygwin). */ @@ -208,6 +211,12 @@ void MultithreadTest::runIndexedTest( int32_t index, UBool exec, TestConditionVariables(); } break; + case 8: + name = "TestUnifiedCache"; + if (exec) { + TestUnifiedCache(); + } + break; default: name = ""; break; //needed to end loop @@ -1691,5 +1700,103 @@ void MultithreadTest::TestConditionVariables() { delete threads[i]; } } - + +static const char *gCacheLocales[] = {"en_US", "en_GB", "fr_FR", "fr"}; +static int32_t gObjectsCreated = 0; +static const int32_t CACHE_LOAD = 3; + +class UCTMultiThreadItem : public SharedObject { + public: + char *value; + UCTMultiThreadItem(const char *x) : value(NULL) { + value = uprv_strdup(x); + } + virtual ~UCTMultiThreadItem() { + uprv_free(value); + } +}; + +template<> U_EXPORT +const UCTMultiThreadItem *LocaleCacheKey::createObject( + const void * /*unused*/, UErrorCode & /* status */) const { + // Since multiple threads are hitting the cache for the first time, + // no objects should be created yet. + umtx_lock(&gCTMutex); + if (gObjectsCreated != 0) { + gThisTest->errln("Expected no objects to be created yet."); + } + umtx_unlock(&gCTMutex); + + // Big, expensive object that takes 1 second to create. + SimpleThread::sleep(1000); + + // Log that we created an object. + umtx_lock(&gCTMutex); + ++gObjectsCreated; + umtx_unlock(&gCTMutex); + UCTMultiThreadItem *result = new UCTMultiThreadItem(fLoc.getName()); + result->addRef(); + return result; +} + +class UnifiedCacheThread: public SimpleThread { + public: + UnifiedCacheThread(const char *loc) : fLoc(loc) {}; + ~UnifiedCacheThread() {}; + void run(); + const char *fLoc; +}; + +void UnifiedCacheThread::run() { + UErrorCode status = U_ZERO_ERROR; + const UnifiedCache *cache = UnifiedCache::getInstance(status); + U_ASSERT(status == U_ZERO_ERROR); + const UCTMultiThreadItem *item = NULL; + cache->get(LocaleCacheKey(fLoc), item, status); + U_ASSERT(item != NULL); + if (uprv_strcmp(fLoc, item->value)) { + gThisTest->errln("Expected %s, got %s", fLoc, item->value); + } + item->removeRef(); + + // Mark this thread as finished + umtx_lock(&gCTMutex); + ++gFinishedThreads; + umtx_condBroadcast(&gCTConditionVar); + umtx_unlock(&gCTMutex); +} + +void MultithreadTest::TestUnifiedCache() { + UErrorCode status = U_ZERO_ERROR; + const UnifiedCache *cache = UnifiedCache::getInstance(status); + U_ASSERT(cache != NULL); + cache->flush(); + gThisTest = this; + gFinishedThreads = 0; + gObjectsCreated = 0; + + UnifiedCacheThread *threads[CACHE_LOAD][LENGTHOF(gCacheLocales)]; + for (int32_t i=0; istart(); + } + } + // Wait on all the threads to complete verify that LENGTHOF(gCacheLocales) + // objects were created. + umtx_lock(&gCTMutex); + while (gFinishedThreads < CACHE_LOAD*LENGTHOF(gCacheLocales)) { + umtx_condWait(&gCTConditionVar, &gCTMutex); + } + assertEquals("Objects created", LENGTHOF(gCacheLocales), gObjectsCreated); + umtx_unlock(&gCTMutex); + + // clean up threads + for (int32_t i=0; i U_EXPORT +const UCTItem *LocaleCacheKey::createObject( + const void * /*unused*/, UErrorCode &status) const { + if (uprv_strcmp(fLoc.getName(), "zh") == 0) { + status = U_MISSING_RESOURCE_ERROR; + return NULL; + } + if (uprv_strcmp(fLoc.getLanguage(), fLoc.getName()) != 0) { + const UCTItem *item = NULL; + UnifiedCache::getByLocale(fLoc.getLanguage(), item, status); + if (U_FAILURE(status)) { + return NULL; + } + return item; + } + UCTItem *result = new UCTItem(fLoc.getName()); + result->addRef(); + return result; +} + +template<> U_EXPORT +const UCTItem2 *LocaleCacheKey::createObject( + const void * /*unused*/, UErrorCode & /*status*/) const { + return NULL; +} + +class UnifiedCacheTest : public IntlTest { +public: + UnifiedCacheTest() { + } + void runIndexedTest(int32_t index, UBool exec, const char *&name, char *par=0); +private: + void TestBasic(); + void TestError(); + void TestHashEquals(); +}; + +void UnifiedCacheTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* /*par*/) { + TESTCASE_AUTO_BEGIN; + TESTCASE_AUTO(TestBasic); + TESTCASE_AUTO(TestError); + TESTCASE_AUTO(TestHashEquals); + TESTCASE_AUTO_END; +} + +void UnifiedCacheTest::TestBasic() { + UErrorCode status = U_ZERO_ERROR; + const UnifiedCache *cache = UnifiedCache::getInstance(status); + assertSuccess("", status); + cache->flush(); + int32_t baseCount = cache->keyCount(); + const UCTItem *en = NULL; + const UCTItem *enGb = NULL; + const UCTItem *enGb2 = NULL; + const UCTItem *enUs = NULL; + const UCTItem *fr = NULL; + const UCTItem *frFr = NULL; + cache->get(LocaleCacheKey("en"), en, status); + cache->get(LocaleCacheKey("en_US"), enUs, status); + cache->get(LocaleCacheKey("en_GB"), enGb, status); + cache->get(LocaleCacheKey("fr_FR"), frFr, status); + cache->get(LocaleCacheKey("fr"), fr, status); + cache->get(LocaleCacheKey("en_GB"), enGb2, status); + SharedObject::clearPtr(enGb2); + if (enGb != enUs) { + errln("Expected en_GB and en_US to resolve to same object."); + } + if (fr != frFr) { + errln("Expected fr and fr_FR to resolve to same object."); + } + if (enGb == fr) { + errln("Expected en_GB and fr to return different objects."); + } + assertSuccess("", status); + // en_US, en_GB, en share one object; fr_FR and fr don't share. + // 5 keys in all. + assertEquals("", baseCount + 5, cache->keyCount()); + SharedObject::clearPtr(enGb); + cache->flush(); + assertEquals("", baseCount + 5, cache->keyCount()); + SharedObject::clearPtr(enUs); + SharedObject::clearPtr(en); + cache->flush(); + // With en_GB and en_US and en cleared there are no more hard references to + // the "en" object, so it gets flushed and the keys that refer to it + // get removed from the cache. + assertEquals("", baseCount + 2, cache->keyCount()); + SharedObject::clearPtr(fr); + cache->flush(); + assertEquals("", baseCount + 2, cache->keyCount()); + SharedObject::clearPtr(frFr); + cache->flush(); + assertEquals("", baseCount + 0, cache->keyCount()); +} + +void UnifiedCacheTest::TestError() { + UErrorCode status = U_ZERO_ERROR; + const UnifiedCache *cache = UnifiedCache::getInstance(status); + assertSuccess("", status); + cache->flush(); + int32_t baseCount = cache->keyCount(); + const UCTItem *zh = NULL; + const UCTItem *zhTw = NULL; + const UCTItem *zhHk = NULL; + + status = U_ZERO_ERROR; + cache->get(LocaleCacheKey("zh"), zh, status); + if (status != U_MISSING_RESOURCE_ERROR) { + errln("Expected U_MISSING_RESOURCE_ERROR"); + } + status = U_ZERO_ERROR; + cache->get(LocaleCacheKey("zh_TW"), zhTw, status); + if (status != U_MISSING_RESOURCE_ERROR) { + errln("Expected U_MISSING_RESOURCE_ERROR"); + } + status = U_ZERO_ERROR; + cache->get(LocaleCacheKey("zh_HK"), zhHk, status); + if (status != U_MISSING_RESOURCE_ERROR) { + errln("Expected U_MISSING_RESOURCE_ERROR"); + } + // 3 keys in cache zh, zhTW, zhHk all pointing to error placeholders + assertEquals("", baseCount + 3, cache->keyCount()); + cache->flush(); + // error placeholders have no hard references so they always get flushed. + assertEquals("", baseCount + 0, cache->keyCount()); +} + +void UnifiedCacheTest::TestHashEquals() { + LocaleCacheKey key1("en_US"); + LocaleCacheKey key2("en_US"); + LocaleCacheKey diffKey1("en_UT"); + LocaleCacheKey diffKey2("en_US"); + assertTrue("", key1.hashCode() == key2.hashCode()); + assertTrue("", key1.hashCode() != diffKey1.hashCode()); + assertTrue("", key1.hashCode() != diffKey2.hashCode()); + assertTrue("", diffKey1.hashCode() != diffKey2.hashCode()); + assertTrue("", key1 == key2); + assertTrue("", key1 != diffKey1); + assertTrue("", key1 != diffKey2); + assertTrue("", diffKey1 != diffKey2); +} + +extern IntlTest *createUnifiedCacheTest() { + return new UnifiedCacheTest(); +} -- 2.40.0