Ptex

PtexCache.cpp
Go to the documentation of this file.
1/*
2 PTEX SOFTWARE
3 Copyright 2014 Disney Enterprises, Inc. All rights reserved
4
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are
7 met:
8
9 * Redistributions of source code must retain the above copyright
10 notice, this list of conditions and the following disclaimer.
11
12 * Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in
14 the documentation and/or other materials provided with the
15 distribution.
16
17 * The names "Disney", "Walt Disney Pictures", "Walt Disney Animation
18 Studios" or the names of its contributors may NOT be used to
19 endorse or promote products derived from this software without
20 specific prior written permission from Walt Disney Pictures.
21
22 Disclaimer: THIS SOFTWARE IS PROVIDED BY WALT DISNEY PICTURES AND
23 CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
24 BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
25 FOR A PARTICULAR PURPOSE, NONINFRINGEMENT AND TITLE ARE DISCLAIMED.
26 IN NO EVENT SHALL WALT DISNEY PICTURES, THE COPYRIGHT HOLDER OR
27 CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
28 EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
29 PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND BASED ON ANY
31 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
34*/
35
68
69#include "PtexPlatform.h"
70#include <algorithm>
71#include <sys/types.h>
72#include <sys/stat.h>
73#include <stdlib.h>
74#include <iostream>
75#include <ctype.h>
76#include "Ptexture.h"
77#include "PtexReader.h"
78#include "PtexCache.h"
79
80
81 PTEX_NAMESPACE_BEGIN
82
83 void PtexCachedReader::release()
84{
85 // Unref the texture, and log the texture as recently used if the ref count becomes 0. Note: If
86 // the ref is negative after unref is called, it means the unref'd texture was not previously in
87 // use which is an application error. In this case, the texture was presumably already logged as
88 // recently used and we shouldn't log it again.
89 if (0 == unref()) {
90 _cache->logRecentlyUsed(this);
91 }
92}
93
94
95 bool PtexReaderCache::findFile(const char*& filename, std::string& buffer, Ptex::String& error)
96{
97 bool isAbsolute = (filename[0] == '/'
98#ifdef PTEX_PLATFORM_WINDOWS
99 || filename[0] == '\\'
100 || (isalpha(filename[0]) && filename[1] == ':')
101#endif
102 );
103 if (isAbsolute || _searchdirs.empty()) return true; // no need to search
104
105 // file is relative, search in searchpath
106 buffer.reserve(256); // minimize reallocs (will grow automatically)
107 struct stat statbuf;
108 for (size_t i = 0, size = _searchdirs.size(); i < size; i++) {
109 buffer = _searchdirs[i];
110 buffer += "/";
111 buffer += filename;
112 if (stat(buffer.c_str(), &statbuf) == 0) {
113 filename = buffer.c_str();
114 return true;
115 }
116 }
117 // not found
118 std::string errstr = "Can't find ptex file: ";
119 errstr += filename;
120 error = errstr.c_str();
121 return false;
122}
123
124
125 PtexTexture* PtexReaderCache::get(const char* filename, Ptex::String& error)
126{
127 // lookup reader in map
128 StringKey key(filename);
129 PtexCachedReader* reader = _files.get(key);
130
131 if (reader) {
132 if (!reader->ok()) return 0;
133 if (reader->pendingPurge()) {
134 // a previous purge attempt was made and file was busy. Try again now.
135 purge(reader);
136 }
137 reader->ref();
138 } else {
139 reader = new PtexCachedReader(_premultiply, _io, _err, this);
140 size_t newMemUsed = 0;
141 PtexCachedReader* newreader = reader;
142 reader = _files.tryInsert(key, reader, newMemUsed);
143 adjustMemUsed(newMemUsed);
144 if (reader != newreader) {
145 // another thread got here first
146 reader->ref();
147 delete newreader;
148 }
149 }
150
151 if (reader->needToOpen()) {
152 std::string buffer;
153 const char* pathToOpen = filename;
154 // search for the file (unless we have an I/O handler)
155 if (_io || findFile(pathToOpen, buffer, error)) {
156 if (reader->open(pathToOpen, error)) {
157 reader->logOpen();
158 }
159 } else {
160 // flag reader as invalid so we don't try to open it again on next lookup
161 reader->invalidate();
162 }
163 }
164
165 if (!reader->ok()) {
166 reader->unref();
167 return 0;
168 }
169
170 return reader;
171}
172
173 PtexCache* PtexCache::create(int maxFiles, size_t maxMem, bool premultiply,
174 PtexInputHandler* inputHandler,
175 PtexErrorHandler* errorHandler)
176{
177 // set default files to 100
178 if (maxFiles <= 0) maxFiles = 100;
179
180 return new PtexReaderCache(maxFiles, maxMem, premultiply, inputHandler, errorHandler);
181}
182
183
184 void PtexReaderCache::logRecentlyUsed(PtexCachedReader* reader)
185{
186 while (1) {
187 MruList* mruList = _mruList;
188 int slot = AtomicIncrement(&mruList->next)-1;
189 if (slot < maxMruFiles) {
190 mruList->files[slot] = reader;
191 processMru();
192 return;
193 }
194 // no mru slot available, process mru list and try again
195 do processMru();
196 while (_mruList->next >= maxMruFiles);
197 }
198}
199
200 void PtexReaderCache::processMru()
201{
202 // use a non-blocking lock so we can proceed as soon as space has been freed in the mru list
203 // (which happens almost immediately in the processMru thread that has the lock)
204 if (!_mruLock.trylock()) return;
205
206 if (!_mruList->next) {
207 // nothing to do
208 _mruLock.unlock();
209 return;
210 }
211
212 // switch mru buffers and reset slot counter so other threads can proceed immediately
213 MruList* mruList = _mruList;
214 AtomicStore(&_mruList, _prevMruList);
215 _prevMruList = mruList;
216
217 // extract relevant stats and add to open/active list
218 size_t memUsedChange = 0, filesOpenChange = 0;
219 int numFilesToProcess = std::min(int(mruList->next), maxMruFiles);
220 for (int i = 0; i < numFilesToProcess; ++i) {
221 PtexCachedReader* reader;
222 do { reader = mruList->files[i]; } while (!reader); // loop on (unlikely) race condition
223 mruList->files[i] = 0;
224 memUsedChange += reader->getMemUsedChange();
225 size_t opens = reader->getOpensChange();
226 size_t blockReads = reader->getBlockReadsChange();
227 filesOpenChange += opens;
228 if (opens || blockReads) {
229 _fileOpens += opens;
230 _blockReads += blockReads;
231 _openFiles.push(reader);
232 }
233 if (_maxMem) {
234 _activeFiles.push(reader);
235 }
236 }
237 AtomicStore(&mruList->next, 0);
238 adjustMemUsed(memUsedChange);
239 adjustFilesOpen(filesOpenChange);
240 _mruLock.unlock();
241
242 pruneFilesIfNeeded();
243 if (_maxMem) {
244 pruneDataIfNeeded();
245 }
246}
247
248
249 void PtexReaderCache::pruneFilesIfNeeded()
250{
251 while (_filesOpen > _maxFiles) {
252 // close one file not currently in use, and then check again
253 _mruLock.lock();
254 PtexCachedReader* reader = _openFiles.pop();
255 _mruLock.unlock();
256
257 if (!reader) {
258 // no inactive open file available to close
259 break;
260 }
261 if (reader->tryClose()) {
262 --_filesOpen;
263 }
264 }
265}
266
267
268 void PtexReaderCache::pruneDataIfNeeded()
269{
270 size_t memUsedChangeTotal = 0;
271 while (_memUsed > _maxMem) {
272 // prune one texture not currently in use, and then check again
273 _mruLock.lock();
274 PtexCachedReader* reader = _activeFiles.pop();
275 _mruLock.unlock();
276 if (!reader) {
277 // no inactive texture available to prune
278 break;
279 }
280 size_t memUsedChange;
281 if (reader->tryPrune(memUsedChange)) {
282 // Note: after clearing, memUsedChange is negative
283 memUsedChangeTotal += memUsedChange;
284 }
285 }
286 if (memUsedChangeTotal) {
287 adjustMemUsed(memUsedChangeTotal);
288 }
289}
290
291
292 void PtexReaderCache::purge(PtexTexture* texture)
293{
294 PtexCachedReader* reader = static_cast<PtexCachedReader*>(texture);
295 reader->unref();
296 purge(reader);
297 reader->ref();
298}
299
300
301 void PtexReaderCache::purge(const char* filename)
302{
303 StringKey key(filename);
304 PtexCachedReader* reader = _files.get(key);
305 if (reader) purge(reader);
306}
307
308 void PtexReaderCache::purge(PtexCachedReader* reader)
309{
310 size_t memUsedChange;
311 if (reader->tryPurge(memUsedChange)) {
312 adjustMemUsed(memUsedChange);
313 }
314}
315
316 void PtexReaderCache::Purger::operator()(PtexCachedReader* reader)
317{
318 size_t memUsedChange;
319 if (reader->tryPurge(memUsedChange)) {
320 memUsedChangeTotal += memUsedChange;
321 }
322}
323
324 void PtexReaderCache::purgeAll()
325{
326 Purger purger;
327 _files.foreach(purger);
328 adjustMemUsed(purger.memUsedChangeTotal);
329}
330
331 void PtexReaderCache::getStats(Stats& stats)
332{
333 stats.memUsed = _memUsed;
334 stats.peakMemUsed = _peakMemUsed;
335 stats.filesOpen = _filesOpen;
336 stats.peakFilesOpen = _peakFilesOpen;
337 stats.filesAccessed = _files.size();
338 stats.fileReopens = _fileOpens < stats.filesAccessed ? 0 : _fileOpens - stats.filesAccessed;
339 stats.blockReads = _blockReads;
340}
341
342 PTEX_NAMESPACE_END
PtexCache.h
PtexPlatform.h
Platform-specific classes, functions, and includes.
AtomicStore
PTEX_INLINE void AtomicStore(T volatile *target, T value)
Definition PtexPlatform.h:286
AtomicIncrement
PTEX_INLINE T AtomicIncrement(volatile T *target)
Definition PtexPlatform.h:227
PtexReader.h
PTEX_NAMESPACE_END
#define PTEX_NAMESPACE_END
Definition PtexVersion.h:62
Ptexture.h
Public API classes for reading, writing, caching, and filtering Ptex files.
PtexCache
File-handle and memory cache for reading ptex files.
Definition Ptexture.h:693
PtexCache::create
static PtexCache * create(int maxFiles, size_t maxMem, bool premultiply=false, PtexInputHandler *inputHandler=0, PtexErrorHandler *errorHandler=0)
Create a cache with the specified limits.
Definition PtexCache.cpp:173
PtexCachedReader
Definition PtexCache.h:109
PtexCachedReader::_cache
PtexReaderCache * _cache
Definition PtexCache.h:110
PtexCachedReader::unref
int32_t unref()
Definition PtexCache.h:151
PtexCachedReader::tryPurge
bool tryPurge(size_t &memUsedChange)
Definition PtexCache.h:175
PtexCachedReader::getMemUsedChange
size_t getMemUsedChange()
Definition PtexCache.h:186
PtexCachedReader::tryPrune
bool tryPrune(size_t &memUsedChange)
Definition PtexCache.h:165
PtexCachedReader::release
virtual void release()
Release resources held by this pointer (pointer becomes invalid).
Definition PtexCache.cpp:83
PtexCachedReader::ref
void ref()
Definition PtexCache.h:138
PtexCachedReader::getBlockReadsChange
size_t getBlockReadsChange()
Definition PtexCache.h:200
PtexCachedReader::getOpensChange
size_t getOpensChange()
Definition PtexCache.h:193
PtexErrorHandler
Custom handler interface redirecting Ptex error messages.
Definition Ptexture.h:667
PtexInputHandler
Custom handler interface for intercepting and redirecting Ptex input stream calls.
Definition Ptexture.h:628
PtexReaderCache
Cache for reading Ptex texture files.
Definition PtexCache.h:211
PtexReaderCache::_activeFiles
PtexLruList< PtexCachedReader, &PtexCachedReader::_activeFilesItem > _activeFiles
Definition PtexCache.h:314
PtexReaderCache::_io
PtexInputHandler * _io
Definition PtexCache.h:293
PtexReaderCache::_maxMem
size_t _maxMem
Definition PtexCache.h:292
PtexReaderCache::adjustFilesOpen
void adjustFilesOpen(size_t amount)
Definition PtexCache.h:272
PtexReaderCache::_fileOpens
size_t _fileOpens
Definition PtexCache.h:318
PtexReaderCache::_memUsed
volatile size_t _memUsed
Definition PtexCache.h:300
PtexReaderCache::_mruLock
Mutex _mruLock
Definition PtexCache.h:302
PtexReaderCache::logRecentlyUsed
void logRecentlyUsed(PtexCachedReader *reader)
Definition PtexCache.cpp:184
PtexReaderCache::_prevMruList
MruList *volatile _prevMruList
Definition PtexCache.h:311
PtexReaderCache::pruneFilesIfNeeded
void pruneFilesIfNeeded()
Definition PtexCache.cpp:249
PtexReaderCache::_files
FileMap _files
Definition PtexCache.h:298
PtexReaderCache::adjustMemUsed
void adjustMemUsed(size_t amount)
Definition PtexCache.h:266
PtexReaderCache::_premultiply
bool _premultiply
Definition PtexCache.h:299
PtexReaderCache::_peakMemUsed
size_t _peakMemUsed
Definition PtexCache.h:316
PtexReaderCache::get
virtual PtexTexture * get(const char *path, Ptex::String &error)
Access a texture.
Definition PtexCache.cpp:125
PtexReaderCache::findFile
bool findFile(const char *&filename, std::string &buffer, Ptex::String &error)
Definition PtexCache.cpp:95
PtexReaderCache::maxMruFiles
static const int maxMruFiles
Definition PtexCache.h:304
PtexReaderCache::getStats
virtual void getStats(Stats &stats)
Get stats.
Definition PtexCache.cpp:331
PtexReaderCache::_maxFiles
size_t _maxFiles
Definition PtexCache.h:291
PtexReaderCache::_peakFilesOpen
size_t _peakFilesOpen
Definition PtexCache.h:317
PtexReaderCache::_err
PtexErrorHandler * _err
Definition PtexCache.h:294
PtexReaderCache::_openFiles
PtexLruList< PtexCachedReader, &PtexCachedReader::_openFilesItem > _openFiles
Definition PtexCache.h:313
PtexReaderCache::processMru
void processMru()
Definition PtexCache.cpp:200
PtexReaderCache::purgeAll
virtual void purgeAll()
Remove all texture files from the cache.
Definition PtexCache.cpp:324
PtexReaderCache::_blockReads
size_t _blockReads
Definition PtexCache.h:319
PtexReaderCache::pruneDataIfNeeded
void pruneDataIfNeeded()
Definition PtexCache.cpp:268
PtexReaderCache::_filesOpen
volatile size_t _filesOpen
Definition PtexCache.h:301
PtexReaderCache::_mruList
MruList *volatile _mruList
Definition PtexCache.h:310
PtexReaderCache::purge
virtual void purge(PtexTexture *)
Remove a texture file from the cache.
Definition PtexCache.cpp:292
PtexReaderCache::_searchdirs
std::vector< std::string > _searchdirs
Definition PtexCache.h:296
PtexReader::ok
bool ok() const
Definition PtexReader.h:65
PtexReader::needToOpen
bool needToOpen() const
Definition PtexReader.h:58
PtexReader::pendingPurge
bool pendingPurge() const
Definition PtexReader.h:63
PtexReader::invalidate
void invalidate()
Definition PtexReader.h:67
PtexReader::logOpen
void logOpen()
Definition PtexReader.h:73
PtexReader::open
bool open(const char *path, Ptex::String &error)
Definition PtexReader.cpp:137
PtexReader::tryClose
bool tryClose()
Definition PtexReader.cpp:222
PtexTexture
Interface for reading data from a ptex file.
Definition Ptexture.h:460
Ptex::String
Memory-managed string.
Definition Ptexture.h:299
StringKey
Definition PtexHashMap.h:64
PTEX_NAMESPACE_BEGIN
Definition PtexSeparableKernel.cpp:42
PtexCache::Stats
Definition Ptexture.h:788
PtexCache::Stats::filesAccessed
uint64_t filesAccessed
Definition Ptexture.h:793
PtexCache::Stats::filesOpen
uint64_t filesOpen
Definition Ptexture.h:791
PtexCache::Stats::peakFilesOpen
uint64_t peakFilesOpen
Definition Ptexture.h:792
PtexCache::Stats::fileReopens
uint64_t fileReopens
Definition Ptexture.h:794
PtexCache::Stats::memUsed
uint64_t memUsed
Definition Ptexture.h:789
PtexCache::Stats::blockReads
uint64_t blockReads
Definition Ptexture.h:795
PtexCache::Stats::peakMemUsed
uint64_t peakMemUsed
Definition Ptexture.h:790
PtexReaderCache::MruList
Definition PtexCache.h:305
PtexReaderCache::MruList::files
PtexCachedReader *volatile files[maxMruFiles]
Definition PtexCache.h:307
PtexReaderCache::MruList::next
volatile int next
Definition PtexCache.h:306
PtexReaderCache::Purger
Definition PtexCache.h:281
PtexReaderCache::Purger::operator()
void operator()(PtexCachedReader *reader)
Definition PtexCache.cpp:316
PtexReaderCache::Purger::memUsedChangeTotal
size_t memUsedChangeTotal
Definition PtexCache.h:282