From 43ed57b278a9411f293cb6149e1a6b46e92ad692 Mon Sep 17 00:00:00 2001 From: Charles Kerr Date: Thu, 9 Dec 2010 20:43:23 +0000 Subject: [PATCH] (trunk libT) #2955 "Lazy Verification (aka Just-in-Time Verification)" -- implemented. --- libtransmission/peer-mgr.c | 7 +- libtransmission/peer-msgs.c | 6 + libtransmission/resume.c | 75 ++------ libtransmission/torrent.c | 323 ++++++++++++++++++--------------- libtransmission/torrent.h | 39 ++-- libtransmission/transmission.h | 1 + libtransmission/verify.c | 138 ++++---------- 7 files changed, 252 insertions(+), 337 deletions(-) diff --git a/libtransmission/peer-mgr.c b/libtransmission/peer-mgr.c index 0601f2ef2..7ee2fefdb 100644 --- a/libtransmission/peer-mgr.c +++ b/libtransmission/peer-mgr.c @@ -28,7 +28,6 @@ #include "completion.h" #include "crypto.h" #include "handshake.h" -#include "inout.h" /* tr_ioTestPiece */ #include "net.h" #include "peer-io.h" #include "peer-mgr.h" @@ -1453,7 +1452,9 @@ peerCallbackFunc( tr_peer * peer, const tr_peer_event * e, void * vt ) if( tr_cpPieceIsComplete( &tor->completion, e->pieceIndex ) ) { const tr_piece_index_t p = e->pieceIndex; - const tr_bool ok = tr_ioTestPiece( tor, p ); + const tr_bool ok = tr_torrentCheckPiece( tor, p ); + + tr_tordbg( tor, "[LAZY] checked just-completed piece %zu", (size_t)p ); if( !ok ) { @@ -1461,8 +1462,6 @@ peerCallbackFunc( tr_peer * peer, const tr_peer_event * e, void * vt ) (unsigned long)p ); } - tr_torrentSetHasPiece( tor, p, ok ); - tr_torrentSetPieceChecked( tor, p, TRUE ); tr_peerMgrSetBlame( tor, p, ok ); if( !ok ) diff --git a/libtransmission/peer-msgs.c b/libtransmission/peer-msgs.c index 1b9d9338a..ad33e37ba 100644 --- a/libtransmission/peer-msgs.c +++ b/libtransmission/peer-msgs.c @@ -1920,6 +1920,12 @@ fillOutputBuffer( tr_peermsgs * msgs, time_t now ) tr_peerIoWriteUint32( io, out, req.offset ); err = tr_cacheReadBlock( getSession(msgs)->cache, msgs->torrent, req.index, req.offset, req.length, EVBUFFER_DATA(out)+EVBUFFER_LENGTH(out) ); + + /* check the piece if it needs checking... */ + if( !err && tr_torrentPieceNeedsCheck( msgs->torrent, req.index ) ) + if(( err = !tr_torrentCheckPiece( msgs->torrent, req.index ))) + tr_torrentSetLocalError( msgs->torrent, _( "Piece #%zu is corrupt! Please Verify Local Data." ), (size_t)piece ); + if( err ) { if( fext ) diff --git a/libtransmission/resume.c b/libtransmission/resume.c index d6d3c3559..c83b3954c 100644 --- a/libtransmission/resume.c +++ b/libtransmission/resume.c @@ -60,7 +60,7 @@ #define KEY_IDLELIMIT_MINS "idle-limit" #define KEY_IDLELIMIT_MODE "idle-mode" -#define KEY_PROGRESS_MTIMES "mtimes" +#define KEY_PROGRESS_CHECKTIME "time-checked" #define KEY_PROGRESS_BITFIELD "bitfield" #define KEY_PROGRESS_HAVE "have" @@ -415,7 +415,6 @@ saveProgress( tr_benc * dict, const tr_torrent * tor ) { size_t i, n; - time_t * mtimes; tr_benc * p; tr_benc * m; const tr_bitfield * bitfield; @@ -423,15 +422,11 @@ saveProgress( tr_benc * dict, p = tr_bencDictAdd( dict, KEY_PROGRESS ); tr_bencInitDict( p, 2 ); - /* add the mtimes */ - mtimes = tr_torrentGetMTimes( tor, &n ); - m = tr_bencDictAddList( p, KEY_PROGRESS_MTIMES, n ); - for( i = 0; i < n; ++i ) - { - if( !tr_torrentIsFileChecked( tor, i ) ) - mtimes[i] = ~(time_t)0; /* force a recheck */ - tr_bencListAddInt( m, mtimes[i] ); - } + /* add each piece's timeChecked */ + n = tor->info.pieceCount; + m = tr_bencDictAddList( p, KEY_PROGRESS_CHECKTIME, n ); + for( i=0; iinfo.pieces[i].timeChecked ); /* add the progress */ if( tor->completeness == TR_SEED ) @@ -439,18 +434,19 @@ saveProgress( tr_benc * dict, bitfield = tr_cpBlockBitfield( &tor->completion ); tr_bencDictAddRaw( p, KEY_PROGRESS_BITFIELD, bitfield->bits, bitfield->byteCount ); - - /* cleanup */ - tr_free( mtimes ); } static uint64_t loadProgress( tr_benc * dict, tr_torrent * tor ) { + size_t i, n; uint64_t ret = 0; tr_benc * p; + for( i=0, n=tor->info.pieceCount; iinfo.pieces[i].timeChecked = 0; + if( tr_bencDictFindDict( dict, KEY_PROGRESS, &p ) ) { const char * err; @@ -458,47 +454,13 @@ loadProgress( tr_benc * dict, const uint8_t * raw; size_t rawlen; tr_benc * m; - size_t n; - time_t * curMTimes = tr_torrentGetMTimes( tor, &n ); + int64_t timeChecked; - if( tr_bencDictFindList( p, KEY_PROGRESS_MTIMES, &m ) - && ( n == tor->info.fileCount ) - && ( n == tr_bencListSize( m ) ) ) - { - size_t i; - for( i = 0; i < n; ++i ) - { - int64_t tmp; - if( !tr_bencGetInt( tr_bencListChild( m, i ), &tmp ) ) - { - tr_tordbg( - tor, - "File #%zu needs to be verified - couldn't find benc entry", - i ); - tr_torrentSetFileChecked( tor, i, FALSE ); - } - else - { - const time_t t = (time_t) tmp; - if( t == curMTimes[i] ) - tr_torrentSetFileChecked( tor, i, TRUE ); - else - { - tr_tordbg( - tor, - "File #%zu needs to be verified - times %lu and %lu don't match", - i, t, curMTimes[i] ); - tr_torrentSetFileChecked( tor, i, FALSE ); - } - } - } - } - else - { - tr_torrentUncheck( tor ); - tr_tordbg( - tor, "Torrent needs to be verified - unable to find mtimes" ); - } + /* load in the timestamp of when we last checked each piece */ + if( tr_bencDictFindList( p, KEY_PROGRESS_CHECKTIME, &m ) ) + for( i=0, n=tor->info.pieceCount; iinfo.pieces[i].timeChecked = (time_t)timeChecked; err = NULL; if( tr_bencDictFindStr( p, KEY_PROGRESS_HAVE, &str ) ) @@ -518,13 +480,10 @@ loadProgress( tr_benc * dict, err = "Error loading bitfield"; } else err = "Couldn't find 'have' or 'bitfield'"; + if( err != NULL ) - { - tr_torrentUncheck( tor ); tr_tordbg( tor, "Torrent needs to be verified - %s", err ); - } - tr_free( curMTimes ); ret = TR_FR_PROGRESS; } diff --git a/libtransmission/torrent.c b/libtransmission/torrent.c index 58b3d9c7f..8a51d532f 100644 --- a/libtransmission/torrent.c +++ b/libtransmission/torrent.c @@ -34,6 +34,7 @@ #include "crypto.h" /* for tr_sha1 */ #include "resume.h" #include "fdlimit.h" /* tr_fdTorrentClose */ +#include "inout.h" /* tr_ioTestPiece() */ #include "magnet.h" #include "metainfo.h" #include "peer-common.h" /* MAX_BLOCK_SIZE */ @@ -459,6 +460,8 @@ tr_torrentSetLocalError( tr_torrent * tor, const char * fmt, ... ) evutil_vsnprintf( tor->errorString, sizeof( tor->errorString ), fmt, ap ); va_end( ap ); + tr_torerr( tor, "%s", tor->errorString ); + if( tor->isRunning ) tor->isStopping = TRUE; } @@ -604,11 +607,11 @@ calculatePiecePriority( const tr_torrent * tor, static void tr_torrentInitFilePieces( tr_torrent * tor ) { - tr_file_index_t f; + int * firstFiles; + tr_file_index_t f; tr_piece_index_t p; uint64_t offset = 0; tr_info * inf = &tor->info; - int * firstFiles; /* assign the file offsets */ for( f=0; ffileCount; ++f ) { @@ -720,8 +723,6 @@ torrentInitFromInfo( tr_torrent * tor ) tr_torrentInitFilePieces( tor ); - tr_bitfieldConstruct( &tor->checkedPieces, tor->info.pieceCount ); - tor->completeness = tr_cpGetStatus( &tor->completion ); } @@ -735,12 +736,28 @@ tr_torrentGotNewInfoDict( tr_torrent * tor ) tr_torrentFireMetadataCompleted( tor ); } +static tr_bool +setLocalErrorIfFilesDisappeared( tr_torrent * tor ) +{ + const tr_bool disappeared = ( tr_cpHaveTotal( &tor->completion ) > 0 ) && ( tr_torrentGetCurrentSizeOnDisk( tor ) == 0 ); + + if( disappeared ) + { + tr_tordbg( tor, "%s", "[LAZY] uh oh, the files disappeared" ); + tr_torrentSetLocalError( tor, "%s", _( "No data found! Ensure your drives are connected or use \"Set Location\". To re-download, remove the torrent and re-add it." ) ); + } + + return disappeared; +} + static void torrentInit( tr_torrent * tor, const tr_ctor * ctor ) { int doStart; uint64_t loaded; const char * dir; + tr_bool isNewTorrent; + struct stat st; static int nextUniqueId = 1; tr_session * session = tr_ctorGetSession( ctor ); @@ -778,14 +795,13 @@ torrentInit( tr_torrent * tor, const tr_ctor * ctor ) assert( !tor->downloadedCur ); assert( !tor->uploadedCur ); - tr_torrentUncheck( tor ); - tr_torrentSetAddedDate( tor, tr_time( ) ); /* this is a default value to be overwritten by the resume file */ torrentInitFromInfo( tor ); loaded = tr_torrentLoadResume( tor, ~0, ctor ); tor->completeness = tr_cpGetStatus( &tor->completion ); + setLocalErrorIfFilesDisappeared( tor ); tr_ctorInitTorrentPriorities( ctor, tor ); tr_ctorInitTorrentWanted( ctor, tor ); @@ -829,6 +845,9 @@ torrentInit( tr_torrent * tor, const tr_ctor * ctor ) ++session->torrentCount; } + /* if we don't have a local .torrent file already, assume the torrent is new */ + isNewTorrent = stat( tor->info.torrent, &st ); + /* maybe save our own copy of the metainfo */ if( tr_ctorGetSave( ctor ) ) { @@ -845,8 +864,15 @@ torrentInit( tr_torrent * tor, const tr_ctor * ctor ) tor->tiers = tr_announcerAddTorrent( tor->session->announcer, tor, onTrackerResponse, NULL ); - if( doStart ) + if( isNewTorrent ) + { + tor->startAfterVerify = doStart; + tr_torrentVerify( tor ); + } + else if( doStart ) + { torrentStart( tor ); + } tr_sessionUnlock( session ); } @@ -1048,6 +1074,21 @@ tr_torrentGetActivity( tr_torrent * tor ) return TR_STATUS_SEED; } +static double +getVerifyProgress( const tr_torrent * tor ) +{ + tr_piece_index_t i, n; + tr_piece_index_t checked = 0; + + assert( tr_isTorrent( tor ) ); + + for( i=0, n=tor->info.pieceCount; i!=n; ++i ) + if( tor->info.pieces[i].timeChecked ) + ++checked; + + return checked / (double)tor->info.pieceCount; +} + const tr_stat * tr_torrentStat( tr_torrent * tor ) { @@ -1097,18 +1138,14 @@ tr_torrentStat( tr_torrent * tor ) s->percentComplete = tr_cpPercentComplete ( &tor->completion ); s->metadataPercentComplete = tr_torrentGetMetadataPercent( tor ); - s->percentDone = tr_cpPercentDone ( &tor->completion ); - s->leftUntilDone = tr_cpLeftUntilDone( &tor->completion ); - s->sizeWhenDone = tr_cpSizeWhenDone ( &tor->completion ); - - s->recheckProgress = s->activity == TR_STATUS_CHECK - ? 1.0 - ( tr_torrentCountUncheckedPieces( tor ) / (float) tor->info.pieceCount ) - : 0.0; - - s->activityDate = tor->activityDate; - s->addedDate = tor->addedDate; - s->doneDate = tor->doneDate; - s->startDate = tor->startDate; + s->percentDone = tr_cpPercentDone ( &tor->completion ); + s->leftUntilDone = tr_cpLeftUntilDone( &tor->completion ); + s->sizeWhenDone = tr_cpSizeWhenDone ( &tor->completion ); + s->recheckProgress = s->activity == TR_STATUS_CHECK ? getVerifyProgress( tor ) : 0; + s->activityDate = tor->activityDate; + s->addedDate = tor->addedDate; + s->doneDate = tor->doneDate; + s->startDate = tor->startDate; if ((s->activity == TR_STATUS_DOWNLOAD || s->activity == TR_STATUS_SEED) && s->startDate != 0) s->idleSecs = difftime(tr_time(), MAX(s->startDate, s->activityDate)); @@ -1441,8 +1478,6 @@ freeTorrent( tr_torrent * tor ) tr_announcerRemoveTorrent( session->announcer, tor ); - tr_bitfieldDestruct( &tor->checkedPieces ); - tr_free( tor->downloadDir ); tr_free( tor->incompleteDir ); tr_free( tor->peer_id ); @@ -1472,47 +1507,54 @@ freeTorrent( tr_torrent * tor ) **/ static void -checkAndStartImpl( void * vtor ) +torrentStartImpl( void * vtor ) { + time_t now; tr_torrent * tor = vtor; assert( tr_isTorrent( tor ) ); tr_sessionLock( tor->session ); - /** If we had local data before, but it's disappeared, - stop the torrent and log an error. */ - if( tor->preVerifyTotal && !tr_cpHaveTotal( &tor->completion ) ) - { - tr_torrentSetLocalError( tor, "%s", _( "No data found! Reconnect any disconnected drives, use \"Set Location\", or restart the torrent to re-download." ) ); - } - else - { - const time_t now = tr_time( ); - tor->isRunning = TRUE; - tor->completeness = tr_cpGetStatus( &tor->completion ); - tor->startDate = tor->anyDate = now; - tr_torrentClearError( tor ); - tor->finishedSeedingByIdle = FALSE; + tr_torrentRecheckCompleteness( tor ); - tr_torrentResetTransferStats( tor ); - tr_announcerTorrentStarted( tor ); - tor->dhtAnnounceAt = now + tr_cryptoWeakRandInt( 20 ); - tor->dhtAnnounce6At = now + tr_cryptoWeakRandInt( 20 ); - tor->lpdAnnounceAt = now; - tr_peerMgrStartTorrent( tor ); - } + now = tr_time( ); + tor->isRunning = TRUE; + tor->completeness = tr_cpGetStatus( &tor->completion ); + tor->startDate = tor->anyDate = now; + tr_torrentClearError( tor ); + tor->finishedSeedingByIdle = FALSE; + + tr_torrentResetTransferStats( tor ); + tr_announcerTorrentStarted( tor ); + tor->dhtAnnounceAt = now + tr_cryptoWeakRandInt( 20 ); + tor->dhtAnnounce6At = now + tr_cryptoWeakRandInt( 20 ); + tor->lpdAnnounceAt = now; + tr_peerMgrStartTorrent( tor ); tr_sessionUnlock( tor->session ); } -static void -checkAndStartCB( tr_torrent * tor ) +uint64_t +tr_torrentGetCurrentSizeOnDisk( const tr_torrent * tor ) { - assert( tr_isTorrent( tor ) ); - assert( tr_isSession( tor->session ) ); + tr_file_index_t i; + uint64_t byte_count = 0; + const tr_file_index_t n = tor->info.fileCount; + + for( i=0; isession, checkAndStartImpl, tor ); + tr_free( filename ); + } + + return byte_count; } static void @@ -1520,33 +1562,36 @@ torrentStart( tr_torrent * tor ) { assert( tr_isTorrent( tor ) ); - tr_sessionLock( tor->session ); - - if( !tor->isRunning ) - { - /* allow finished torrents to be resumed */ - if( tr_torrentIsSeedRatioDone( tor ) ) - { - tr_torinf( tor, "Restarted manually -- disabling its seed ratio" ); - tr_torrentSetRatioMode( tor, TR_RATIOLIMIT_UNLIMITED ); - } + /* already running... */ + if( tor->isRunning ) + return; - tr_verifyRemove( tor ); + /* don't allow the torrent to be started if the files disappeared */ + if( setLocalErrorIfFilesDisappeared( tor ) ) + return; - /* corresponds to the peer_id sent as a tracker request parameter. - * one tracker admin says: "When the same torrent is opened and - * closed and opened again without quitting Transmission ... - * change the peerid. It would help sometimes if a stopped event - * was missed to ensure that we didn't think someone was cheating. */ - tr_free( tor->peer_id ); - tor->peer_id = tr_peerIdNew( ); + /* otherwise, start it now... */ + tr_sessionLock( tor->session ); - tor->isRunning = 1; - tr_torrentSetDirty( tor ); - tor->preVerifyTotal = tr_cpHaveTotal( &tor->completion ); - tr_verifyAdd( tor, checkAndStartCB ); + /* allow finished torrents to be resumed */ + if( tr_torrentIsSeedRatioDone( tor ) ) { + tr_torinf( tor, _( "Restarted manually -- disabling its seed ratio" ) ); + tr_torrentSetRatioMode( tor, TR_RATIOLIMIT_UNLIMITED ); } + tr_verifyRemove( tor ); + + /* corresponds to the peer_id sent as a tracker request parameter. + * one tracker admin says: "When the same torrent is opened and + * closed and opened again without quitting Transmission ... + * change the peerid. It would help sometimes if a stopped event + * was missed to ensure that we didn't think someone was cheating. */ + tr_free( tor->peer_id ); + tor->peer_id = tr_peerIdNew( ); + tor->isRunning = 1; + tr_torrentSetDirty( tor ); + tr_runInEventThread( tor->session, torrentStartImpl, tor ); + tr_sessionUnlock( tor->session ); } @@ -1561,19 +1606,13 @@ static void torrentRecheckDoneImpl( void * vtor ) { tr_torrent * tor = vtor; - assert( tr_isTorrent( tor ) ); + tr_torrentRecheckCompleteness( tor ); - if( tor->preVerifyTotal && !tr_cpHaveTotal( &tor->completion ) ) - { - tr_torrentSetLocalError( tor, "%s", _( "Can't find local data. Try \"Set Location\" to find it, or restart the torrent to re-download." ) ); - } - else if( tor->startAfterVerify ) - { + if( tor->startAfterVerify ) { tor->startAfterVerify = FALSE; - - tr_torrentStart( tor ); + torrentStart( tor ); } } @@ -1604,10 +1643,10 @@ verifyTorrent( void * vtor ) tor->startAfterVerify = startAfter; } - /* add the torrent to the recheck queue */ - tor->preVerifyTotal = tr_cpHaveTotal( &tor->completion ); - tr_torrentUncheck( tor ); - tr_verifyAdd( tor, torrentRecheckDoneCB ); + if( setLocalErrorIfFilesDisappeared( tor ) ) + tor->startAfterVerify = FALSE; + else + tr_verifyAdd( tor, torrentRecheckDoneCB ); tr_sessionUnlock( tor->session ); } @@ -2208,96 +2247,88 @@ tr_pieceOffset( const tr_torrent * tor, ***/ void -tr_torrentSetPieceChecked( tr_torrent * tor, - tr_piece_index_t piece, - tr_bool isChecked ) +tr_torrentSetPieceChecked( tr_torrent * tor, tr_piece_index_t pieceIndex ) { assert( tr_isTorrent( tor ) ); + assert( pieceIndex < tor->info.pieceCount ); - if( isChecked ) - tr_bitfieldAdd( &tor->checkedPieces, piece ); - else - tr_bitfieldRem( &tor->checkedPieces, piece ); + tr_tordbg( tor, "[LAZY] setting piece %zu timeChecked to now", (size_t)pieceIndex ); + tor->info.pieces[pieceIndex].timeChecked = tr_time( ); } void -tr_torrentSetFileChecked( tr_torrent * tor, - tr_file_index_t fileIndex, - tr_bool isChecked ) +tr_torrentSetChecked( tr_torrent * tor, time_t when ) { - const tr_file * file = &tor->info.files[fileIndex]; - const tr_piece_index_t begin = file->firstPiece; - const tr_piece_index_t end = file->lastPiece + 1; + tr_piece_index_t i, n; assert( tr_isTorrent( tor ) ); - if( isChecked ) - tr_bitfieldAddRange( &tor->checkedPieces, begin, end ); - else - tr_bitfieldRemRange( &tor->checkedPieces, begin, end ); + for( i=0, n=tor->info.pieceCount; i!=n; ++i ) + tor->info.pieces[i].timeChecked = when; } tr_bool -tr_torrentIsFileChecked( const tr_torrent * tor, - tr_file_index_t fileIndex ) -{ - const tr_file * file = &tor->info.files[fileIndex]; - const tr_piece_index_t begin = file->firstPiece; - const tr_piece_index_t end = file->lastPiece + 1; - tr_piece_index_t i; - tr_bool isChecked = TRUE; - - assert( tr_isTorrent( tor ) ); - - for( i = begin; isChecked && i < end; ++i ) - if( !tr_torrentIsPieceChecked( tor, i ) ) - isChecked = FALSE; - - return isChecked; -} - -void -tr_torrentUncheck( tr_torrent * tor ) +tr_torrentCheckPiece( tr_torrent * tor, tr_piece_index_t pieceIndex ) { - assert( tr_isTorrent( tor ) ); - - tr_bitfieldRemRange( &tor->checkedPieces, 0, tor->info.pieceCount ); -} + const tr_bool pass = tr_ioTestPiece( tor, pieceIndex ); -int -tr_torrentCountUncheckedPieces( const tr_torrent * tor ) -{ - assert( tr_isTorrent( tor ) ); + tr_tordbg( tor, "[LAZY] tr_torrentCheckPiece tested piece %zu, pass==%d", (size_t)pieceIndex, (int)pass ); + tr_torrentSetHasPiece( tor, pieceIndex, pass ); + tr_torrentSetPieceChecked( tor, pieceIndex ); + tor->anyDate = tr_time( ); + tr_torrentSetDirty( tor ); - return tor->info.pieceCount - tr_bitfieldCountTrueBits( &tor->checkedPieces ); + return pass; } -time_t* -tr_torrentGetMTimes( const tr_torrent * tor, size_t * setme_n ) +static time_t +getFileMTime( const tr_torrent * tor, tr_file_index_t i ) { - size_t i; - const size_t n = tor->info.fileCount; - time_t * m = tr_new0( time_t, n ); - - assert( tr_isTorrent( tor ) ); + struct stat sb; + time_t mtime = 0; + char * path = tr_torrentFindFile( tor, i ); - for( i = 0; i < n; ++i ) + if( ( path != NULL ) && !stat( path, &sb ) && S_ISREG( sb.st_mode ) ) { - struct stat sb; - char * path = tr_torrentFindFile( tor, i ); - if( ( path != NULL ) && !stat( path, &sb ) && S_ISREG( sb.st_mode ) ) - { #ifdef SYS_DARWIN - m[i] = sb.st_mtimespec.tv_sec; + mtime = sb.st_mtimespec.tv_sec; #else - m[i] = sb.st_mtime; + mtime = sb.st_mtime; #endif + } + + tr_free( path ); + return mtime; +} + +tr_bool +tr_torrentPieceNeedsCheck( const tr_torrent * tor, tr_piece_index_t p ) +{ + uint64_t unused; + tr_file_index_t f; + const tr_info * inf = tr_torrentInfo( tor ); + + /* if we've never checked this piece, then it needs to be checked */ + if( !inf->pieces[p].timeChecked ) { + tr_tordbg( tor, "[LAZY] piece %zu needs to be tested because it's never been tested", (size_t)p ); + return TRUE; + } + + /* If we think we've completed one of the files in this piece, + * but it's been modified since we last checked it, + * then it needs to be rechecked */ + tr_ioFindFileLocation( tor, p, 0, &f, &unused ); + for( ; f < inf->fileCount && pieceHasFile( p, &inf->files[f] ); ++f ) { + if( tr_cpFileIsComplete( &tor->completion, f ) ) { + if( getFileMTime( tor, f ) > inf->pieces[p].timeChecked ) { + tr_tordbg( tor, "[LAZY] piece %zu needs to be tested because file %zu mtime is newer than check time %zu", (size_t)p, (size_t)f, (size_t)inf->pieces[p].timeChecked ); + return TRUE; + } } - tr_free( path ); } - *setme_n = n; - return m; + tr_tordbg( tor, "[LAZY] piece %zu does not need to be tested", (size_t)p ); + return FALSE; } /*** diff --git a/libtransmission/torrent.h b/libtransmission/torrent.h index dbb8a89ff..f7274f603 100644 --- a/libtransmission/torrent.h +++ b/libtransmission/torrent.h @@ -86,23 +86,10 @@ void tr_torrentInitFilePriority( tr_torrent * tor, tr_file_index_t fileIndex, tr_priority_t priority ); -int tr_torrentCountUncheckedPieces( const tr_torrent * ); - -tr_bool tr_torrentIsFileChecked( const tr_torrent * tor, - tr_file_index_t file ); - void tr_torrentSetPieceChecked( tr_torrent * tor, - tr_piece_index_t piece, - tr_bool isChecked ); - -void tr_torrentSetFileChecked( tr_torrent * tor, - tr_file_index_t file, - tr_bool isChecked ); - -void tr_torrentUncheck( tr_torrent * tor ); + tr_piece_index_t piece ); -time_t* tr_torrentGetMTimes( const tr_torrent * tor, - size_t * setmeCount ); +void tr_torrentSetChecked( tr_torrent * tor, time_t when ); tr_torrent* tr_torrentNext( tr_session * session, tr_torrent * current ); @@ -190,7 +177,6 @@ struct tr_torrent struct tr_completion completion; - struct tr_bitfield checkedPieces; tr_completeness completeness; struct tr_torrent_tiers * tiers; @@ -261,8 +247,6 @@ struct tr_torrent uint16_t idleLimitMinutes; tr_idlelimit idleLimitMode; tr_bool finishedSeedingByIdle; - - uint64_t preVerifyTotal; }; /* get the index of this piece's first block */ @@ -353,12 +337,6 @@ static inline tr_bool tr_torrentAllowsLPD( const tr_torrent * tor ) && ( !tr_torrentIsPrivate( tor ) ); } -static inline tr_bool tr_torrentIsPieceChecked( const tr_torrent * tor, - tr_piece_index_t i ) -{ - return tr_bitfieldHasFast( &tor->checkedPieces, i ); -} - /*** **** ***/ @@ -431,5 +409,18 @@ void tr_torrentGotNewInfoDict( tr_torrent * tor ); void tr_torrentSetSpeedLimit_Bps ( tr_torrent *, tr_direction, int Bps ); int tr_torrentGetSpeedLimit_Bps ( const tr_torrent *, tr_direction ); +/** + * @return true if this piece needs to be tested + */ +tr_bool tr_torrentPieceNeedsCheck( const tr_torrent * tor, tr_piece_index_t pieceIndex ); + +/** + * @brief Test a piece against its info dict checksum + * @return true if the piece's passes the checksum test + */ +tr_bool tr_torrentCheckPiece( tr_torrent * tor, tr_piece_index_t pieceIndex ); + +uint64_t tr_torrentGetCurrentSizeOnDisk( const tr_torrent * tor ); + #endif diff --git a/libtransmission/transmission.h b/libtransmission/transmission.h index 9d37f4c49..d1fc66ce3 100644 --- a/libtransmission/transmission.h +++ b/libtransmission/transmission.h @@ -1609,6 +1609,7 @@ tr_file; /** @brief a part of tr_info that represents a single piece of the torrent's content */ typedef struct tr_piece { + time_t timeChecked; /* the last time we tested this piece */ uint8_t hash[SHA_DIGEST_LENGTH]; /* pieces hash */ int8_t priority; /* TR_PRI_HIGH, _NORMAL, or _LOW */ int8_t dnd; /* nonzero if the piece shouldn't be diff --git a/libtransmission/verify.c b/libtransmission/verify.c index 2cd3ec9a5..94886c1e2 100644 --- a/libtransmission/verify.c +++ b/libtransmission/verify.c @@ -10,14 +10,9 @@ * $Id$ */ -#include /* S_ISREG */ -#include - #ifdef HAVE_POSIX_FADVISE #define _XOPEN_SOURCE 600 -#endif -#if defined(HAVE_POSIX_FADVISE) || defined(SYS_DARWIN) - #include /* posix_fadvise() / fcntl() */ + #include /* posix_fadvise() */ #endif #include @@ -25,14 +20,12 @@ #include "transmission.h" #include "completion.h" #include "fdlimit.h" -#include "inout.h" #include "list.h" -#include "platform.h" +#include "platform.h" /* tr_lock() */ #include "torrent.h" -#include "utils.h" /* tr_buildPath */ +#include "utils.h" /* tr_valloc(), tr_free() */ #include "verify.h" - /*** **** ***/ @@ -42,11 +35,10 @@ enum MSEC_TO_SLEEP_PER_SECOND_DURING_VERIFY = 100 }; -/* #define STOPWATCH */ - static tr_bool verifyTorrent( tr_torrent * tor, tr_bool * stopFlag ) { + time_t end; SHA_CTX sha; int fd = -1; int64_t filePos = 0; @@ -58,14 +50,13 @@ verifyTorrent( tr_torrent * tor, tr_bool * stopFlag ) tr_file_index_t prevFileIndex = !fileIndex; tr_piece_index_t pieceIndex = 0; const time_t begin = tr_time( ); - time_t end; const size_t buflen = 1024 * 128; /* 128 KiB buffer */ uint8_t * buffer = tr_valloc( buflen ); - tr_torrentUncheck( tor ); - SHA1_Init( &sha ); + tr_tordbg( tor, "%s", "[LAZY] verifying torrent..." ); + tr_torrentSetChecked( tor, 0 ); while( !*stopFlag && ( pieceIndex < tor->info.pieceCount ) ) { uint32_t leftInPiece; @@ -75,17 +66,13 @@ verifyTorrent( tr_torrent * tor, tr_bool * stopFlag ) /* if we're starting a new piece... */ if( piecePos == 0 ) - { hadPiece = tr_cpPieceIsComplete( &tor->completion, pieceIndex ); - /* fprintf( stderr, "starting piece %d of %d\n", (int)pieceIndex, (int)tor->info.pieceCount ); */ - } /* if we're starting a new file... */ if( !filePos && (fd<0) && (fileIndex!=prevFileIndex) ) { char * filename = tr_torrentFindFile( tor, fileIndex ); fd = filename == NULL ? -1 : tr_open_file_for_scanning( filename ); - /* fprintf( stderr, "opening file #%d (%s) -- %d\n", fileIndex, filename, fd ); */ tr_free( filename ); prevFileIndex = fileIndex; } @@ -95,7 +82,6 @@ verifyTorrent( tr_torrent * tor, tr_bool * stopFlag ) leftInFile = file->length - filePos; bytesThisPass = MIN( leftInFile, leftInPiece ); bytesThisPass = MIN( bytesThisPass, buflen ); - /* fprintf( stderr, "reading this pass: %d\n", (int)bytesThisPass ); */ /* read a bit */ if( fd >= 0 ) { @@ -124,17 +110,12 @@ verifyTorrent( tr_torrent * tor, tr_bool * stopFlag ) SHA1_Final( hash, &sha ); hasPiece = !memcmp( hash, tor->info.pieces[pieceIndex].hash, SHA_DIGEST_LENGTH ); - /* fprintf( stderr, "do the hashes match? %s\n", (hasPiece?"yes":"no") ); */ - - if( hasPiece ) { - tr_torrentSetHasPiece( tor, pieceIndex, TRUE ); - if( !hadPiece ) - changed = TRUE; - } else if( hadPiece ) { - tr_torrentSetHasPiece( tor, pieceIndex, FALSE ); - changed = TRUE; + + if( hasPiece || hadPiece ) { + tr_torrentSetHasPiece( tor, pieceIndex, hasPiece ); + changed |= hasPiece != hadPiece; } - tr_torrentSetPieceChecked( tor, pieceIndex, TRUE ); + tr_torrentSetPieceChecked( tor, pieceIndex ); now = tr_time( ); tor->anyDate = now; @@ -153,12 +134,12 @@ verifyTorrent( tr_torrent * tor, tr_bool * stopFlag ) /* if we're finishing a file... */ if( leftInFile == 0 ) { - /* fprintf( stderr, "closing file\n" ); */ if( fd >= 0 ) { tr_close_file( fd ); fd = -1; } ++fileIndex; filePos = 0; } } + tr_tordbg( tor, "%s", "[LAZY] DONE verifying torrent..." ); /* cleanup */ if( fd >= 0 ) @@ -168,7 +149,8 @@ verifyTorrent( tr_torrent * tor, tr_bool * stopFlag ) /* stopwatch */ end = tr_time( ); tr_tordbg( tor, "it took %d seconds to verify %"PRIu64" bytes (%"PRIu64" bytes per second)", - (int)(end-begin), tor->info.totalSize, (uint64_t)(tor->info.totalSize/(1+(end-begin))) ); + (int)(end-begin), tor->info.totalSize, + (uint64_t)(tor->info.totalSize/(1+(end-begin))) ); return changed; } @@ -202,7 +184,6 @@ static tr_lock* getVerifyLock( void ) { static tr_lock * lock = NULL; - if( lock == NULL ) lock = tr_lockNew( ); return lock; @@ -213,8 +194,8 @@ verifyThreadFunc( void * unused UNUSED ) { for( ;; ) { - int changed = 0; - tr_torrent * tor; + int changed = 0; + tr_torrent * tor; struct verify_node * node; tr_lockLock( getVerifyLock( ) ); @@ -250,28 +231,6 @@ verifyThreadFunc( void * unused UNUSED ) tr_lockUnlock( getVerifyLock( ) ); } -static uint64_t -getCurrentSize( tr_torrent * tor ) -{ - tr_file_index_t i; - uint64_t byte_count = 0; - const tr_file_index_t n = tor->info.fileCount; - - for( i=0; itorrent ); - const tr_priority_t pb = tr_torrentGetPriority( b->torrent ); + const tr_priority_t pa = tr_torrentGetPriority( a->torrent ); + const tr_priority_t pb = tr_torrentGetPriority( b->torrent ); if( pa != pb ) return pa > pb ? -1 : 1; - /* smaller size comes before larger size... smaller sizes are faster to verify */ + /* smaller torrents come before larger ones because they verify faster */ if( a->current_size < b->current_size ) return -1; if( a->current_size > b->current_size ) return 1; return 0; } void -tr_verifyAdd( tr_torrent * tor, - tr_verify_done_cb verify_done_cb ) +tr_verifyAdd( tr_torrent * tor, tr_verify_done_cb verify_done_cb ) { - assert( tr_isTorrent( tor ) ); - - if( tr_torrentCountUncheckedPieces( tor ) == 0 ) - { - /* doesn't need to be checked... */ - fireCheckDone( tor, verify_done_cb ); - } - else - { - const uint64_t current_size = getCurrentSize( tor ); - - if( !current_size ) - { - /* we haven't downloaded anything for this torrent yet... - * no need to leave it waiting in the back of the queue. - * we can mark it as all-missing from here and fire - * the "done" callback */ - const tr_bool hadAny = tr_cpHaveTotal( &tor->completion ) != 0; - tr_piece_index_t i; - for( i=0; iinfo.pieceCount; ++i ) { - tr_torrentSetHasPiece( tor, i, FALSE ); - tr_torrentSetPieceChecked( tor, i, TRUE ); - } - if( hadAny ) /* if we thought we had some, flag as dirty */ - tr_torrentSetDirty( tor ); - fireCheckDone( tor, verify_done_cb ); - } - else - { - struct verify_node * node; + struct verify_node * node; - tr_torinf( tor, "%s", _( "Queued for verification" ) ); + assert( tr_isTorrent( tor ) ); + tr_torinf( tor, "%s", _( "Queued for verification" ) ); - node = tr_new( struct verify_node, 1 ); - node->torrent = tor; - node->verify_done_cb = verify_done_cb; - node->current_size = current_size; + node = tr_new( struct verify_node, 1 ); + node->torrent = tor; + node->verify_done_cb = verify_done_cb; + node->current_size = tr_torrentGetCurrentSizeOnDisk( tor ); - tr_lockLock( getVerifyLock( ) ); - tr_torrentSetVerifyState( tor, TR_VERIFY_WAIT ); - tr_list_insert_sorted( &verifyList, node, compareVerifyByPriorityAndSize ); - if( verifyThread == NULL ) - verifyThread = tr_threadNew( verifyThreadFunc, NULL ); - tr_lockUnlock( getVerifyLock( ) ); - } - } + tr_lockLock( getVerifyLock( ) ); + tr_torrentSetVerifyState( tor, TR_VERIFY_WAIT ); + tr_list_insert_sorted( &verifyList, node, compareVerifyByPriorityAndSize ); + if( verifyThread == NULL ) + verifyThread = tr_threadNew( verifyThreadFunc, NULL ); + tr_lockUnlock( getVerifyLock( ) ); } static int -- 2.40.0