AGNFolderDownloader - Class for downloading a folder of files from a remote server. It will also make sure that a folder that is being updated is not corrupted whilst downloading. Folders are created in a temporary folder and then moved into place when completed.

AGNFolderDownloader.h

//
//  AGNFolderDownloader.h
//  Agnitio iPlanner
//
//  Created by Matt Gough on 11/03/2013.
//  Copyright (c) 2013 Agnitio. All rights reserved.
//

#import <Foundation/Foundation.h>

/*!
 @brief AGNDownloaderFileInfo maintains information about each individual file in a folder downloder operation. 
 */
@interface AGNDownloaderFileInfo : NSObject

/*!
 @brief Creates a new AGNDownloaderFileInfo
 @param relativePath  	The relative path of the file from its base URL
 @param size				The expected size of the file
 @param modificationTime	The modification time of the file (in UNIX) time
 */
- (AGNDownloaderFileInfo*)initWithRelativePath:(NSString*)relativePath
										  size:(uint64_t)size
							  modificationTime:(NSUInteger)modificationTime;

@property (nonatomic, readonly) NSString*	relativePath;
@property (nonatomic, readonly) int64_t		size;
@property (nonatomic, readonly) NSUInteger	modificationTime;

/*!
 Creates an array of AGNDownloaderFileInfo objects for existing files in the specified file URL
 
 @param rootFileURL The URL of the local folder to scan.
 @result An array of AGNDownloaderFileInfo objects.
 */
+ (NSArray*)arrayOfFileInfoForFileURL:(NSURL*)rootFileURL;
@end

@class AGNFolderDownloader;

//! @brief Delegate methods for use by AGNFolderDownloader
@protocol AGNFolderDownloaderDelegate <NSObject>

@optional
/*!
 @discussion Called if there are local files that are not known on the remote side. Gives you a chance to delete
 those files. If you do not implement this method (and delete such files) they will be left alone.
 
 @param downloader		The folder downloader
 @param relativePaths	An Array of relative path strings of obsolete files
 */
- (void)folderDownloader:(AGNFolderDownloader*)downloader didFindObsoleteLocalFiles:(NSArray*)relativePaths;

/*!
 @discussion Called just before starting to download the first file. Will not be called if no files need downloading.
 i.e they are all up-to-date.
 
 @param downloader		The folder downloader
 */
- (void)folderDownloaderDidStartDownloading:(AGNFolderDownloader*)downloader;

/*!
 @discussion Called when the downloader has received data. To throttle the number of such calls, consider setting
 AGNFolderDownloader.didUpdateBytesReceivedThrottleTimeInterval to coalesce multiple calls into one when using this
 information to update UI.
 
 @param downloader		The folder downloader
 @param bytesReceived	The total number of bytes received so far for the entire download operation across all files.
 @param delta			The difference in byte count since the last call to this method. MAY be negative if
						a failed download had to be retried.
 */
- (void)folderDownloader:(AGNFolderDownloader*)downloader didUpdateBytesReceived:(int64_t)bytesReceived delta:(int64_t)delta;

/*!
 @discussion Called when the downloader has finished the download of each individual file.
 
 @param downloader		The folder downloader
 @param fileURL			The URL of the local file
 @param error			Any error associated with the download

 */
- (void)folderDownloader:(AGNFolderDownloader*)downloader didFinishDownloadOfFile:(NSURL*)fileURL error:(NSError*)error;

/*!
 @discussion Called when the download has finished the entire download of all files. Will still be called if no files
 actually needed downloading. i.e they are all up-to-date. To determine if there were any failures, check AGNFolderDownloader.failedURLs
 
 @param downloader		The folder downloader
 */
- (void)folderDownloaderDidFinish:(AGNFolderDownloader*)downloader;

@end

/*!
 @brief AGNFolderDownloader handles replicating a folder from one root URL to the local file system
 
 @discussion There are 2 stages to its use. During the first stage, you tell it the files to download
 via addRemoteFileInfo. Once all of them have been added you then call startDownloading
 to get it to do the downloading of only those files that need downloading.
 
 Downloading of individual files is handled on a concurrent dispatch queue appropriate to the file size.
 Smaller files end up on a queue with more concurrent operations since latency is an issue for such requests.
 Larger files go on queues with fewer concurrent operations.
 
 You can have multiple downloaders in-flight at the same time; they all share the same download queues.
 */
@interface AGNFolderDownloader : NSObject <AGNNetworkDataReceiver>

//! Returns all the NSOperationQueues used by this class. For debugging purposes only
+ (NSArray*)allDownloaderOperationQueues;

//! Returns the NSOperationQueue that would be used for a file of a certain size. For debugging purposes only
+ (NSOperationQueue*)operationQueueForDownloadOfSize:(uint64_t)size;

/*!
 @brief Create a new Downloader object
 
 @param remoteRootURL		The root URL of the remote files to download from
 @param localRootFileURL	The root URL of the local folder to download to
 */
- (AGNFolderDownloader*)initWithRemoteRootURL:(NSURL*)remoteRootURL
							 localRootFileURL:(NSURL*)localRootFileURL;

/*!
 @brief Create a new Downloader object
 
 @param remoteRootURL		The root URL of the remote files to download from
 @param localRootFileURL	The root URL of the local folder to download to
 */
+ (AGNFolderDownloader*)folderDownloaderWithRemoteRootURL:(NSURL*)remoteRootURL
										 localRootFileURL:(NSURL*)localRootFileURL;

/*!
 @brief Add a remote file to the list of files to consider for download.
 @discussion You must add all remote files to the downloaded before calling startDownloading.
 
 @param remoteFileInfo The info about the remote file.
 */
- (void)addRemoteFileInfo:(AGNDownloaderFileInfo*)remoteFileInfo;

/*!
 @brief Initiates the downloading of new and modified files
 @discussion Ensure that you have added all relevant remote files before calling this.
 Calling this method twice on the same downloader is undefined.
 @see startedDownloading
 */
- (void)startDownloading;

/*!
 @brief Cancels all pending and inflight downloads for the downloader.
 @discussion Sorry, I can't rememeber now what delegate methods will/won't get
 called due to cancellation. 
 */
- (void)cancelDownloading;

//! The URL of the remote Root folder
@property (nonatomic, readonly) NSURL* remoteRootURL;

//! The URL of the local Root folder
@property (nonatomic, readonly) NSURL* localRootFileURL;

//! The AGNFolderDownloader's delegate
@property (nonatomic, weak) id <AGNFolderDownloaderDelegate> delegate;

//! Use this to throttle delegate calls to folderDownloader:didUpdateBytesReceived:delta
@property (nonatomic, assign) NSTimeInterval didUpdateBytesReceivedThrottleTimeInterval;

//! An object for your own use
@property (nonatomic, strong) id context;

//! Returns true if startDownloading has been called
@property (nonatomic, readonly) BOOL startedDownloading;

// The following have no meaning before startDownloading is called

//! The total size of all files on the remote side, regardless of whether they need downloading or not
@property (nonatomic, readonly) int64_t totalRemoteSize;

/*!
 The total number of bytes up to date on the local side, including files currently downloading.
 This amount may decrease during downloading if downloads need retrying or fail.
 */
@property (nonatomic, readonly) int64_t upToDateSize;

/*! 
 The total number of files that need downloading (i.e not those already up to date).
 This value does not change as individual files complete.
 */
@property (nonatomic, readonly) NSUInteger totalFilesToDownload;

//! The number of file downloads that have been attempted and have finished (regardless of their success or failure)
@property (nonatomic, readonly) NSUInteger fileDownloadsAttempted;

//! An array of remote NSURLs that failed to download
@property (nonatomic, readonly) NSArray* failedURLs;

@end

AGNFolderDownloader.m

//
//  AGNFolderDownloader.m
//  Agnitio iPlanner
//
//  Created by Matt Gough on 11/03/2013.
//  Copyright (c) 2013 Agnitio. All rights reserved.
//
#if ! __has_feature(objc_arc)
#warning This file must be compiled with ARC. Use -fobjc-arc flag (or convert project to ARC).
#endif

#import "AGNFolderDownloader.h"
#import <libkern/OSAtomic.h>

@implementation AGNDownloaderFileInfo

- (AGNDownloaderFileInfo*)initWithRelativePath:(NSString*)relativePath
  									  size:(unsigned long long)size
							  modificationTime:(NSUInteger)modificationTime
{
	self = [super init];
	if (self)
	{
		_relativePath = [relativePath copy];
		_size = size;
		_modificationTime = modificationTime;
	}
	return self;
}

- (NSString*)debugDescription
{
	NSString* result = [NSString stringWithFormat:@"%@ \"%@\" sz:%lld md:%@", [super debugDescription], _relativePath, _size, [NSDate dateWithTimeIntervalSince1970:_modificationTime]];
	return result;
}


+ (NSArray*)arrayOfFileInfoForFileURL:(NSURL*)rootFileURL
{
	NSMutableArray* result = [NSMutableArray array];
	
	NSFileManager* fm = [[NSFileManager alloc] init];
	
	NSDirectoryEnumerator* dirEnumerator = [fm enumeratorAtURL:rootFileURL
									includingPropertiesForKeys:@[NSURLIsDirectoryKey, NSURLContentModificationDateKey, NSURLFileSizeKey]
													   options:0
												  errorHandler:^BOOL(NSURL *url, NSError *error) {
													  AGNLogLError(@"%@", error);
													  return YES;
												  }];
	
	NSUInteger rootCharsToRemove = [[rootFileURL path] length] + 1;
	
	for (NSURL* oneItem in dirEnumerator)
	{
		NSNumber* value = nil;
		NSError* error = nil;
		if ([oneItem getResourceValue:&value forKey:NSURLIsDirectoryKey error:&error] && [value boolValue])
			continue;
		
		NSUInteger modTime = 0;
		NSDate* modDate = nil;
		if ([oneItem getResourceValue:&modDate forKey:NSURLContentModificationDateKey error:&error])
			modTime = [modDate timeIntervalSince1970];
		
		long long size = 0;
		if ([oneItem getResourceValue:&value forKey:NSURLFileSizeKey error:&error])
			size = [value longLongValue];
		
		NSString* itemPath = [oneItem path];
		
		NSString* relativePath = [itemPath substringFromIndex:rootCharsToRemove];
		[result addObject:[[AGNDownloaderFileInfo alloc] initWithRelativePath:relativePath size:size modificationTime:modTime]];
	}
	
	
	return result;
}


@end



/*	The purpose of AGNFolderDownloaderNetworkObjectGenerator is to lazily generate
	the required AGNNetworkObjects needed for a downloader.
 
	startDownloading adds its generator to a special queue to ensure that when multiple downloaders
	are started (i.e when syncing a lot of presentations) the underlying network objects do not need
	creating up front, but only when it is time for the downloading to really start.
 
	Early during development, I was seeing many tens of thousands of network objects sitting in a queue for all pending
	presentation syncing.
 
 */
@interface AGNFolderDownloaderNetworkObjectGenerator : AGNGenericConcurrentOperation
{
	AGNFolderDownloader*	_owningDownloader;
	NSArray*				_downloaderFileInfo;
}

- (id)initWithDownloader:(AGNFolderDownloader*)owningDownloader filesToDownload:(NSArray*)downloaderFileInfo;

@end

@implementation AGNFolderDownloaderNetworkObjectGenerator

- (id)initWithDownloader:(AGNFolderDownloader*)owningDownloader filesToDownload:(NSArray*)downloaderFileInfo
{
	self = [super init];
	if (self)
	{
		_owningDownloader = owningDownloader; // NOT retained - Avoid cycles
		_downloaderFileInfo = downloaderFileInfo;
	}
	
	return self;
}


- (void)dealloc
{
	_owningDownloader = nil;
}

- (void)main
{
	for (AGNDownloaderFileInfo* remoteInfo in _downloaderFileInfo)
	{
		if ([self isCancelled])
			break;
		
		AGNNetworkObject* request;
		request = [[AGNNetworkObject alloc] initWithURL:[_owningDownloader.remoteRootURL URLWithUnencodedRelativeString:remoteInfo.relativePath]
											   delegate:_owningDownloader];
		
		request.fileURLForReceivingData = [_owningDownloader.localRootFileURL URLWithUnencodedRelativeString:remoteInfo.relativePath];
											  		
		NSOperationQueue* queue = [AGNFolderDownloader operationQueueForDownloadOfSize:remoteInfo.size];
		[queue addOperation:request];
	}
	// Finished with info - lets release
	_downloaderFileInfo = nil;
}

@end

@interface AGNFolderDownloader () {
	NSMutableDictionary*	_remoteFiles;
	int64_t					_upToDateSize;
	int64_t					_accumulatedDelta;
	NSTimeInterval			_lastDidUpdate;
	NSTimer*				_didUpdateTimer;
	NSMutableArray*			_failedURLs;
	NSUInteger				_totalFilesToDownload;
	NSUInteger				_fileDownloadsAttempted;
	AGNFolderDownloaderNetworkObjectGenerator* _netObjectGenerator;
	struct {
		// Only cache the oft-called methods
		unsigned int didUpdateBytesReceived:1;
		unsigned int didFinishDownloadOfFile:1;
	} _delegateSupports;

}

@end

@implementation AGNFolderDownloader

// All downloads get done in one of these queues. Each queue has a different number of concurrent
// operations. Smaller sized downloads go in a queue with more concurrent operations. This helps
// to reduce the latency overhead of small downloads compared to large ones.

static NSOperationQueue* sTinySizeQueue;
static NSOperationQueue* sMediumSizeQueue;
static NSOperationQueue* sLargeSizeQueue;
static NSOperationQueue* sHugeSizeQueue;

static NSOperationQueue* sNetworkObjectGeneratorQueue;

+ (void)prepareAllDownloaderOperationQueues
{
	static dispatch_once_t onceToken;
	dispatch_once(&onceToken, ^{
		sTinySizeQueue = [[NSOperationQueue alloc] init];
		[sTinySizeQueue setMaxConcurrentOperationCount:5];
		[sTinySizeQueue setName:@"Tiny Downloader request queue"];
		
		sMediumSizeQueue = [[NSOperationQueue alloc] init];
		[sMediumSizeQueue setMaxConcurrentOperationCount:3];
		[sMediumSizeQueue setName:@"Medium Downloader request queue"];
		
		sLargeSizeQueue = [[NSOperationQueue alloc] init];
		[sLargeSizeQueue setMaxConcurrentOperationCount:2];
		[sLargeSizeQueue setName:@"Large Downloader request queue"];
		
		sHugeSizeQueue = [[NSOperationQueue alloc] init];
		[sHugeSizeQueue setMaxConcurrentOperationCount:1];
		[sHugeSizeQueue setName:@"Huge Downloader request queue"];
		
		sNetworkObjectGeneratorQueue = [[NSOperationQueue alloc] init];
		[sNetworkObjectGeneratorQueue setMaxConcurrentOperationCount:3];
		[sNetworkObjectGeneratorQueue setName:@"Downloader NetworkObject generator queue"];
	});
}

+ (NSArray*)allDownloaderOperationQueues
{
	[self prepareAllDownloaderOperationQueues];
	return @[sTinySizeQueue, sMediumSizeQueue, sLargeSizeQueue, sHugeSizeQueue];
}

+ (NSOperationQueue*)operationQueueForDownloadOfSize:(unsigned long long)size
{
	[self prepareAllDownloaderOperationQueues];
	if (size <= 10 * 1024)
		return sTinySizeQueue;
	
	if (size <= 128 * 1024)
		return sMediumSizeQueue;
	
	if (size <= 1024 * 1024)
		return sLargeSizeQueue;
	
	return sHugeSizeQueue;
}


+ (NSURL*)fixRootURL:(NSURL*)rootURL
{
	NSURL* result = nil;
	NSString* absoluteString = [rootURL absoluteString];
	if ([absoluteString length] && ![absoluteString hasSuffix:@"/"])
	{
		/*	Since we use URLWithString:relativeToURL: for each file we want to downlaod, we need to ensure that the root URL
			is for a directory, otherwise URLWithString:relativeToURL: will nicely take away
			the last path component of the baseURL and leave us one level too high up.
		 */
		absoluteString = [absoluteString stringByAppendingString:@"/"];
		result = [NSURL URLWithString:absoluteString];
	}
	else
		result = rootURL;
	
	return result;
}

- (AGNFolderDownloader*)initWithRemoteRootURL:(NSURL*)remoteRootURL
							 localRootFileURL:(NSURL*)localRootFileURL
{
	self = [super init];
	if (self)
	{
		_remoteRootURL = [[[self class] fixRootURL:remoteRootURL] copy];
		_localRootFileURL = [[[self class] fixRootURL:localRootFileURL] copy];
		
		_remoteFiles = [[NSMutableDictionary alloc] init];
		_failedURLs = [[NSMutableArray alloc] init];
	}
	return self;
}

+ (AGNFolderDownloader*)folderDownloaderWithRemoteRootURL:(NSURL*)remoteRootURL
										 localRootFileURL:(NSURL*)localRootFileURL
{
	return [[self alloc] initWithRemoteRootURL:remoteRootURL localRootFileURL:localRootFileURL];
}


- (void) dealloc
{
	[_didUpdateTimer invalidate];
	[self cancelDownloading];
}

- (void)setDelegate:(id<AGNFolderDownloaderDelegate>)delegate
{
	_delegate = delegate;
	_delegateSupports.didUpdateBytesReceived = [delegate respondsToSelector:@selector(folderDownloader:didUpdateBytesReceived:delta:)];
	_delegateSupports.didFinishDownloadOfFile = [delegate respondsToSelector:@selector(folderDownloader:didFinishDownloadOfFile:error:)];
}

- (void)addRemoteFileInfo:(AGNDownloaderFileInfo*)remoteFileInfo
{
	_remoteFiles[remoteFileInfo.relativePath] = remoteFileInfo;
	_totalRemoteSize += remoteFileInfo.size;
}


- (NSArray*)arrayOfRemoteFileInfoToDownloadForLocalFiles:(NSDictionary*)localFiles
{
	NSMutableArray* result = [NSMutableArray array];
	for (NSString* relativePath in _remoteFiles)
	{
		AGNDownloaderFileInfo* remoteInfo = _remoteFiles[relativePath];
		AGNDownloaderFileInfo* localInfo = localFiles[relativePath];
		
		BOOL doDownload = YES;
		if (localInfo)
		{
			if ((remoteInfo.size == localInfo.size) && (remoteInfo.modificationTime < localInfo.modificationTime))
				doDownload = NO;
		}
		
		if (doDownload)
			[result addObject:remoteInfo];
		else
			_upToDateSize += remoteInfo.size;
	}
	return result;
}

- (NSArray*)relativePathsOfObsoleteLocalFiles:(NSDictionary*)localFiles
{
	NSMutableArray* obsolete = [NSMutableArray array];
	for (NSString* relativePath in localFiles)
	{
		if (!_remoteFiles[relativePath])
			[obsolete addObject:relativePath];
	}
	return obsolete;
}

- (void)startDownloading
{
	@autoreleasepool // This method generates a lot of temporaries along the way. Lets not let them live too long
	{
		// Scan local files
		NSArray* files = [AGNDownloaderFileInfo arrayOfFileInfoForFileURL:_localRootFileURL];
		NSMutableDictionary* localFiles = [[NSMutableDictionary alloc] initWithCapacity:[files count]];
		
		for (AGNDownloaderFileInfo* fileInfo in files)
			localFiles[fileInfo.relativePath] = fileInfo;
		
		NSArray* toDo = [self arrayOfRemoteFileInfoToDownloadForLocalFiles:localFiles];
		
		id <AGNFolderDownloaderDelegate> delegate = _delegate;
		if ([delegate respondsToSelector:@selector(folderDownloader:didFindObsoleteLocalFiles:)])
		{ // See if delegate wants to know about obsolete files
			NSArray* obsoleteLocalFiles = [self relativePathsOfObsoleteLocalFiles:localFiles];
			if ([obsoleteLocalFiles count])
				[delegate folderDownloader:self didFindObsoleteLocalFiles:obsoleteLocalFiles];
		}
		
		// We no longer need these - remove now to free up memory
		localFiles = nil;
		[_remoteFiles removeAllObjects];
		
		_totalFilesToDownload = [toDo count];
		_fileDownloadsAttempted = 0;
		_startedDownloading = YES;
		
		if (![toDo count])
		{
			if ([delegate respondsToSelector:@selector(folderDownloaderDidFinish:)])
				[delegate folderDownloaderDidFinish:self];
		}
		else
		{
			[[self class] prepareAllDownloaderOperationQueues];
			_netObjectGenerator = [[AGNFolderDownloaderNetworkObjectGenerator alloc] initWithDownloader:self filesToDownload:toDo];
			[sNetworkObjectGeneratorQueue addOperation:_netObjectGenerator];
			
			if ([delegate respondsToSelector:@selector(folderDownloaderDidStartDownloading:)])
				[delegate folderDownloaderDidStartDownloading:self];
		}
	}
}

- (void)cancelDownloading
{
	[_netObjectGenerator cancel];
	
	for (NSOperationQueue* queue in [[self class] allDownloaderOperationQueues])
	{
		for (AGNNetworkObject* operation in [queue operationsWithNetworkObjectTag:kAGNNetworkObjectMatchAnyTag andDelegate:self])
		{
			[operation cancel];
			operation.delegate = nil;
		}
	}
}


#pragma mark AGNNetworkObjectDelegates

- (void)requestFinishedWithObject:(AGNNetworkObject *)object
{
	NSError* error = object.error;
	if (error)
	{
		[self shouldRetryFailedRequestForObject:object]; // Get it to update byteCount
		@synchronized (_failedURLs)
		{
			[_failedURLs addObject:object.URL];
		}
	}
	NSAssert(sizeof(_fileDownloadsAttempted) == 4, @"NSUInteger is not 32 bits!)");
	NSUInteger doneSoFar = (NSUInteger)OSAtomicAdd32(1, (int32_t*)&_fileDownloadsAttempted);
	
	id <AGNFolderDownloaderDelegate> delegate = _delegate;
	if ([delegate respondsToSelector:@selector(folderDownloader:didFinishDownloadOfFile:error:)])
		[delegate folderDownloader:self didFinishDownloadOfFile:object.fileURLForReceivingData error:error];
	
	if (doneSoFar == _totalFilesToDownload)
	{ // Finished
		[_netObjectGenerator cancel];
		
		if (_didUpdateTimer) // Ensure delegate knows about most recent update to bytes.
			[self doDidUpdateDelegate:nil];
		
		if ([delegate respondsToSelector:@selector(folderDownloaderDidFinish:)])
			[delegate folderDownloaderDidFinish:self];
	}
}

- (void)cancelDidUpdateTimer
{
	[_didUpdateTimer invalidate];
	_didUpdateTimer = nil;
}

- (void)doDidUpdateDelegate:(NSTimer*)timer
{
	[self cancelDidUpdateTimer];
	AGNLogDebug(@"didUpdateBytesReceived:%lld delta:%lld", _upToDateSize, _accumulatedDelta);
	id <AGNFolderDownloaderDelegate> delegate = _delegate;
	[delegate folderDownloader:self didUpdateBytesReceived:_upToDateSize delta:_accumulatedDelta];
	_accumulatedDelta = 0;
	_lastDidUpdate = [NSDate timeIntervalSinceReferenceDate];

}

- (void)updateBytesReceivedWithDelta:(int64_t)bytesReceivedDelta
{
	OSAtomicAdd64(bytesReceivedDelta, &_upToDateSize);
	OSAtomicAdd64(bytesReceivedDelta, &_accumulatedDelta);
	
	if (_delegateSupports.didUpdateBytesReceived)
	{
		NSTimeInterval now = [NSDate timeIntervalSinceReferenceDate];
		
		if (_didUpdateTimer)
		{	// Cancel a previously delayed update, its already out of date
			[self cancelDidUpdateTimer];
		}
		
		if (fabs(now - _lastDidUpdate) < _didUpdateBytesReceivedThrottleTimeInterval)
		{
			// We've had another notification within our short delay time since the last actual update.
			// Just set it to happen after our delay.
			_didUpdateTimer = [NSTimer scheduledTimerWithTimeInterval:_didUpdateBytesReceivedThrottleTimeInterval
																target:self
															  selector:@selector(doDidUpdateDelegate:)
															  userInfo:nil
															   repeats:NO];
		}
		else
		{
			// Its been longer than our delay time since we updated. Just do it immediately
			[self doDidUpdateDelegate:nil];
		}
	}

}

- (void)receivedBytesForObject:(AGNNetworkObject *)object bytes:(NSUInteger)bytes
{
	[self updateBytesReceivedWithDelta:bytes];
}

- (BOOL)shouldRetryFailedRequestForObject:(AGNNetworkObject *)object
{
	if (object.bytesReceived)
	{
		[self updateBytesReceivedWithDelta:-object.bytesReceived];
	}
	return YES;
}


@end