From c97ace997a7f69f082bff2ad075237b4ae22a6f2 Mon Sep 17 00:00:00 2001 From: Michael Haney Date: Wed, 6 Feb 2019 14:10:58 -0800 Subject: [PATCH 1/8] Move the -removeLog API to be file-private, it's unused publicly. --- .../GoogleDataLogger/Classes/GDLLogStorage.h | 7 --- .../GoogleDataLogger/Classes/GDLLogStorage.m | 51 ++++++++++--------- .../Tests/Unit/GDLLogStorageTest.m | 9 ++-- 3 files changed, 33 insertions(+), 34 deletions(-) diff --git a/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.h b/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.h index 5622d03dc71..337adf4ba46 100644 --- a/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.h +++ b/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.h @@ -38,13 +38,6 @@ NS_ASSUME_NONNULL_BEGIN */ - (void)storeLog:(GDLLogEvent *)log; -/** Removes the corresponding log file from disk. - * - * @param logHash The hash value of the original log. - * @param logTarget The logTarget of the original log. - */ -- (void)removeLog:(NSNumber *)logHash logTarget:(NSNumber *)logTarget; - /** Removes a set of log fields specified by their filenames. * * @param logHashes The set of log files to remove. diff --git a/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m b/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m index ae243e1651f..4a7f07f69dd 100644 --- a/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m +++ b/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m @@ -105,30 +105,6 @@ - (void)storeLog:(GDLLogEvent *)log { }); } -- (void)removeLog:(NSNumber *)logHash logTarget:(NSNumber *)logTarget { - dispatch_async(_storageQueue, ^{ - NSURL *logFile = self.logHashToLogFile[logHash]; - - // Remove from disk, first and foremost. - NSError *error; - [[NSFileManager defaultManager] removeItemAtURL:logFile error:&error]; - GDLAssert(error == nil, @"There was an error removing a logFile: %@", error); - - // Remove from the tracking collections. - [self.logHashToLogFile removeObjectForKey:logHash]; - NSMutableSet *logHashes = self.logTargetToLogHashSet[logTarget]; - GDLAssert(logHashes, @"There wasn't a logSet for this logTarget."); - [logHashes removeObject:logHash]; - // It's fine to not remove the set if it's empty. - - // Check that a log prioritizer is available for this logTarget. - id logPrioritizer = - [GDLRegistrar sharedInstance].logTargetToPrioritizer[logTarget]; - GDLAssert(logPrioritizer, @"There's no prioritizer registered for the given logTarget."); - [logPrioritizer unprioritizeLog:logHash]; - }); -} - - (void)removeLogs:(NSSet *)logHashes logTarget:(NSNumber *)logTarget { dispatch_sync(_storageQueue, ^{ for (NSNumber *logHash in logHashes) { @@ -151,6 +127,33 @@ - (void)removeLogs:(NSSet *)logHashes logTarget:(NSNumber *)logTarge #pragma mark - Private helper methods +/** Removes the corresponding log file from disk. + * + * @param logHash The hash value of the original log. + * @param logTarget The logTarget of the original log. + */ +- (void)removeLog:(NSNumber *)logHash logTarget:(NSNumber *)logTarget { + NSURL *logFile = self.logHashToLogFile[logHash]; + + // Remove from disk, first and foremost. + NSError *error; + [[NSFileManager defaultManager] removeItemAtURL:logFile error:&error]; + GDLAssert(error == nil, @"There was an error removing a logFile: %@", error); + + // Remove from the tracking collections. + [self.logHashToLogFile removeObjectForKey:logHash]; + NSMutableSet *logHashes = self.logTargetToLogHashSet[logTarget]; + GDLAssert(logHashes, @"There wasn't a logSet for this logTarget."); + [logHashes removeObject:logHash]; + // It's fine to not remove the set if it's empty. + + // Check that a log prioritizer is available for this logTarget. + id logPrioritizer = + [GDLRegistrar sharedInstance].logTargetToPrioritizer[logTarget]; + GDLAssert(logPrioritizer, @"There's no prioritizer registered for the given logTarget."); + [logPrioritizer unprioritizeLog:logHash]; +} + /** Creates the log directory if it does not exist. */ - (void)createLogDirectoryIfNotExists { NSError *error; diff --git a/GoogleDataLogger/Tests/Unit/GDLLogStorageTest.m b/GoogleDataLogger/Tests/Unit/GDLLogStorageTest.m index aaa64b5bd4f..9b3f28adb00 100644 --- a/GoogleDataLogger/Tests/Unit/GDLLogStorageTest.m +++ b/GoogleDataLogger/Tests/Unit/GDLLogStorageTest.m @@ -112,7 +112,8 @@ - (void)testRemoveLog { logFile = [GDLLogStorage sharedInstance].logHashToLogFile[@(logHash)]; XCTAssertTrue([[NSFileManager defaultManager] fileExistsAtPath:logFile.path]); }); - [[GDLLogStorage sharedInstance] removeLog:@(logHash) logTarget:@(logTarget)]; + [[GDLLogStorage sharedInstance] removeLogs:[NSSet setWithObject:@(logHash)] + logTarget:@(logTarget)]; dispatch_sync([GDLLogStorage sharedInstance].storageQueue, ^{ XCTAssertFalse([[NSFileManager defaultManager] fileExistsAtPath:logFile.path]); XCTAssertEqual([GDLLogStorage sharedInstance].logHashToLogFile.count, 0); @@ -228,7 +229,8 @@ - (void)testLogEventDeallocationIsEnforced { }); // Ensure log was removed. - [[GDLLogStorage sharedInstance] removeLog:@(logEvent.hash) logTarget:@(logTarget)]; + NSNumber *logHash = @(logEvent.hash); + [[GDLLogStorage sharedInstance] removeLogs:[NSSet setWithObject:logHash] logTarget:@(logTarget)]; dispatch_sync([GDLLogStorage sharedInstance].storageQueue, ^{ XCTAssertFalse([[NSFileManager defaultManager] fileExistsAtPath:logFile.path]); XCTAssertEqual([GDLLogStorage sharedInstance].logHashToLogFile.count, 0); @@ -248,7 +250,8 @@ - (void)testNSSecureCoding { dispatch_sync([GDLLogStorage sharedInstance].storageQueue, ^{ XCTAssertNotNil([GDLLogStorage sharedInstance].logHashToLogFile[@(logHash)]); }); - [[GDLLogStorage sharedInstance] removeLog:@(logHash) logTarget:@(logTarget)]; + [[GDLLogStorage sharedInstance] removeLogs:[NSSet setWithObject:@(logHash)] + logTarget:@(logTarget)]; dispatch_sync([GDLLogStorage sharedInstance].storageQueue, ^{ XCTAssertNil([GDLLogStorage sharedInstance].logHashToLogFile[@(logHash)]); }); From e963d78928d9dc4e81bb3d7d1f8b82fc366fb5bf Mon Sep 17 00:00:00 2001 From: Michael Haney Date: Wed, 6 Feb 2019 14:11:29 -0800 Subject: [PATCH 2/8] Remove altering of in flight log set, that's done in the onComplete block --- GoogleDataLogger/Tests/Unit/GDLUploadCoordinatorTest.m | 1 - 1 file changed, 1 deletion(-) diff --git a/GoogleDataLogger/Tests/Unit/GDLUploadCoordinatorTest.m b/GoogleDataLogger/Tests/Unit/GDLUploadCoordinatorTest.m index 00bea3742b8..f375ee7f6f2 100644 --- a/GoogleDataLogger/Tests/Unit/GDLUploadCoordinatorTest.m +++ b/GoogleDataLogger/Tests/Unit/GDLUploadCoordinatorTest.m @@ -112,7 +112,6 @@ - (void)testForceUploadLogsEnqueuesIfLogTargetAlreadyHasLogsInFlight { target:_logTarget]); dispatch_sync([GDLUploadCoordinator sharedInstance].coordinationQueue, ^{ XCTAssertEqual([GDLUploadCoordinator sharedInstance].forcedUploadQueue.count, 1); - [[GDLUploadCoordinator sharedInstance].logTargetToInFlightLogSet removeAllObjects]; [GDLUploadCoordinator sharedInstance].onCompleteBlock( self.logTarget, [GDLClock clockSnapshotInTheFuture:1000], nil); }); From 8d64aac506e002574f74c8e50a114b22a03b8253 Mon Sep 17 00:00:00 2001 From: Michael Haney Date: Wed, 6 Feb 2019 14:12:31 -0800 Subject: [PATCH 3/8] Copy the set of logs given to upload so it's not altered while the pipeline is operating on it. --- .../GoogleDataLogger/Classes/GDLUploadCoordinator.m | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/GoogleDataLogger/GoogleDataLogger/Classes/GDLUploadCoordinator.m b/GoogleDataLogger/GoogleDataLogger/Classes/GDLUploadCoordinator.m index 2ec735d09bd..e4fe24e18ac 100644 --- a/GoogleDataLogger/GoogleDataLogger/Classes/GDLUploadCoordinator.m +++ b/GoogleDataLogger/GoogleDataLogger/Classes/GDLUploadCoordinator.m @@ -102,6 +102,7 @@ - (GDLUploaderCompletionBlock)onCompleteBlock { strongSelf->_logTargetToNextUploadTimes[logTarget] = nextUploadAttemptUTC; NSSet *logHashSet = [strongSelf->_logTargetToInFlightLogSet objectForKey:logTarget]; + GDLAssert(logHashSet, @"There should be an in-flight log set to remove."); [strongSelf.logStorage removeLogs:logHashSet logTarget:logTarget]; [strongSelf->_logTargetToInFlightLogSet removeObjectForKey:logTarget]; if (strongSelf->_forcedUploadQueue.count) { @@ -158,15 +159,16 @@ - (void)checkPrioritizersAndUploadLogs { GDLAssert(prioritizer && uploader, @"log target '%@' is missing an implementation", logTarget); GDLUploadConditions conds = [self uploadConditions]; - NSSet *logHashesToUpload = [prioritizer logsToUploadGivenConditions:conds]; + NSSet *logHashesToUpload = + [[prioritizer logsToUploadGivenConditions:conds] copy]; if (logHashesToUpload && logHashesToUpload.count > 0) { NSAssert(logHashesToUpload.count > 0, @""); NSSet *logFilesToUpload = [strongSelf.logStorage logHashesToFiles:logHashesToUpload]; NSAssert(logFilesToUpload.count == logHashesToUpload.count, @"There should be the same number of files to logs"); - [uploader uploadLogs:logFilesToUpload onComplete:self.onCompleteBlock]; strongSelf->_logTargetToInFlightLogSet[logTarget] = logHashesToUpload; + [uploader uploadLogs:logFilesToUpload onComplete:self.onCompleteBlock]; } } } From bf241e9b4f1ad78212f22a99390b4b5f354e6a6b Mon Sep 17 00:00:00 2001 From: Michael Haney Date: Wed, 6 Feb 2019 14:13:40 -0800 Subject: [PATCH 4/8] Implement an integration/E2E test of the library's pipeline Includes a dependency on GCDWebServer in the test_spec to run an HTTP server endpoint that the uploader can upload to. --- GoogleDataLogger.podspec | 11 +- .../Tests/Integration/GDLIntegrationTest.m | 173 ++++++++++++++++++ .../Helpers/GDLIntegrationTestPrioritizer.h | 27 +++ .../Helpers/GDLIntegrationTestPrioritizer.m | 75 ++++++++ .../Helpers/GDLIntegrationTestUploader.h | 33 ++++ .../Helpers/GDLIntegrationTestUploader.m | 74 ++++++++ .../Integration/TestServer/GDLTestServer.h | 55 ++++++ .../Integration/TestServer/GDLTestServer.m | 110 +++++++++++ scripts/build.sh | 7 + 9 files changed, 563 insertions(+), 2 deletions(-) create mode 100644 GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m create mode 100644 GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestPrioritizer.h create mode 100644 GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestPrioritizer.m create mode 100644 GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestUploader.h create mode 100644 GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestUploader.m create mode 100644 GoogleDataLogger/Tests/Integration/TestServer/GDLTestServer.h create mode 100644 GoogleDataLogger/Tests/Integration/TestServer/GDLTestServer.m diff --git a/GoogleDataLogger.podspec b/GoogleDataLogger.podspec index 5b1c8267a50..f3f2d74f3be 100644 --- a/GoogleDataLogger.podspec +++ b/GoogleDataLogger.podspec @@ -3,7 +3,6 @@ Pod::Spec.new do |s| s.version = '0.1.0' s.summary = 'Google Data Logging iOS SDK.' - s.description = <<-DESC Shared library for iOS SDK data logging needs. DESC @@ -38,8 +37,16 @@ Shared library for iOS SDK data logging needs. common_test_sources = ['GoogleDataLogger/Tests/Common/**/*.{h,m}'] # Unit test specs - s.test_spec do |test_spec| + s.test_spec 'Tests-Unit' do |test_spec| test_spec.requires_app_host = false test_spec.source_files = ['GoogleDataLogger/Tests/Unit/**/*.{h,m}'] + common_test_sources end + + # Integration test specs + s.test_spec 'Tests-Integration' do |test_spec| + test_spec.requires_app_host = false + test_spec.source_files = ['GoogleDataLogger/Tests/Integration/**/*.{h,m}'] + common_test_sources + test_spec.compiler_flags = '-DGDL_LOG_TRACE_ENABLED=1' + test_spec.dependency 'GCDWebServer' + end end diff --git a/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m b/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m new file mode 100644 index 00000000000..d2e72054f3e --- /dev/null +++ b/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m @@ -0,0 +1,173 @@ +/* + * Copyright 2019 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import + +#import "GDLIntegrationTestPrioritizer.h" +#import "GDLIntegrationTestUploader.h" +#import "GDLTestServer.h" + +#import "GDLLogStorage_Private.h" +#import "GDLUploadCoordinator+Testing.h" + +/** A test-only log object used in this integration test. */ +@interface GDLIntegrationTestLog : NSObject + +@end + +@implementation GDLIntegrationTestLog + +- (NSData *)protoBytes { + // In real usage, protobuf's -data method or a custom implementation using nanopb are used. + return [[NSString stringWithFormat:@"%@", [NSDate date]] dataUsingEncoding:NSUTF8StringEncoding]; +} + +@end + +/** A test-only log transformer. */ +@interface GDLIntegrationTestTransformer : NSObject + +@end + +@implementation GDLIntegrationTestTransformer + +- (GDLLogEvent *)transform:(GDLLogEvent *)logEvent { + // drop half the logs during transforming. + if (arc4random_uniform(2) == 1) { + logEvent = nil; + } + return logEvent; +} + +@end + +@interface GDLIntegrationTest : XCTestCase + +/** A test prioritizer. */ +@property(nonatomic) GDLIntegrationTestPrioritizer *prioritizer; + +/** A test uploader. */ +@property(nonatomic) GDLIntegrationTestUploader *uploader; + +/** The first test logger. */ +@property(nonatomic) GDLLogger *logger1; + +/** The second test logger. */ +@property(nonatomic) GDLLogger *logger2; + +@end + +@implementation GDLIntegrationTest + +- (void)tearDown { + dispatch_sync([GDLLogStorage sharedInstance].storageQueue, ^{ + XCTAssertEqual([GDLLogStorage sharedInstance].logHashToLogFile.count, 0); + }); +} + +- (void)testEndToEndLog { + XCTestExpectation *expectation = [self expectationWithDescription:@"server got the request"]; + expectation.assertForOverFulfill = NO; + + // Create the server. + GDLTestServer *testServer = [[GDLTestServer alloc] init]; + [testServer setResponseCompletedBlock:^(GCDWebServerRequest *_Nonnull request, + GCDWebServerResponse *_Nonnull response) { + [expectation fulfill]; + }]; + [testServer registerTestPaths]; + [testServer start]; + + // Create loggers. + self.logger1 = [[GDLLogger alloc] initWithLogMapID:@"logMap1" + logTransformers:nil + logTarget:kGDLIntegrationTestTarget]; + + self.logger2 = [[GDLLogger alloc] initWithLogMapID:@"logMap2" + logTransformers:nil + logTarget:kGDLIntegrationTestTarget]; + + // Create a prioritizer and uploader. + self.prioritizer = [[GDLIntegrationTestPrioritizer alloc] init]; + self.uploader = [[GDLIntegrationTestUploader alloc] initWithServerURL:testServer.serverURL]; + + // Set the interval to be much shorter than the standard timer. + [GDLUploadCoordinator sharedInstance].timerInterval = NSEC_PER_SEC / 100; + [GDLUploadCoordinator sharedInstance].timerLeeway = NSEC_PER_SEC / 1000; + + // Confirm no logs are in disk. + XCTAssertEqual([GDLLogStorage sharedInstance].logHashToLogFile.count, 0); + XCTAssertEqual([GDLLogStorage sharedInstance].logTargetToLogHashSet.count, 0); + + // Generate some logs data. + [self generateLogs]; + + // Confirm logs are on disk. + dispatch_sync([GDLLogStorage sharedInstance].storageQueue, ^{ + XCTAssertGreaterThan([GDLLogStorage sharedInstance].logHashToLogFile.count, 0); + XCTAssertGreaterThan([GDLLogStorage sharedInstance].logTargetToLogHashSet.count, 0); + }); + + // Confirm logs were sent and received. + [self waitForExpectations:@[ expectation ] timeout:10.0]; + + // Generate logs for a big. + NSUInteger lengthOfTestToRunInSeconds = 30; + [GDLUploadCoordinator sharedInstance].timerInterval = NSEC_PER_SEC / 5000; + [GDLUploadCoordinator sharedInstance].timerLeeway = NSEC_PER_SEC / 100; + dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); + dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, queue); + dispatch_source_set_timer(timer, DISPATCH_TIME_NOW, 1 * NSEC_PER_SEC, 0.1 * NSEC_PER_SEC); + dispatch_source_set_event_handler(timer, ^{ + static int numberOfTimesCalled = 0; + numberOfTimesCalled++; + if (numberOfTimesCalled < lengthOfTestToRunInSeconds) { + [self generateLogs]; + } else { + dispatch_source_cancel(timer); + } + }); + dispatch_resume(timer); + + // Run for a bit, a couple seconds longer than the previous bit. + [[NSRunLoop currentRunLoop] + runUntilDate:[NSDate dateWithTimeIntervalSinceNow:lengthOfTestToRunInSeconds + 2]]; + + [testServer stop]; +} + +/** Generates and logs a bunch of random logs. */ +- (void)generateLogs { + for (int i = 0; i < 50; i++) { + // Choose a random logger, and randomly choose if it's a telemetry log. + GDLLogger *logger = arc4random_uniform(2) ? self.logger1 : self.logger2; + BOOL isTelemetryLog = arc4random_uniform(2); + + // Create a log + GDLLogEvent *logEvent = [logger newEvent]; + logEvent.extension = [[GDLIntegrationTestLog alloc] init]; + + if (isTelemetryLog) { + [logger logTelemetryEvent:logEvent]; + } else { + [logger logDataEvent:logEvent]; + } + } +} + +@end diff --git a/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestPrioritizer.h b/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestPrioritizer.h new file mode 100644 index 00000000000..30489f12b9b --- /dev/null +++ b/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestPrioritizer.h @@ -0,0 +1,27 @@ +/* + * Copyright 2019 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import + +/** The integration test log target. Normally, you should use a value in GDLLogTargets.h. */ +static GDLLogTarget kGDLIntegrationTestTarget = 100; + +/** An integration test prioritization class. */ +@interface GDLIntegrationTestPrioritizer : NSObject + +@end diff --git a/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestPrioritizer.m b/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestPrioritizer.m new file mode 100644 index 00000000000..a8b4d94fd21 --- /dev/null +++ b/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestPrioritizer.m @@ -0,0 +1,75 @@ +/* + * Copyright 2019 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "GDLIntegrationTestPrioritizer.h" + +@interface GDLIntegrationTestPrioritizer () + +/** Logs that are only supposed to be uploaded whilst on wifi. */ +@property(nonatomic) NSMutableSet *wifiOnlyLogs; + +/** Logs that can be uploaded on any type of connection. */ +@property(nonatomic) NSMutableSet *nonWifiLogs; + +/** The queue on which this prioritizer operates. */ +@property(nonatomic) dispatch_queue_t queue; + +@end + +@implementation GDLIntegrationTestPrioritizer + +- (instancetype)init { + self = [super init]; + if (self) { + _queue = + dispatch_queue_create("com.google.GDLIntegrationTestPrioritizer", DISPATCH_QUEUE_SERIAL); + _wifiOnlyLogs = [[NSMutableSet alloc] init]; + _nonWifiLogs = [[NSMutableSet alloc] init]; + [[GDLRegistrar sharedInstance] registerPrioritizer:self logTarget:kGDLIntegrationTestTarget]; + } + return self; +} + +- (void)prioritizeLog:(GDLLogEvent *)logEvent { + dispatch_sync(_queue, ^{ + if (logEvent.qosTier == GDLLogQoSWifiOnly) { + [self.wifiOnlyLogs addObject:@(logEvent.hash)]; + } else { + [self.nonWifiLogs addObject:@(logEvent.hash)]; + } + }); +} + +- (void)unprioritizeLog:(NSNumber *)logHash { + dispatch_sync(_queue, ^{ + [self.wifiOnlyLogs removeObject:logHash]; + [self.nonWifiLogs removeObject:logHash]; + }); +} + +- (nonnull NSSet *)logsToUploadGivenConditions:(GDLUploadConditions)conditions { + __block NSSet *logs; + dispatch_sync(_queue, ^{ + if ((conditions & GDLUploadConditionWifiData) == GDLUploadConditionWifiData) { + logs = self.wifiOnlyLogs; + } else { + logs = self.nonWifiLogs; + } + }); + return logs; +} + +@end diff --git a/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestUploader.h b/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestUploader.h new file mode 100644 index 00000000000..6b42b22cd5b --- /dev/null +++ b/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestUploader.h @@ -0,0 +1,33 @@ +/* + * Copyright 2019 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import + +#import "GDLIntegrationTestPrioritizer.h" + +/** An integration test uploader. */ +@interface GDLIntegrationTestUploader : NSObject + +/** Instantiates an instance of this uploader with the given server URL. + * + * @param serverURL The server URL this uploader should upload to. + * @return An instance of this class. + */ +- (instancetype)initWithServerURL:(NSURL *)serverURL; + +@end diff --git a/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestUploader.m b/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestUploader.m new file mode 100644 index 00000000000..8216c920c74 --- /dev/null +++ b/GoogleDataLogger/Tests/Integration/Helpers/GDLIntegrationTestUploader.m @@ -0,0 +1,74 @@ +/* + * Copyright 2019 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "GDLIntegrationTestUploader.h" + +#import "GDLIntegrationTestPrioritizer.h" + +#import "GDLTestServer.h" + +@implementation GDLIntegrationTestUploader { + /** The current upload task. */ + NSURLSessionUploadTask *_currentUploadTask; + + /** The server URL to upload to. */ + NSURL *_serverURL; +} + +- (instancetype)initWithServerURL:(NSURL *)serverURL { + self = [super init]; + if (self) { + _serverURL = serverURL; + [[GDLRegistrar sharedInstance] registerUploader:self logTarget:kGDLIntegrationTestTarget]; + } + return self; +} + +- (void)uploadLogs:(NSSet *)logFiles onComplete:(GDLUploaderCompletionBlock)onComplete { + NSAssert(!_currentUploadTask, @"An upload shouldn't be initiated with another in progress."); + NSURL *serverURL = arc4random_uniform(2) ? [_serverURL URLByAppendingPathComponent:@"log"] + : [_serverURL URLByAppendingPathComponent:@"logBatch"]; + NSURLSession *session = [NSURLSession sharedSession]; + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:serverURL]; + request.HTTPMethod = @"POST"; + NSMutableData *uploadData = [[NSMutableData alloc] init]; + + NSLog(@"Uploading batch of %lu logs: ", (unsigned long)logFiles.count); + + // In real usage, you'd create an instance of whatever request proto your server needs. + for (NSURL *logFile in logFiles) { + NSData *fileData = [NSData dataWithContentsOfURL:logFile]; + NSAssert(fileData, @"A log file shouldn't be empty"); + [uploadData appendData:fileData]; + } + NSURLSessionUploadTask *uploadTask = + [session uploadTaskWithRequest:request + fromData:uploadData + completionHandler:^(NSData *_Nullable data, NSURLResponse *_Nullable response, + NSError *_Nullable error) { + NSLog(@"Batch upload complete."); + // Remove from the prioritizer if there were no errors. + NSAssert(!error, @"There should be no errors uploading logs: %@", error); + if (onComplete) { + // In real usage, the server would/should return a desired next upload time. + GDLClock *nextUploadTime = [GDLClock clockSnapshotInTheFuture:1000]; + onComplete(kGDLIntegrationTestTarget, nextUploadTime, error); + } + }]; + [uploadTask resume]; +} + +@end diff --git a/GoogleDataLogger/Tests/Integration/TestServer/GDLTestServer.h b/GoogleDataLogger/Tests/Integration/TestServer/GDLTestServer.h new file mode 100644 index 00000000000..cc47b5fc793 --- /dev/null +++ b/GoogleDataLogger/Tests/Integration/TestServer/GDLTestServer.h @@ -0,0 +1,55 @@ +/* + * Copyright 2019 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "GCDWebServer.h" +#import "GCDWebServerDataResponse.h" +#import "GCDWebServerFileResponse.h" + +NS_ASSUME_NONNULL_BEGIN + +@class GCDWebServerRequest; +@class GCDWebServerResponse; + +/** This class provides a hermetic test service that runs on the test device/simulator. */ +@interface GDLTestServer : NSObject + +/** The URL of the server. */ +@property(nonatomic, readonly) NSURL *serverURL; + +/** Just before responding, this block will be scheduled to run on a global queue. */ +@property(nonatomic, copy, nullable) void (^responseCompletedBlock) + (GCDWebServerRequest *request, GCDWebServerResponse *response); + +/** Registers the paths used for testing. */ +- (void)registerTestPaths; + +/** Starts the server. Can be called after calling `-stop`. */ +- (void)start; + +/** Stops the server. */ +- (void)stop; + +/** Returns YES if the server is running, NO otherwise. + * + * @return YES if the server is running, NO otherwise. + */ +- (BOOL)isRunning; + +@end + +NS_ASSUME_NONNULL_END diff --git a/GoogleDataLogger/Tests/Integration/TestServer/GDLTestServer.m b/GoogleDataLogger/Tests/Integration/TestServer/GDLTestServer.m new file mode 100644 index 00000000000..707f0fe79db --- /dev/null +++ b/GoogleDataLogger/Tests/Integration/TestServer/GDLTestServer.m @@ -0,0 +1,110 @@ +/* + * Copyright 2019 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "GDLTestServer.h" + +@interface GDLTestServer () + +/** The server object. */ +@property(nonatomic) GCDWebServer *server; + +// Redeclare as readwrite and mutable. +@property(nonatomic, readwrite) NSMutableDictionary *registeredTestPaths; + +@end + +@implementation GDLTestServer + +- (instancetype)init { + self = [super init]; + if (self) { + [GCDWebServer setLogLevel:3]; + _server = [[GCDWebServer alloc] init]; + _registeredTestPaths = [[NSMutableDictionary alloc] init]; + } + return self; +} + +- (void)dealloc { + [_server stop]; +} + +- (void)registerTestPaths { + [self registerLogPath]; + [self registerLogBatchPath]; +} + +- (void)start { + NSAssert(self.server.isRunning == NO, @"The server should not be already running."); + NSError *error; + [self.server + startWithOptions:@{GCDWebServerOption_Port : @0, GCDWebServerOption_BindToLocalhost : @YES} + error:&error]; + NSAssert(error == nil, @"Error when starting server: %@", error); +} + +- (void)stop { + NSAssert(self.server.isRunning, @"The server should be running before stopping."); + [self.server stop]; +} + +- (BOOL)isRunning { + return [self.server isRunning]; +} + +- (NSURL *)serverURL { + return _server.serverURL; +} + +#pragma mark - HTTP Path handling methods + +/** Registers the /log path, which responds with some JSON. */ +- (void)registerLogPath { + id processBlock = ^GCDWebServerResponse *(GCDWebServerRequest *request) { + GCDWebServerDataResponse *response = [[GCDWebServerDataResponse alloc] initWithHTML:@"Hello!"]; + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 1 * NSEC_PER_SEC), + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + if (self.responseCompletedBlock) { + self.responseCompletedBlock(request, response); + } + }); + return response; + }; + [self.server addHandlerForMethod:@"POST" + path:@"/log" + requestClass:[GCDWebServerRequest class] + processBlock:processBlock]; +} + +/** Registers the /logBatch path, which responds with some JSON. */ +- (void)registerLogBatchPath { + id processBlock = ^GCDWebServerResponse *(__kindof GCDWebServerRequest *request) { + GCDWebServerDataResponse *response = [[GCDWebServerDataResponse alloc] initWithHTML:@"Hello2!"]; + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 1 * NSEC_PER_SEC), + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + if (self.responseCompletedBlock) { + self.responseCompletedBlock(request, response); + } + }); + return response; + }; + [self.server addHandlerForMethod:@"POST" + path:@"/logBatch" + requestClass:[GCDWebServerRequest class] + processBlock:processBlock]; +} + +@end diff --git a/scripts/build.sh b/scripts/build.sh index 823d222aba4..c74860e548a 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -331,6 +331,13 @@ case "$product-$method-$platform" in "${xcb_flags[@]}" \ build \ test + + RunXcodebuild \ + -workspace 'GoogleDataLogger/gen/GoogleDataLogger/GoogleDataLogger.xcworkspace' \ + -scheme "GoogleDataLogger-Unit-EndToEnd" \ + "${xcb_flags[@]}" \ + build \ + test ;; *) From ce7cbf9842ae2ffaec34a76e288154ac82b5e4b2 Mon Sep 17 00:00:00 2001 From: Michael Haney Date: Wed, 6 Feb 2019 14:24:34 -0800 Subject: [PATCH 5/8] Rename -protoBytes to -transportBytes --- GoogleDataLogger/GoogleDataLogger/Classes/GDLLogEvent.m | 4 ++-- GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m | 6 +++--- .../GoogleDataLogger/Classes/Public/GDLLogProto.h | 2 +- .../GoogleDataLogger/DependencyWrappers/GDLConsoleLogger.h | 2 +- GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m | 2 +- .../Tests/Unit/Helpers/GDLLogExtensionTesterClasses.m | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogEvent.m b/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogEvent.m index 5f7df90135e..71b7cc2d540 100644 --- a/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogEvent.m +++ b/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogEvent.m @@ -52,12 +52,12 @@ - (NSUInteger)hash { } - (void)setExtension:(id)extension { - // If you're looking here because of a performance issue in -protoBytes slowing the assignment + // If you're looking here because of a performance issue in -transportBytes slowing the assignment // of extension, one way to address this is to add a queue to this class, // dispatch_(barrier_ if concurrent)async here, and implement the getter with a dispatch_sync. if (extension != _extension) { _extension = extension; - _extensionBytes = [extension protoBytes]; + _extensionBytes = [extension transportBytes]; } } diff --git a/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m b/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m index 4a7f07f69dd..ff707ce3d12 100644 --- a/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m +++ b/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m @@ -171,16 +171,16 @@ - (void)createLogDirectoryIfNotExists { * @note This method should only be called from a method within a block on _storageQueue to maintain * thread safety. * - * @param logProtoBytes The extensionBytes of the log, presumably proto bytes. + * @param logTransportBytes The extensionBytes of the log, presumably proto bytes. * @param logHash The hash value of the log. * @return The filename */ -- (NSURL *)saveLogProtoToDisk:(NSData *)logProtoBytes logHash:(NSUInteger)logHash { +- (NSURL *)saveLogProtoToDisk:(NSData *)logTransportBytes logHash:(NSUInteger)logHash { NSString *storagePath = GDLStoragePath(); NSString *logFile = [NSString stringWithFormat:@"log-%lu", (unsigned long)logHash]; NSURL *logFilePath = [NSURL fileURLWithPath:[storagePath stringByAppendingPathComponent:logFile]]; - BOOL writingSuccess = [logProtoBytes writeToURL:logFilePath atomically:YES]; + BOOL writingSuccess = [logTransportBytes writeToURL:logFilePath atomically:YES]; if (!writingSuccess) { GDLLogError(GDLMCEFileWriteError, @"A log file could not be written: %@", logFilePath); } diff --git a/GoogleDataLogger/GoogleDataLogger/Classes/Public/GDLLogProto.h b/GoogleDataLogger/GoogleDataLogger/Classes/Public/GDLLogProto.h index dfe4d2d549a..e10d8737256 100644 --- a/GoogleDataLogger/GoogleDataLogger/Classes/Public/GDLLogProto.h +++ b/GoogleDataLogger/GoogleDataLogger/Classes/Public/GDLLogProto.h @@ -27,7 +27,7 @@ NS_ASSUME_NONNULL_BEGIN * * @return the serialized proto bytes of the implementing log proto. */ -- (NSData *)protoBytes; +- (NSData *)transportBytes; @end diff --git a/GoogleDataLogger/GoogleDataLogger/DependencyWrappers/GDLConsoleLogger.h b/GoogleDataLogger/GoogleDataLogger/DependencyWrappers/GDLConsoleLogger.h index 154da49f856..5f5260fafb9 100644 --- a/GoogleDataLogger/GoogleDataLogger/DependencyWrappers/GDLConsoleLogger.h +++ b/GoogleDataLogger/GoogleDataLogger/DependencyWrappers/GDLConsoleLogger.h @@ -30,7 +30,7 @@ static GULLoggerService kGDLConsoleLogger = @"[GoogleDataLogger]"; */ typedef NS_ENUM(NSInteger, GDLMessageCode) { - /** For warning messages concerning protoBytes: not being implemented by a log extension. */ + /** For warning messages concerning transportBytes: not being implemented by a log extension. */ GDLMCWExtensionMissingBytesImpl = 1, /** For warning message concerning a failed log upload. */ diff --git a/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m b/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m index d2e72054f3e..89c360fed2f 100644 --- a/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m +++ b/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m @@ -32,7 +32,7 @@ @interface GDLIntegrationTestLog : NSObject @implementation GDLIntegrationTestLog -- (NSData *)protoBytes { +- (NSData *)transportBytes { // In real usage, protobuf's -data method or a custom implementation using nanopb are used. return [[NSString stringWithFormat:@"%@", [NSDate date]] dataUsingEncoding:NSUTF8StringEncoding]; } diff --git a/GoogleDataLogger/Tests/Unit/Helpers/GDLLogExtensionTesterClasses.m b/GoogleDataLogger/Tests/Unit/Helpers/GDLLogExtensionTesterClasses.m index e071e98658f..e0aaf25cce5 100644 --- a/GoogleDataLogger/Tests/Unit/Helpers/GDLLogExtensionTesterClasses.m +++ b/GoogleDataLogger/Tests/Unit/Helpers/GDLLogExtensionTesterClasses.m @@ -26,7 +26,7 @@ - (instancetype)init { return self; } -- (NSData *)protoBytes { +- (NSData *)transportBytes { return [_aString dataUsingEncoding:NSUTF8StringEncoding]; } From 2ceb6a054eb63a057f35531fb199c40f7e436847 Mon Sep 17 00:00:00 2001 From: Michael Haney Date: Wed, 6 Feb 2019 14:59:08 -0800 Subject: [PATCH 6/8] Change the integration test timing --- GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m b/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m index 89c360fed2f..527e94d7a1c 100644 --- a/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m +++ b/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m @@ -107,8 +107,8 @@ - (void)testEndToEndLog { self.uploader = [[GDLIntegrationTestUploader alloc] initWithServerURL:testServer.serverURL]; // Set the interval to be much shorter than the standard timer. - [GDLUploadCoordinator sharedInstance].timerInterval = NSEC_PER_SEC / 100; - [GDLUploadCoordinator sharedInstance].timerLeeway = NSEC_PER_SEC / 1000; + [GDLUploadCoordinator sharedInstance].timerInterval = NSEC_PER_SEC * 0.1; + [GDLUploadCoordinator sharedInstance].timerLeeway = NSEC_PER_SEC * 0.01; // Confirm no logs are in disk. XCTAssertEqual([GDLLogStorage sharedInstance].logHashToLogFile.count, 0); @@ -128,8 +128,8 @@ - (void)testEndToEndLog { // Generate logs for a big. NSUInteger lengthOfTestToRunInSeconds = 30; - [GDLUploadCoordinator sharedInstance].timerInterval = NSEC_PER_SEC / 5000; - [GDLUploadCoordinator sharedInstance].timerLeeway = NSEC_PER_SEC / 100; + [GDLUploadCoordinator sharedInstance].timerInterval = NSEC_PER_SEC * 5; + [GDLUploadCoordinator sharedInstance].timerLeeway = NSEC_PER_SEC * 1; dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, queue); dispatch_source_set_timer(timer, DISPATCH_TIME_NOW, 1 * NSEC_PER_SEC, 0.1 * NSEC_PER_SEC); From 19b9be8d00808fbbc33cb6bdcefa04668afe244a Mon Sep 17 00:00:00 2001 From: Michael Haney Date: Wed, 6 Feb 2019 15:04:18 -0800 Subject: [PATCH 7/8] Spelling. --- GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m b/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m index 527e94d7a1c..c2b818dc6a0 100644 --- a/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m +++ b/GoogleDataLogger/Tests/Integration/GDLIntegrationTest.m @@ -126,7 +126,7 @@ - (void)testEndToEndLog { // Confirm logs were sent and received. [self waitForExpectations:@[ expectation ] timeout:10.0]; - // Generate logs for a big. + // Generate logs for a bit. NSUInteger lengthOfTestToRunInSeconds = 30; [GDLUploadCoordinator sharedInstance].timerInterval = NSEC_PER_SEC * 5; [GDLUploadCoordinator sharedInstance].timerLeeway = NSEC_PER_SEC * 1; From c9b9a0633404a779be7f53f1c3d05919c6df79c4 Mon Sep 17 00:00:00 2001 From: Michael Haney Date: Wed, 6 Feb 2019 15:10:22 -0800 Subject: [PATCH 8/8] Fix the scheme names in build.sh --- scripts/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index c74860e548a..20c0b754923 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -327,14 +327,14 @@ case "$product-$method-$platform" in GoogleDataLogger-xcodebuild-iOS) RunXcodebuild \ -workspace 'GoogleDataLogger/gen/GoogleDataLogger/GoogleDataLogger.xcworkspace' \ - -scheme "GoogleDataLogger-Unit-Tests" \ + -scheme "GoogleDataLogger-Unit-Tests-Unit" \ "${xcb_flags[@]}" \ build \ test RunXcodebuild \ -workspace 'GoogleDataLogger/gen/GoogleDataLogger/GoogleDataLogger.xcworkspace' \ - -scheme "GoogleDataLogger-Unit-EndToEnd" \ + -scheme "GoogleDataLogger-Unit-Tests-Integration" \ "${xcb_flags[@]}" \ build \ test