[Firebase Segmentation] Add custom installation id cache layer and tests for it. (#524)

* Add type arguments in StorageTaskManager (#517)

* Output artifact list during local publishing. (#515)

This effort replaces #494.

* Implement Firebase segmentation SDK device local cache

* fix functions (#523)

* fix functions

* update minsdk version

* remove idea

* Set test type to release only in CI. (#522)

* Set test type to release only in CI.

This fixes Android Studio issue, where it is impossible to run
integration tests in debug mode.

Additionally move build type configuration to FirebaseLibraryPlugin to
avoid projects.all configuration in gradle.

* Add comment back.

* [Firebase Segmentation] Add custom installation id cache layer and tests for it.

* Add test for updating cache

* Switch to use SQLiteOpenHelper

* Minor fix to error message to match the admin sdk. (#525)

* Minor fix to error message to match the admin sdk.

In particular, it *is* allowed to have slashes, etc in field paths.

* Added clean task to smoke tests. (#527)

This change allows the smoke tests to clean all build variants created
by the infrastructure.

* Update deps to post-androidx gms versions. (#526)

* Update deps to post-androidx gms versions.

Additionally configure sources.jar for SDKs.

* Update functions-ktx deps

* Fix versions.

* unbump fiam version in fiamui-app

* Switch to use SharedPreferences from SQLite.

* Change the cache class to be singleton

* Copy firebase-firestore-ktx dependencies on firestore into its own subfolder (#528)

* Wrap shared pref commit in a async task.

* Address comments

* Bump firestore version for release (#530)

Additionally fix pom filter to exclude multidex from deps.

* Google format fix

Author: diwu-arete

buildSrc/build.gradle

@@ -37,14 +37,14 @@ dependencies {
     implementation 'org.jsoup:jsoup:1.11.2'
     implementation 'digital.wup:android-maven-publish:3.6.2'
     implementation 'org.jetbrains.kotlin:kotlin-gradle-plugin:1.3.20'
+    implementation 'org.json:json:20180813'
 
     implementation 'io.opencensus:opencensus-api:0.18.0'
     implementation 'io.opencensus:opencensus-exporter-stats-stackdriver:0.18.0'
     runtime 'io.opencensus:opencensus-impl:0.18.0'
 
     implementation 'com.android.tools.build:gradle:3.2.1'
     testImplementation 'junit:junit:4.12'
-    testImplementation 'org.json:json:20180813'
     testImplementation('org.spockframework:spock-core:1.1-groovy-2.4') {
         exclude group: 'org.codehaus.groovy'
     }

buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/FirebaseLibraryPlugin.java

@@ -20,7 +20,6 @@
 import com.google.firebase.gradle.plugins.ci.device.FirebaseTestServer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
-import org.gradle.api.tasks.bundling.Jar;
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile;
 
 public class FirebaseLibraryPlugin implements Plugin<Project> {
@@ -33,6 +32,28 @@ public void apply(Project project) {
 
     LibraryExtension android = project.getExtensions().getByType(LibraryExtension.class);
 
+    // In the case of and android library signing config only affects instrumentation test APK.
+    // We need it signed with default debug credentials in order for FTL to accept the APK.
+    android.buildTypes(
+        types ->
+            types
+                .getByName("release")
+                .setSigningConfig(types.getByName("debug").getSigningConfig()));
+
+    // skip debug tests in CI
+    // TODO(vkryachko): provide ability for teams to control this if needed
+    if (System.getenv().containsKey("FIREBASE_CI")) {
+      android.setTestBuildType("release");
+      project
+          .getTasks()
+          .all(
+              task -> {
+                if ("testDebugUnitTest".equals(task.getName())) {
+                  task.setEnabled(false);
+                }
+              });
+    }
+
     android.testServer(new FirebaseTestServer(project, firebaseLibrary.testLab));
 
     // reduce the likelihood of kotlin module files colliding.

buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/AffectedProjectFinder.groovy

@@ -25,6 +25,10 @@ class AffectedProjectFinder {
     Set<String> changedPaths;
 
     @Builder
+    AffectedProjectFinder(Project project, List<Pattern> ignorePaths) {
+        this(project, changedPaths(project.rootDir), ignorePaths)
+    }
+
     AffectedProjectFinder(Project project,
                           Set<String> changedPaths,
                           List<Pattern> ignorePaths) {
@@ -49,6 +53,13 @@ class AffectedProjectFinder {
         return project.subprojects
     }
 
+    private static Set<String> changedPaths(File workDir) {
+        return 'git diff --name-only --submodule=diff HEAD@{0} HEAD@{1}'
+                .execute([], workDir)
+                .text
+                .readLines()
+    }
+
     /**
      * Performs a post-order project tree traversal and returns a set of projects that own the
      * 'changedPaths'.

buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationPlugin.groovy

@@ -97,7 +97,6 @@ class ContinuousIntegrationPlugin implements Plugin<Project> {
 
         def affectedProjects = AffectedProjectFinder.builder()
                 .project(project)
-                .changedPaths(changedPaths(project.rootDir))
                 .ignorePaths(extension.ignorePaths)
                 .build()
                 .find()
@@ -143,13 +142,6 @@ class ContinuousIntegrationPlugin implements Plugin<Project> {
         }
     }
 
-    private static Set<String> changedPaths(File workDir) {
-        return 'git diff --name-only --submodule=diff HEAD@{0} HEAD@{1}'
-                .execute([], workDir)
-                .text
-                .readLines()
-    }
-
     private static final ANDROID_PLUGINS = ["com.android.application", "com.android.library",
                                            "com.android.test"]
 

buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/SmokeTestsPlugin.groovy

@@ -0,0 +1,105 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.gradle.plugins.ci
+
+import com.google.firebase.gradle.plugins.FirebaseLibraryExtension
+import com.google.firebase.gradle.plugins.ci.AffectedProjectFinder
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.artifacts.ProjectDependency
+import org.json.JSONArray
+import org.json.JSONObject
+
+/** Builds Firebase libraries for consumption by the smoke tests. */
+class SmokeTestsPlugin implements Plugin<Project> {
+  @Override
+  public void apply(Project project) {
+    def assembleAllTask = project.task("assembleAllForSmokeTests")
+
+    // Wait until after the projects have been evaluated or else we might skip projects.
+    project.gradle.projectsEvaluated {
+      def changedProjects = getChangedProjects(project)
+      def changedArtifacts = new HashSet<String>()
+      def allArtifacts = new HashSet<String>()
+
+      // Visit each project and add the artifacts to the appropriate sets.
+      project.subprojects {
+        def firebaseLibrary = it.extensions.findByType(FirebaseLibraryExtension)
+	if (firebaseLibrary == null) {
+	  return
+	}
+
+        def groupId = firebaseLibrary.groupId.get()
+        def artifactId = firebaseLibrary.artifactId.get()
+        def artifact = "$groupId:$artifactId:$it.version-SNAPSHOT"
+        allArtifacts.add(artifact)
+
+        if (changedProjects.contains(it)) {
+          changedArtifacts.add(artifact)
+        }
+      }
+
+      // Reuse the publish task for building the libraries.
+      def publishAllTask = project.tasks.getByPath("publishAllToBuildDir")
+      assembleAllTask.dependsOn(publishAllTask)
+
+      // Generate a JSON file listing the artifacts after everything is complete.
+      assembleAllTask.doLast {
+        def changed = new JSONArray()
+        changedArtifacts.each { changed.put(it) }
+
+        def all = new JSONArray()
+        allArtifacts.each { all.put(it) }
+
+        def json = new JSONObject()
+        json.put("all", all)
+        json.put("changed", changed)
+
+        def path = project.buildDir.toPath()
+        path.resolve("m2repository/changed-artifacts.json").write(json.toString())
+      }
+    }
+  }
+
+  private static Set<Project> getChangedProjects(Project p) {
+    Set<Project> roots = new AffectedProjectFinder(p, []).find()
+    HashSet<Project> changed = new HashSet<>()
+
+    getChangedProjectsLoop(roots, changed)
+    return changed
+  }
+
+  private static void getChangedProjectsLoop(Collection<Project> projects, Set<Project> changed) {
+    for (Project p : projects) {
+      // Skip project if it is not a Firebase library.
+      if (p.extensions.findByType(FirebaseLibraryExtension) == null) {
+        continue;
+      }
+
+      // Skip processing and recursion if this project has already been added to the set.
+      if (!changed.add(p)) {
+        continue;
+      }
+
+      // Find all (head) dependencies to other projects in this respository.
+      def all = p.configurations.releaseRuntimeClasspath.allDependencies
+      def affected =
+          all.findAll { it instanceof ProjectDependency }.collect { it.getDependencyProject() }
+
+      // Recurse with the new dependencies.
+      getChangedProjectsLoop(affected, changed)
+    }
+  }
+}

buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/publish/Publisher.groovy

@@ -74,7 +74,7 @@ class Publisher {
         pom.dependencies.dependency.each {
             // remove multidex as it is supposed to be added by final applications and is needed for
             // some libraries only for instrumentation tests to build.
-            if (it.groupId.text() in ['com.android.support', 'androidx'] && it.artifactId.text() == 'multidex') {
+            if (it.groupId.text() in ['com.android.support', 'androidx.multidex'] && it.artifactId.text() == 'multidex') {
                 it.parent().remove(it)
             }
             it.appendNode('type', [:], deps["${it.groupId.text()}:${it.artifactId.text()}"])

fiamui-app/fiamui-app.gradle

@@ -53,11 +53,11 @@ android {
 
 dependencies {
   implementation project(path: ":firebase-inappmessaging-display")
-  implementation "com.google.firebase:firebase-measurement-connector:17.0.1"
+  implementation "com.google.firebase:firebase-measurement-connector:18.0.0"
   implementation('com.google.firebase:firebase-inappmessaging:17.0.3') {
     exclude group: 'com.google.firebase', module: 'firebase-common'
   }
-  implementation('com.google.firebase:firebase-analytics:16.0.4') {
+  implementation('com.google.firebase:firebase-analytics:17.0.0') {
     exclude group: 'com.google.firebase', module: 'firebase-common'
   }
 
@@ -67,7 +67,7 @@ dependencies {
   implementation "com.google.code.findbugs:jsr305:3.0.2"
   implementation "com.squareup.okhttp:okhttp:2.7.5"
   implementation "com.google.auto.value:auto-value-annotations:1.6.5"
-  implementation "com.google.android.gms:play-services-basement:16.2.0"
+  implementation "com.google.android.gms:play-services-basement:17.0.0"
 
   // The following dependencies are not required to use the FIAM UI library.
   // They are used to make some aspects of the demo app implementation simpler for

firebase-common/firebase-common.gradle

@@ -58,8 +58,8 @@ android {
 }
 
 dependencies {
-    implementation 'com.google.android.gms:play-services-basement:16.2.0'
-    implementation "com.google.android.gms:play-services-tasks:16.0.1"
+    implementation 'com.google.android.gms:play-services-basement:17.0.0'
+    implementation "com.google.android.gms:play-services-tasks:17.0.0"
 
     api 'com.google.auto.value:auto-value-annotations:1.6.5'
     compileOnly 'com.google.code.findbugs:jsr305:3.0.2'

firebase-common/gradle.properties

@@ -1,2 +1,2 @@
-version=17.1.1
-latestReleasedVersion=17.1.0
+version=18.0.1
+latestReleasedVersion=18.0.0

firebase-database-collection/firebase-database-collection.gradle

@@ -29,7 +29,7 @@ android {
 }
 
 dependencies {
-    implementation 'com.google.android.gms:play-services-base:16.1.0'
+    implementation 'com.google.android.gms:play-services-base:17.0.0'
 
     testImplementation 'junit:junit:4.12'
     testImplementation 'net.java:quickcheck:0.6'

firebase-database-collection/gradle.properties

@@ -1,2 +1,2 @@
-version=16.0.2
-latestReleasedVersion=16.0.1
+version=17.0.1
+latestReleasedVersion=17.0.0

firebase-database/firebase-database.gradle

@@ -73,10 +73,10 @@ dependencies {
     implementation project(':firebase-common')
     implementation project(':firebase-database-collection')
 
-    implementation 'com.google.android.gms:play-services-basement:16.2.0'
-    implementation 'com.google.android.gms:play-services-base:16.1.0'
-    implementation 'com.google.android.gms:play-services-tasks:16.0.1'
-    implementation('com.google.firebase:firebase-auth-interop:17.0.0') {
+    implementation 'com.google.android.gms:play-services-basement:17.0.0'
+    implementation 'com.google.android.gms:play-services-base:17.0.0'
+    implementation 'com.google.android.gms:play-services-tasks:17.0.0'
+    implementation('com.google.firebase:firebase-auth-interop:18.0.0') {
         exclude group: "com.google.firebase", module: "firebase-common"
     }
 

firebase-database/gradle.properties

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version=17.0.0
-latestReleasedVersion=16.1.0
+version=18.0.1
+latestReleasedVersion=18.0.0
 android.enableUnitTestBinaryResources=true

firebase-datatransport/gradle.properties

@@ -1,3 +1,3 @@
-version=16.0.1
-latestReleasedVersion=16.0.0
+version=17.0.1
+latestReleasedVersion=17.0.0
 android.enableUnitTestBinaryResources=true

firebase-firestore/firebase-firestore.gradle

@@ -106,13 +106,13 @@ dependencies {
     implementation 'io.grpc:grpc-protobuf-lite:1.21.0'
     implementation 'io.grpc:grpc-okhttp:1.21.0'
     implementation 'io.grpc:grpc-android:1.21.0'
-    implementation 'com.google.android.gms:play-services-basement:16.2.0'
-    implementation 'com.google.android.gms:play-services-tasks:16.0.1'
-    implementation 'com.google.android.gms:play-services-base:16.1.0'
+    implementation 'com.google.android.gms:play-services-basement:17.0.0'
+    implementation 'com.google.android.gms:play-services-tasks:17.0.0'
+    implementation 'com.google.android.gms:play-services-base:17.0.0'
     implementation 'com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava'
 
     implementation 'com.squareup.okhttp:okhttp:2.7.5'
-    implementation('com.google.firebase:firebase-auth-interop:17.0.0') {
+    implementation('com.google.firebase:firebase-auth-interop:18.0.0') {
         exclude group: "com.google.firebase", module: "firebase-common"
     }
 

firebase-firestore/gradle.properties

@@ -1,2 +1,2 @@
-version=19.0.2
-latestReleasedVersion=19.0.1
+version=20.1.0
+latestReleasedVersion=20.0.0

firebase-firestore/ktx/ktx.gradle

@@ -19,10 +19,11 @@ plugins {
 
 firebaseLibrary {
     releaseWith project(':firebase-firestore')
+    publishSources = true
 }
 
 android {
-    compileSdkVersion project.targetSdkVersion
+    compileSdkVersion 28
     defaultConfig {
         minSdkVersion project.minSdkVersion
         multiDexEnabled true
@@ -33,21 +34,22 @@ android {
         main.java.srcDirs += 'src/main/kotlin'
         test.java {
             srcDir 'src/test/kotlin'
-            srcDir '../src/testUtil/java'
-            srcDir '../src/roboUtil/java'
+            srcDir 'src/test/java'
         }
     }
     testOptions.unitTests.includeAndroidResources = true
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
 }
 
 dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlinVersion"
-
     implementation project(':firebase-common')
     implementation project(':firebase-common:ktx')
     implementation project(':firebase-firestore')
     implementation 'androidx.annotation:annotation:1.1.0'
-
     testImplementation project(':firebase-database-collection')
     testImplementation 'org.mockito:mockito-core:2.25.0'
     testImplementation 'com.fasterxml.jackson.core:jackson-databind:2.9.8'

firebase-firestore/ktx/src/test/java/com/google/firebase/firestore/TestAccessHelper.java

@@ -0,0 +1,31 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.firestore;
+
+import com.google.firebase.firestore.model.DocumentKey;
+
+public final class TestAccessHelper {
+
+  /** Makes the DocumentReference constructor accessible. */
+  public static DocumentReference createDocumentReference(DocumentKey documentKey) {
+    // We can use null here because the tests only use this as a wrapper for documentKeys.
+    return new DocumentReference(documentKey, null);
+  }
+
+  /** Makes the getKey() method accessible. */
+  public static DocumentKey referenceKey(DocumentReference documentReference) {
+    return documentReference.getKey();
+  }
+}