Skip to content

Commit e7fee87

Browse files
julientinguely-dajulientinguely-da
committed
[ci] added integration test and summary for transactions
Signed-off-by: Julien Tinguely <[email protected]>
1 parent 33ad0f6 commit e7fee87

File tree

12 files changed

+190
-43
lines changed

12 files changed

+190
-43
lines changed

‎apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/PeriodicBackupIntegrationTest.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import org.lfdecentralizedtrust.splice.config.{
44
BackupDumpConfig,
55
ConfigTransforms,
66
GcpBucketConfig,
7+
IdentityDumpTest,
78
PeriodicBackupDumpConfig,
89
}
910
import org.lfdecentralizedtrust.splice.identities.NodeIdentitiesDump
@@ -98,7 +99,7 @@ final class DirectoryPeriodicBackupIntegrationTest
9899
final class GcpBucketPeriodicBackupIntegrationTest
99100
extends PeriodicBackupIntegrationTestBase[BackupDumpConfig.Gcp] {
100101
override def backupDumpLocation =
101-
BackupDumpConfig.Gcp(GcpBucketConfig.inferForTesting, None)
102+
BackupDumpConfig.Gcp(GcpBucketConfig.inferForTesting(IdentityDumpTest), None)
102103
val bucket = new GcpBucket(backupDumpLocation.bucket, loggerFactory)
103104
override def readDump(filename: String) = {
104105
bucket.readStringFromBucket(Paths.get(filename))

‎apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/PeriodicTopologySnapshotTest.scala‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package org.lfdecentralizedtrust.splice.integration.tests
2+
3+
import com.digitalasset.canton.config.NonNegativeFiniteDuration
4+
import com.digitalasset.canton.logging.SuppressionRule
5+
import org.lfdecentralizedtrust.splice.config.*
6+
import org.lfdecentralizedtrust.splice.integration.EnvironmentDefinition
7+
import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.IntegrationTest
8+
import org.lfdecentralizedtrust.splice.util.GcpBucket
9+
import org.slf4j.event.Level
10+
11+
import java.nio.file.Paths
12+
import java.time.{ZoneOffset, ZonedDateTime}
13+
import scala.concurrent.duration.DurationInt
14+
15+
abstract class PeriodicTopologySnapshotIntegrationTestBase[T <: TopologySnapshotConfig]
16+
extends IntegrationTest {
17+
18+
protected val topologySnapshotInterval: NonNegativeFiniteDuration =
19+
NonNegativeFiniteDuration.ofMinutes(10)
20+
21+
protected def topologySnapshotConfig: PeriodicTopologySnapshotConfig =
22+
PeriodicTopologySnapshotConfig(topologySnapshotLocation, topologySnapshotInterval)
23+
24+
protected def topologySnapshotLocation: T
25+
26+
protected def readDump(filename: String): String
27+
28+
override def environmentDefinition: SpliceEnvironmentDefinition =
29+
EnvironmentDefinition
30+
.simpleTopology1Sv(this.getClass.getSimpleName)
31+
.addConfigTransformsToFront((_, conf) =>
32+
ConfigTransforms.updateAllSvAppConfigs((_, c) =>
33+
c.copy(topologySnapshotConfig = Some(topologySnapshotConfig))
34+
)(conf)
35+
)
36+
.withManualStart
37+
38+
"sv1" should {
39+
"produces a topology snapshot in the background" in { implicit env =>
40+
val utcDate = ZonedDateTime.now(ZoneOffset.UTC).toLocalDate.toString
41+
clue("topology snapshot is being produced ")(
42+
loggerFactory.assertEventuallyLogsSeq(SuppressionRule.Level(Level.INFO))(
43+
timeUntilSuccess = 40.seconds,
44+
within = {
45+
initDsoWithSv1Only()
46+
},
47+
assertion = logEntries => {
48+
forAtLeast(
49+
1,
50+
logEntries,
51+
)(logEntry =>
52+
logEntry.message should (include(s"Took a new topology snapshot for $utcDate") or
53+
include("Today's topology snapshot already exists."))
54+
)
55+
},
56+
)
57+
)
58+
}
59+
}
60+
}
61+
62+
final class GcpBucketPeriodicTopologySnapshotIntegrationTest
63+
extends PeriodicTopologySnapshotIntegrationTestBase[TopologySnapshotConfig.Gcp] {
64+
override def topologySnapshotLocation: TopologySnapshotConfig.Gcp =
65+
TopologySnapshotConfig.Gcp(GcpBucketConfig.inferForTesting(TopologySnapshotTest), None)
66+
val bucket = new GcpBucket(topologySnapshotLocation.bucket, loggerFactory)
67+
override def readDump(filename: String): String = {
68+
bucket.readStringFromBucket(Paths.get(filename))
69+
}
70+
}

‎apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/NonDsoNonDevNetPreflightIntegrationTest.scala‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package org.lfdecentralizedtrust.splice.integration.tests.runbook
22

3+
import org.lfdecentralizedtrust.splice.config.{IdentityDump, TopologySnapshot}
34
import org.lfdecentralizedtrust.splice.integration.EnvironmentDefinition
45
import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.{
5-
SpliceTestConsoleEnvironment,
66
IntegrationTestWithSharedEnvironment,
7+
SpliceTestConsoleEnvironment,
78
}
89
import org.lfdecentralizedtrust.splice.util.DataExportTestUtil
910

@@ -41,11 +42,15 @@ final class NonDsoNonDevNetPreflightIntegrationTest
4142
}
4243

4344
"Check that there is a recent participant identities backup on GCP for validator1" in { _ =>
44-
testRecentParticipantIdentitiesDump("validator1")
45+
testRecentParticipantIdentitiesDump("validator1", IdentityDump)
4546
}
4647

4748
"Check that there is a recent participant identities backup on GCP for splitwell validator" in {
4849
_ =>
49-
testRecentParticipantIdentitiesDump("splitwell")
50+
testRecentParticipantIdentitiesDump("splitwell", IdentityDump)
51+
}
52+
53+
"Check that there is a recent topology snapshot on GCP for sv1" in { _ =>
54+
testRecentParticipantIdentitiesDump("sv1", TopologySnapshot)
5055
}
5156
}

‎apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/SvNonDevNetPreflightintegrationTest.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
package org.lfdecentralizedtrust.splice.integration.tests.runbook
22

3+
import org.lfdecentralizedtrust.splice.config.IdentityDump
34
import org.lfdecentralizedtrust.splice.integration.EnvironmentDefinition
45
import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.SpliceTestConsoleEnvironment
56
import org.lfdecentralizedtrust.splice.integration.tests.FrontendIntegrationTestWithSharedEnvironment
67
import org.lfdecentralizedtrust.splice.util.DataExportTestUtil
7-
88
import org.lfdecentralizedtrust.splice.util.FrontendLoginUtil
9+
910
import scala.util.Try
1011

1112
abstract class SvNonDevNetPreflightIntegrationTestBase
@@ -113,7 +114,7 @@ abstract class SvNonDevNetPreflightIntegrationTestBase
113114
}
114115

115116
"Check that there is a recent participant identities backup on GCP" in { _ =>
116-
testRecentParticipantIdentitiesDump(svNamespace)
117+
testRecentParticipantIdentitiesDump(svNamespace, IdentityDump)
117118
}
118119
}
119120

‎apps/app/src/test/scala/org/lfdecentralizedtrust/splice/util/DataExportTestUtil.scala‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package org.lfdecentralizedtrust.splice.util
22

3-
import org.lfdecentralizedtrust.splice.config.GcpBucketConfig
3+
import org.lfdecentralizedtrust.splice.config.{BucketName, GcpBucketConfig}
44
import org.lfdecentralizedtrust.splice.identities.{NodeIdentitiesDump, NodeIdentitiesStore}
55
import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.TestCommon
66
import com.digitalasset.canton.topology.ParticipantId
@@ -15,8 +15,9 @@ trait DataExportTestUtil extends TestCommon {
1515
namespace: String,
1616
getFileName: Instant => Path,
1717
decode: String => Either[String, A],
18+
bucketName: BucketName,
1819
) = {
19-
val bucket = new GcpBucket(GcpBucketConfig.inferForCluster, loggerFactory)
20+
val bucket = new GcpBucket(GcpBucketConfig.inferForCluster(bucketName), loggerFactory)
2021
import java.time.Instant
2122
import java.time.temporal.ChronoUnit
2223
val cluster = sys.env("GCP_CLUSTER_BASENAME")
@@ -33,10 +34,11 @@ trait DataExportTestUtil extends TestCommon {
3334
}
3435
}
3536

36-
def testRecentParticipantIdentitiesDump(namespace: String) =
37+
def testRecentParticipantIdentitiesDump(namespace: String, bucketName: BucketName) =
3738
testRecentDump(
3839
namespace,
3940
NodeIdentitiesStore.dumpFilename(_),
4041
NodeIdentitiesDump.fromJsonString(ParticipantId.tryFromProtoPrimitive, _),
42+
bucketName,
4143
)
4244
}

‎apps/common/src/main/scala/org/lfdecentralizedtrust/splice/config/GcpConfig.scala‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,16 +48,36 @@ final case class GcpBucketConfig(
4848
def description: String = s"GCP bucket $bucketName in project $projectId"
4949
}
5050

51+
sealed trait BucketName {
52+
def value: String
53+
}
54+
55+
case object IdentityDumpTest extends BucketName {
56+
val value: String = "da-splice-identity-dumps"
57+
}
58+
59+
case object TopologySnapshotTest extends BucketName {
60+
val value: String = "da-splice-topology-snapshots"
61+
}
62+
63+
case object IdentityDump extends BucketName {
64+
val value: String = "da-cn-data-dumps"
65+
}
66+
67+
case object TopologySnapshot extends BucketName {
68+
val value: String = "da-cn-topology-snapshots"
69+
}
70+
5171
object GcpBucketConfig {
52-
def inferForTesting: GcpBucketConfig =
72+
def inferForTesting(testBucketName: BucketName): GcpBucketConfig =
5373
infer(
5474
"GCP_DATA_EXPORT_INTEGRATION_TEST_SERVICE_ACCOUNT_CREDENTIALS",
5575
"da-cn-splice",
56-
"da-splice-identity-dumps",
76+
testBucketName.value,
5777
)
5878

59-
def inferForCluster: GcpBucketConfig =
60-
infer("GCP_DATA_DUMP_BUCKET_SERVICE_ACCOUNT_CREDENTIALS", "da-cn-devnet", "da-cn-data-dumps")
79+
def inferForCluster(bucketName: BucketName): GcpBucketConfig =
80+
infer("GCP_DATA_DUMP_BUCKET_SERVICE_ACCOUNT_CREDENTIALS", "da-cn-devnet", bucketName.value)
6181

6282
def inferForBundles: GcpBucketConfig =
6383
infer(

‎apps/common/src/main/scala/org/lfdecentralizedtrust/splice/environment/SequencerAdminConnection.scala‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ import com.digitalasset.canton.sequencing.protocol
2222
import com.digitalasset.canton.synchronizer.sequencer.SequencerPruningStatus
2323
import com.digitalasset.canton.synchronizer.sequencer.admin.grpc.InitializeSequencerResponse
2424
import com.digitalasset.canton.time.Clock
25+
import com.digitalasset.canton.topology.admin.grpc.{BaseQuery, TopologyStoreId}
2526
import com.digitalasset.canton.topology.admin.v30.GenesisStateV2Response
2627
import com.digitalasset.canton.topology.store.StoredTopologyTransactions.GenericStoredTopologyTransactions
27-
import com.digitalasset.canton.topology.transaction.SequencerSynchronizerState
28+
import com.digitalasset.canton.topology.store.TimeQuery.HeadState
29+
import com.digitalasset.canton.topology.transaction.{SequencerSynchronizerState, TopologyMapping}
2830
import com.digitalasset.canton.topology.{Member, NodeIdentity, SequencerId}
2931
import com.digitalasset.canton.tracing.TraceContext
3032
import com.google.protobuf.ByteString
@@ -81,6 +83,25 @@ class SequencerAdminConnection(
8183
).flatMap(_ => responseObserver.resultFuture.map(_.map(_.chunk)))
8284
}
8385

86+
def getSummaryOfTransactions(store: TopologyStoreId)(implicit
87+
traceContext: TraceContext
88+
): Future[Map[TopologyMapping.Code, Int]] = {
89+
runCmd(
90+
TopologyAdminCommands.Read.ListAll(
91+
query = BaseQuery(
92+
store = store,
93+
proposals = false,
94+
timeQuery = HeadState,
95+
ops = None,
96+
filterSigningKey = "",
97+
protocolVersion = None,
98+
),
99+
filterNamespace = "",
100+
excludeMappings = Seq.empty,
101+
)
102+
).map(_.result.groupMapReduce(_.mapping.code)(_ => 1)(_ + _))
103+
}
104+
84105
def getOnboardingState(sequencerId: SequencerId)(implicit
85106
traceContext: TraceContext
86107
): Future[ByteString] = {

‎apps/common/src/main/scala/org/lfdecentralizedtrust/splice/util/GcpBucket.scala‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,9 @@ class GcpBucket(config: GcpBucketConfig, override val loggerFactory: NamedLogger
6464

6565
def fileExists(fileName: String): Boolean = {
6666
val blobId = BlobId.of(config.bucketName, fileName)
67-
val blob = storage.get(blobId)
68-
blob.exists()
67+
storage.get(blobId) match {
68+
case blob: Blob => blob.exists()
69+
case _ => false
70+
}
6971
}
7072
}

‎apps/common/src/main/scala/org/lfdecentralizedtrust/splice/util/TopologySnapshot.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import com.digitalasset.canton.logging.NamedLoggerFactory
88
import com.digitalasset.canton.tracing.TraceContext
99
import io.circe.Decoder
1010
import io.grpc.Status
11-
import org.lfdecentralizedtrust.splice.config.{GcpBucketConfig, TopologySnapshotConfig}
11+
import org.lfdecentralizedtrust.splice.config.TopologySnapshotConfig
1212

1313
import java.io.FileNotFoundException
1414
import java.nio.charset.StandardCharsets

0 commit comments

Comments
 (0)