Issue/jperf 273 transplant ssh jira nodes

.editorconfig

@@ -1,2 +1,3 @@
-[*.{kt,kts}]
-indent_size = 4
+[*.{kt, kts}]
+indent_size = 4
+insert_final_newline = true

CHANGELOG.md

@@ -26,6 +26,9 @@ Dropping a requirement of a major version of a dependency is a new contract.
 ## [Unreleased]
 [Unreleased]: https://github.com/atlassian/infrastructure/compare/release-4.17.5...master
 
+### Fixed
+- Increase network-level retries for Jira/browser downloads. Decrease flakiness of such downloads on Ubuntu on WSL2.
+
 ## [4.17.5] - 2020-12-15
 [4.17.5]: https://github.com/atlassian/infrastructure/compare/release-4.17.4...release-4.17.5
 

src/main/kotlin/com/atlassian/performance/tools/infrastructure/HttpResource.kt

@@ -27,7 +27,7 @@ class HttpResource(
     ) {
         Ubuntu().install(ssh, listOf("lftp"), Duration.ofMinutes(2))
         ssh.execute(
-            """lftp -c 'set net:timeout 15; set net:max-retries 10; pget -n 32 -c "$uri" -o $destination'""",
+            """lftp -c 'set net:timeout 15; set net:max-retries 50; pget -n 32 -c "$uri" -o $destination'""",
             timeout
         )
     }

src/main/kotlin/com/atlassian/performance/tools/infrastructure/Iostat.kt

@@ -10,7 +10,7 @@ import java.time.temporal.ChronoUnit
 internal class Iostat : OsMetric {
     companion object {
         private val DELAY: Duration = Duration.ofSeconds(2)
-        private val LOG_PATH: String = "~/jpt-iostat.log"
+        private val LOG_PATH: String = "./jpt-iostat.log"
 
         private val TIME = "date -u \"+%d-%m-%Y %H:%M:%S UTC\""
         private val ADD_TIME =
@@ -32,4 +32,4 @@ internal class Iostat : OsMetric {
         val process = connection.startProcess("iostat -d $seconds -x | $ADD_TIME > $LOG_PATH")
         return MonitoringProcess(process, LOG_PATH)
     }
-}
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DatabaseIpConfig.kt

@@ -0,0 +1,25 @@
+package com.atlassian.performance.tools.infrastructure.api.database
+
+import com.atlassian.performance.tools.infrastructure.api.Sed
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.PostInstallHooks
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.InstalledJira
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.PostInstallHook
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+class DatabaseIpConfig(
+    private val databaseIp: String
+) : PostInstallHook {
+
+    override fun call(
+        ssh: SshConnection,
+        jira: InstalledJira,
+        hooks: PostInstallHooks
+    ) {
+        Sed().replace(
+            connection = ssh,
+            expression = "(<url>.*(@(//)?|//))" + "([^:/]+)" + "(.*</url>)",
+            output = """\1$databaseIp\5""",
+            file = "${jira.home}/dbconfig.xml"
+        )
+    }
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MySqlDatabase.kt

@@ -61,4 +61,4 @@ class MySqlDatabase(
             Thread.sleep(Duration.ofSeconds(10).toMillis())
         }
     }
-}
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt

@@ -0,0 +1,35 @@
+package com.atlassian.performance.tools.infrastructure.api.database
+
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.PostInstallHooks
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.InstalledJira
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.PostInstallHook
+import com.atlassian.performance.tools.jvmtasks.api.Backoff
+import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction
+import com.atlassian.performance.tools.ssh.api.SshConnection
+import java.time.Duration
+
+class MysqlConnector : PostInstallHook {
+
+    override fun call(
+        ssh: SshConnection,
+        jira: InstalledJira,
+        hooks: PostInstallHooks
+    ) {
+        val connector = "https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.40.tar.gz"
+        IdempotentAction(
+            description = "Download MySQL connector",
+            action = { ssh.execute("wget -q $connector") }
+        ).retry(
+            maxAttempts = 3,
+            backoff = StaticBackoff(Duration.ofSeconds(5))
+        )
+        ssh.execute("tar -xzf mysql-connector-java-5.1.40.tar.gz")
+        ssh.execute("cp mysql-connector-java-5.1.40/mysql-connector-java-5.1.40-bin.jar ${jira.installation}/lib")
+    }
+}
+
+private class StaticBackoff(
+    private val backOff: Duration
+) : Backoff {
+    override fun backOff(attempt: Int): Duration = backOff
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt

@@ -0,0 +1,51 @@
+package com.atlassian.performance.tools.infrastructure.api.database
+
+import com.atlassian.performance.tools.infrastructure.DockerImage
+import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage
+import com.atlassian.performance.tools.ssh.api.SshConnection
+import org.apache.logging.log4j.LogManager
+import org.apache.logging.log4j.Logger
+import java.net.URI
+import java.time.Duration
+
+class PostgresDatabase(
+    private val source: DatasetPackage,
+    private val maxConnections: Int
+) : Database {
+    private val logger: Logger = LogManager.getLogger(this::class.java)
+
+    private val image: DockerImage = DockerImage(
+        name = "postgres:9.6.15",
+        pullTimeout = Duration.ofMinutes(5)
+    )
+
+    constructor(
+        source: DatasetPackage
+    ) : this(
+        source = source,
+        maxConnections = 200
+    )
+
+    override fun setup(ssh: SshConnection): String {
+        val data = source.download(ssh)
+        val containerName = image.run(
+            ssh = ssh,
+// TODO Dataset for Postgres
+//            parameters = "-p 5432:5432 -v `realpath $data`:/",
+            parameters = "-p 5432:5432",
+            arguments = "-c 'listen_addresses='*'' -c 'max_connections=$maxConnections'"
+        )
+        Thread.sleep(Duration.ofSeconds(15).toMillis())
+        logger.debug("Postgres - creating jira user and database")
+        ssh.execute("sudo docker exec -u postgres $containerName psql --command \"CREATE USER jira WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN PASSWORD 'jira';\"")
+        ssh.execute("sudo docker exec -u postgres $containerName createdb -E UNICODE -l C -T template0 -O jira jira")
+        return data
+    }
+
+    override fun start(jira: URI, ssh: SshConnection) {
+        // TODO Check logs for the following entry
+        // LOG:  database system is ready to accept connections
+        Thread.sleep(Duration.ofSeconds(15).toMillis())
+    }
+
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/EmptyJiraHome.kt

@@ -0,0 +1,11 @@
+package com.atlassian.performance.tools.infrastructure.api.jira
+
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+class EmptyJiraHome : JiraHomeSource {
+    override fun download(ssh: SshConnection): String {
+        val jiraHome = "jira-home"
+        ssh.execute("mkdir $jiraHome")
+        return jiraHome
+    }
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/JiraNodeConfig.kt

@@ -1,10 +1,6 @@
 package com.atlassian.performance.tools.infrastructure.api.jira
 
-import com.atlassian.performance.tools.infrastructure.api.jvm.DisabledJvmDebug
-import com.atlassian.performance.tools.infrastructure.api.jvm.JavaDevelopmentKit
-import com.atlassian.performance.tools.infrastructure.api.jvm.VersionedJavaDevelopmentKit
-import com.atlassian.performance.tools.infrastructure.api.jvm.JvmDebug
-import com.atlassian.performance.tools.infrastructure.api.jvm.OracleJDK
+import com.atlassian.performance.tools.infrastructure.api.jvm.*
 import com.atlassian.performance.tools.infrastructure.api.jvm.jmx.DisabledRemoteJmx
 import com.atlassian.performance.tools.infrastructure.api.jvm.jmx.RemoteJmx
 import com.atlassian.performance.tools.infrastructure.api.profiler.Profiler
@@ -157,4 +153,4 @@ class JiraNodeConfig private constructor(
             JiraNodeConfig::class.java.getResource("/collectd/conf/jira-default.conf").toURI()
         )
     }
-}
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/hook/JiraNodeHooks.kt

@@ -0,0 +1,25 @@
+package com.atlassian.performance.tools.infrastructure.api.jira.hook
+
+import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.DefaultPostInstallHook
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.PreInstallHooks
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.start.DefaultPostStartHook
+import net.jcip.annotations.ThreadSafe
+
+@ThreadSafe
+class JiraNodeHooks {
+
+    val preInstall = PreInstallHooks()
+    val postInstall = preInstall.postInstall
+    val preStart = postInstall.preStart
+    val postStart = preStart.postStart
+    val reports = postStart.reports
+
+    companion object {
+        fun default(): JiraNodeHooks = JiraNodeHooks()
+            .apply { postStart.insert(DefaultPostStartHook()) }
+            .apply { postInstall.insert(DefaultPostInstallHook(JiraNodeConfig.Builder().build())) }
+
+        fun empty(): JiraNodeHooks = JiraNodeHooks()
+    }
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/hook/install/AsyncProfilerHook.kt

@@ -0,0 +1,55 @@
+package com.atlassian.performance.tools.infrastructure.api.jira.hook.install
+
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.start.PostStartHooks
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.report.Report
+import com.atlassian.performance.tools.infrastructure.api.jira.hook.start.PostStartHook
+import com.atlassian.performance.tools.ssh.api.SshConnection
+import java.net.URI
+
+class AsyncProfilerHook : PreInstallHook {
+
+    override fun call(
+        ssh: SshConnection,
+        server: TcpServer,
+        hooks: PreInstallHooks
+    ) {
+        val directory = "async-profiler"
+        val downloads = URI("https://github.com/jvm-profiling-tools/async-profiler/releases/download/")
+        val distribution = downloads.resolve("v1.4/async-profiler-1.4-linux-x64.tar.gz")
+        ssh.execute("wget -q $distribution")
+        ssh.execute("mkdir $directory")
+        ssh.execute("tar -xzf async-profiler-1.4-linux-x64.tar.gz -C $directory")
+        ssh.execute("sudo sh -c 'echo 1 > /proc/sys/kernel/perf_event_paranoid'")
+        ssh.execute("sudo sh -c 'echo 0 > /proc/sys/kernel/kptr_restrict'")
+        val profilerPath = "./$directory/profiler.sh"
+        val profiler = InstalledAsyncProfiler(profilerPath)
+        hooks.postStart.insert(profiler)
+    }
+}
+
+private class InstalledAsyncProfiler(
+    private val profilerPath: String
+) : PostStartHook {
+
+    override fun call(
+        ssh: SshConnection,
+        jira: StartedJira,
+        hooks: PostStartHooks
+    ) {
+        ssh.execute("$profilerPath -b 20000000 start ${jira.pid}")
+        val profiler = StartedAsyncProfiler(jira.pid, profilerPath)
+        hooks.reports.add(profiler)
+    }
+}
+
+private class StartedAsyncProfiler(
+    private val pid: Int,
+    private val profilerPath: String
+) : Report {
+
+    override fun locate(ssh: SshConnection): List<String> {
+        val flameGraphFile = "flamegraph.svg"
+        ssh.execute("$profilerPath stop $pid -o svg > $flameGraphFile")
+        return listOf(flameGraphFile)
+    }
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/hook/install/DataCenterHook.kt

@@ -0,0 +1,23 @@
+package com.atlassian.performance.tools.infrastructure.api.jira.hook.install
+
+import com.atlassian.performance.tools.infrastructure.api.jira.SharedHome
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+class DataCenterHook(
+    private val nodeId: String,
+    private val sharedHome: SharedHome
+) : PostInstallHook {
+
+    override fun call(
+        ssh: SshConnection,
+        jira: InstalledJira,
+        hooks: PostInstallHooks
+    ) {
+        val localSharedHome = sharedHome.localSharedHome
+        sharedHome.mount(ssh)
+        val jiraHome = jira.home  // TODO what's the difference between localSharedHome and jiraHome? should both be hookable?
+        ssh.execute("echo ehcache.object.port = 40011 >> $jiraHome/cluster.properties")
+        ssh.execute("echo jira.node.id = $nodeId >> $jiraHome/cluster.properties")
+        ssh.execute("echo jira.shared.home = `realpath $localSharedHome` >> $jiraHome/cluster.properties")
+    }
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/hook/install/DefaultPostInstallHook.kt

@@ -0,0 +1,27 @@
+package com.atlassian.performance.tools.infrastructure.api.jira.hook.install
+
+import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig
+import com.atlassian.performance.tools.infrastructure.jira.hook.install.ProfilerHook
+import com.atlassian.performance.tools.infrastructure.jira.hook.install.SplunkForwarderHook
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+class DefaultPostInstallHook(
+    private val config: JiraNodeConfig
+) : PostInstallHook {
+
+    override fun call(
+        ssh: SshConnection,
+        jira: InstalledJira,
+        hooks: PostInstallHooks
+    ) {
+        listOf(
+            JiraHomeProperty(),
+            DisabledAutoBackup(),
+            JvmConfig(config),
+            ProfilerHook(config.profiler),
+            SplunkForwarderHook(config.splunkForwarder),
+            JiraLogs(),
+            LateUbuntuSysstat()
+        ).forEach { it.call(ssh, jira, hooks) }
+    }
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/hook/install/DisabledAutoBackup.kt

@@ -0,0 +1,14 @@
+package com.atlassian.performance.tools.infrastructure.api.jira.hook.install
+
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+class DisabledAutoBackup : PostInstallHook {
+
+    override fun call(
+        ssh: SshConnection,
+        jira: InstalledJira,
+        hooks: PostInstallHooks
+    ) {
+        ssh.execute("echo jira.autoexport=false > ${jira.home}/jira-config.properties")
+    }
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/hook/install/HookedJiraInstallation.kt

@@ -0,0 +1,19 @@
+package com.atlassian.performance.tools.infrastructure.api.jira.hook.install
+
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+class HookedJiraInstallation(
+    private val installation: JiraInstallation,
+    private val hooks: PreInstallHooks
+) : JiraInstallation {
+
+    override fun install(
+        ssh: SshConnection,
+        server: TcpServer
+    ): InstalledJira {
+        hooks.call(ssh, server)
+        val installed = installation.install(ssh, server)
+        hooks.postInstall.call(ssh, installed)
+        return installed
+    }
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/hook/install/InstalledJira.kt

@@ -0,0 +1,16 @@
+package com.atlassian.performance.tools.infrastructure.api.jira.hook.install
+
+import com.atlassian.performance.tools.infrastructure.api.jvm.JavaDevelopmentKit
+
+class InstalledJira(
+    /**
+     * E.g. it contains `./dbconfig.xml`
+     */
+    val home: String,
+    /**
+     * E.g. it contains `./conf/server.xml`
+     */
+    val installation: String,
+    val jdk: JavaDevelopmentKit,
+    val server: TcpServer
+)

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/hook/install/JiraHomeProperty.kt

@@ -0,0 +1,15 @@
+package com.atlassian.performance.tools.infrastructure.api.jira.hook.install
+
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+class JiraHomeProperty : PostInstallHook {
+
+    override fun call(
+        ssh: SshConnection,
+        jira: InstalledJira,
+        hooks: PostInstallHooks
+    ) {
+        val properties = "${jira.installation}/atlassian-jira/WEB-INF/classes/jira-application.properties"
+        ssh.execute("echo jira.home=`realpath ${jira.home}` > $properties")
+    }
+}