Plugin dependencies automatic download system

buildSrc/src/main/kotlin/DependencyDumper.kt

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2019-2022 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.artifacts.ResolvedDependency
+import org.gradle.api.tasks.TaskProvider
+import java.io.File
+
+object DependencyDumper {
+    fun registerDumpTask(project: Project, confName: String, out: File): TaskProvider<Task> {
+        return regDmpTask(project, confName) { deps ->
+            deps.forEach { println("  `- $it") }
+            out.writeText(deps.joinToString("\n", postfix = "\n"))
+        }
+    }
+
+    fun registerDumpTaskKtSrc(project: Project, confName: String, out: File, className: String): TaskProvider<Task> {
+        val pkgName = className.substringBeforeLast(".")
+        val kname = className.substringAfterLast(".")
+        return regDmpTask(project, confName) { deps ->
+            out.printWriter().use { pr ->
+                pr.println("package $pkgName")
+                pr.println()
+                pr.println("internal object $kname {")
+                pr.println("    val dependencies: List<String> = listOf(")
+                deps.forEach { dependency ->
+                    pr.append("      \"").append(dependency).println("\",")
+                }
+                pr.println("    )")
+                pr.println("}")
+            }
+        }
+    }
+
+    private fun regDmpTask(project: Project, confName: String, action: (List<String>) -> Unit): TaskProvider<Task> {
+        val dependenciesDump = project.tasks.maybeCreate("dependenciesDump")
+        dependenciesDump.group = "mirai"
+        return project.tasks.register("dependenciesDump_${confName.capitalize()}") {
+            group = "mirai"
+            doLast {
+                val dependencies = HashSet<String>()
+                fun emit(dep: ResolvedDependency) {
+                    dependencies.add(dep.moduleGroup + ":" + dep.moduleName)
+                    dep.children.forEach { emit(it) }
+                }
+                project.configurations.getByName(confName).resolvedConfiguration.firstLevelModuleDependencies.forEach { dependency ->
+                    emit(dependency)
+                }
+                val stdep = dependencies.toMutableList()
+                stdep.sort()
+                action(stdep)
+            }
+        }
+    }
+
+}

buildSrc/src/main/kotlin/Versions.kt

@@ -53,6 +53,8 @@ object Versions {
     const val difflib = "1.3.0"
     const val netty = "4.1.63.Final"
     const val bouncycastle = "1.64"
+    const val mavenArtifactResolver = "1.7.3"
+    const val mavenResolverProvider = "3.8.4"
 
     const val junit = "5.7.2"
 
@@ -151,3 +153,9 @@ const val `caller-finder` = "io.github.karlatemp:caller:1.1.1"
 const val `android-runtime` = "com.google.android:android:${Versions.android}"
 const val `netty-all` = "io.netty:netty-all:${Versions.netty}"
 const val `bouncycastle` = "org.bouncycastle:bcprov-jdk15on:${Versions.bouncycastle}"
+
+const val `maven-resolver-api` = "org.apache.maven.resolver:maven-resolver-api:${Versions.mavenArtifactResolver}"
+const val `maven-resolver-impl` = "org.apache.maven.resolver:maven-resolver-impl:${Versions.mavenArtifactResolver}"
+const val `maven-resolver-connector-basic` = "org.apache.maven.resolver:maven-resolver-connector-basic:${Versions.mavenArtifactResolver}"
+const val `maven-resolver-transport-http` = "org.apache.maven.resolver:maven-resolver-transport-http:${Versions.mavenArtifactResolver}"
+const val `maven-resolver-provider` = "org.apache.maven:maven-resolver-provider:${Versions.mavenResolverProvider}"

mirai-console/backend/integration-test/test/MiraiConsoleIntegrationTestBootstrap.kt

@@ -11,6 +11,7 @@ package net.mamoe.console.integrationtest
 
 import net.mamoe.console.integrationtest.testpoints.DoNothingPoint
 import net.mamoe.console.integrationtest.testpoints.MCITBSelfAssertions
+import net.mamoe.console.integrationtest.testpoints.PluginSharedLibraries
 import net.mamoe.console.integrationtest.testpoints.plugin.PluginDataRenameToIdTest
 import net.mamoe.console.integrationtest.testpoints.terminal.TestTerminalLogging
 import org.junit.jupiter.api.Test
@@ -36,6 +37,7 @@ class MiraiConsoleIntegrationTestBootstrap {
             MCITBSelfAssertions,
             PluginDataRenameToIdTest,
             TestTerminalLogging,
+            PluginSharedLibraries,
         ).asSequence().map { v ->
             when (v) {
                 is Class<*> -> v

mirai-console/backend/integration-test/test/testpoints/PluginSharedLibraries.kt

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2019-2022 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+package net.mamoe.console.integrationtest.testpoints
+
+import net.mamoe.console.integrationtest.AbstractTestPoint
+import org.objectweb.asm.ClassWriter
+import org.objectweb.asm.Opcodes
+import java.io.File
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+
+internal object PluginSharedLibraries : AbstractTestPoint() {
+    override fun beforeConsoleStartup() {
+        if (System.getenv("CI").orEmpty().toBoolean()) {
+            println("CI env")
+            File("config/Console/PluginDependencies.yml").writeText(
+                "repoLoc: 'https://repo.maven.apache.org/maven2'"
+            )
+        }
+        File("plugin-shared-libraries").mkdirs()
+        File("plugin-shared-libraries/libraries.txt").writeText(
+            """
+                io.github.karlatemp:unsafe-accessor:1.6.2
+            """.trimIndent()
+        )
+        ZipOutputStream(File("plugin-shared-libraries/test.jar").outputStream().buffered()).use { zipOutput ->
+            zipOutput.putNextEntry(ZipEntry("net/mamoe/console/it/psl/PluginSharedLib.class"))
+            ClassWriter(0).also { writer ->
+                writer.visit(
+                    Opcodes.V1_8,
+                    0,
+                    "net/mamoe/console/it/psl/PluginSharedLib",
+                    null,
+                    "java/lang/Object",
+                    null
+                )
+            }.toByteArray().let { zipOutput.write(it) }
+        }
+    }
+}

mirai-console/backend/integration-test/testers/MCITSelfTestPlugin/src/MCITSelfTestPlugin.kt

@@ -30,4 +30,8 @@ public object MCITSelfTestPlugin : KotlinPlugin(
 
         assertTrue { true }
     }
+
+    public fun someAction() {
+        logger.info { "Called!" }
+    }
 }

mirai-console/backend/integration-test/testers/plugin-depend-on-other/build.gradle.kts

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019-2021 Mamoe Technologies and contributors.
+ *
+ *  此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ *  Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ *  https://github.com/mamoe/mirai/blob/master/LICENSE
+ */
+
+@file:Suppress("UnusedImport")
+
+plugins {
+    kotlin("jvm")
+    kotlin("plugin.serialization")
+    id("java")
+}
+
+version = "0.0.0"
+
+kotlin {
+    explicitApiWarning()
+}
+
+dependencies {
+    api(project(":mirai-console.integration-test"))
+    api(parent!!.project("MCITSelfTestPlugin"))
+}

mirai-console/backend/integration-test/testers/plugin-depend-on-other/resources/META-INF/services/net.mamoe.mirai.console.plugin.jvm.JvmPlugin

@@ -0,0 +1,10 @@
+#
+# Copyright 2019-2022 Mamoe Technologies and contributors.
+#
+# 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+# Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+#
+# https://github.com/mamoe/mirai/blob/dev/LICENSE
+#
+
+net.mamoe.console.integrationtest.ep.dependonother.PluginDependOnOther

mirai-console/backend/integration-test/testers/plugin-depend-on-other/src/PluginDependOnOther.kt

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2019-2022 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+package net.mamoe.console.integrationtest.ep.dependonother
+
+import net.mamoe.console.integrationtest.ep.mcitselftest.MCITSelfTestPlugin
+import net.mamoe.mirai.console.plugin.jvm.JvmPluginDescription
+import net.mamoe.mirai.console.plugin.jvm.KotlinPlugin
+import net.mamoe.mirai.utils.info
+import kotlin.test.assertFails
+import kotlin.test.assertFailsWith
+import kotlin.test.assertNotEquals
+import kotlin.test.assertSame
+
+/*
+PluginDependOnOther: 测试插件依赖其他插件的情况
+ */
+public object PluginDependOnOther : KotlinPlugin(
+    JvmPluginDescription(
+        id = "net.mamoe.tester.plugin-depend-on-other",
+        version = "1.0.0",
+        name = "Plugin Depend On Other",
+    ) {
+        dependsOn("net.mamoe.tester.mirai-console-self-test")
+        dependsOn("net.mamoe.tester.plugin-dynamic-dependencies-download")
+    }
+) {
+    override fun onEnable() {
+        logger.info { "Do dependency call: " + MCITSelfTestPlugin::class.java }
+        logger.info { "No Depends on: " + Class.forName("samepkg.P") }
+        logger.info(Throwable("Stack trace"))
+        MCITSelfTestPlugin.someAction()
+        logger.info { "Shared library: " + Class.forName("net.mamoe.console.it.psl.PluginSharedLib") }
+        assertNotEquals(javaClass.classLoader, Class.forName("net.mamoe.console.it.psl.PluginSharedLib").classLoader)
+
+        // dependencies-shared
+        kotlin.run {
+            val pluginDepDynDownload = Class.forName("net.mamoe.console.integrationtest.ep.pddd.P")
+            val gsonC = Class.forName("com.google.gson.Gson")
+            logger.info { "Gson located $gsonC <${gsonC.classLoader}>" }
+            assertSame(gsonC, Class.forName(gsonC.name, false, pluginDepDynDownload.classLoader))
+            assertFailsWith<ClassNotFoundException> {
+                Class.forName("com.zaxxer.sparsebits.SparseBitSet") // private in dynamic-dep-download
+            }
+            assertFailsWith<ClassNotFoundException> {
+                Class.forName("net.mamoe.assertion.something.not.existing")
+            }
+        }
+    }
+}

mirai-console/backend/integration-test/testers/plugin-dynamic-dependencies-download/resources/META-INF/mirai-console-plugin/dependencies-private.txt

@@ -0,0 +1 @@
+com.zaxxer:SparseBitSet:1.2

mirai-console/backend/integration-test/testers/plugin-dynamic-dependencies-download/resources/META-INF/mirai-console-plugin/dependencies-shared.txt

@@ -0,0 +1 @@
+com.google.code.gson:gson:2.8.9

mirai-console/backend/integration-test/testers/plugin-dynamic-dependencies-download/resources/META-INF/services/net.mamoe.mirai.console.plugin.jvm.JvmPlugin

@@ -0,0 +1 @@
+net.mamoe.console.integrationtest.ep.pddd.P

mirai-console/backend/integration-test/testers/plugin-dynamic-dependencies-download/src/P.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019-2022 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+package net.mamoe.console.integrationtest.ep.pddd
+
+import net.mamoe.mirai.console.extension.PluginComponentStorage
+import net.mamoe.mirai.console.plugin.jvm.JvmPluginDescription
+import net.mamoe.mirai.console.plugin.jvm.KotlinPlugin
+
+/*
+PluginDynamicDependenciesDownload: 测试动态运行时下载
+ */
+internal object P : KotlinPlugin(
+    JvmPluginDescription(
+        id = "net.mamoe.tester.plugin-dynamic-dependencies-download",
+        version = "1.0.0",
+        name = "Plugin Dynamic Dependencies Download",
+    )
+) {
+    override fun PluginComponentStorage.onLoad() {
+        Class.forName("com.google.gson.Gson") // shared
+        Class.forName("com.zaxxer.sparsebits.SparseBitSet") // private
+    }
+}

mirai-console/backend/integration-test/testers/same-pkg-1/resources/META-INF/services/net.mamoe.mirai.console.plugin.jvm.JvmPlugin

@@ -0,0 +1,10 @@
+#
+# Copyright 2019-2022 Mamoe Technologies and contributors.
+#
+# 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+# Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+#
+# https://github.com/mamoe/mirai/blob/dev/LICENSE
+#
+
+samepkg.P

mirai-console/backend/integration-test/testers/same-pkg-1/src/P.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019-2022 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+package samepkg
+
+import net.mamoe.mirai.console.plugin.jvm.JvmPluginDescription
+import net.mamoe.mirai.console.plugin.jvm.KotlinPlugin
+
+/*
+same-pkg-1: 测试包名一样时插件可以正常加载
+ */
+internal object P : KotlinPlugin(
+    JvmPluginDescription(
+        id = "net.mamoe.tester.samepkg-1",
+        version = "1.0.0",
+        name = "SamePkg 1",
+    )
+) {}

mirai-console/backend/integration-test/testers/same-pkg-2/resources/META-INF/services/net.mamoe.mirai.console.plugin.jvm.JvmPlugin

@@ -0,0 +1,10 @@
+#
+# Copyright 2019-2022 Mamoe Technologies and contributors.
+#
+# 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+# Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+#
+# https://github.com/mamoe/mirai/blob/dev/LICENSE
+#
+
+samepkg.P

mirai-console/backend/integration-test/testers/same-pkg-2/src/P.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019-2022 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+package samepkg
+
+import net.mamoe.mirai.console.plugin.jvm.JvmPluginDescription
+import net.mamoe.mirai.console.plugin.jvm.KotlinPlugin
+
+/*
+same-pkg-2: 测试包名一样时插件可以正常加载
+ */
+internal object P : KotlinPlugin(
+    JvmPluginDescription(
+        id = "net.mamoe.tester.samepkg-2",
+        version = "1.0.0",
+        name = "SamePkg 2",
+    )
+) {}

mirai-console/backend/mirai-console/.gitignore

@@ -1 +1,2 @@
-src/internal/MiraiConsoleBuildConstants.kt
+src/internal/MiraiConsoleBuildConstants.kt
+src/internal/MiraiConsoleBuildDependencies.kt

mirai-console/backend/mirai-console/build.gradle.kts

@@ -30,6 +30,8 @@ kotlin {
     explicitApiWarning()
 }
 
+configurations.register("consoleRuntimeClasspath")
+
 dependencies {
     compileAndTestRuntime(project(":mirai-core-api"))
     compileAndTestRuntime(project(":mirai-core-utils"))
@@ -46,10 +48,18 @@ dependencies {
     smartImplementation(`yamlkt-jvm`)
     smartImplementation(`jetbrains-annotations`)
     smartImplementation(`caller-finder`)
+    smartImplementation(`maven-resolver-api`)
+    smartImplementation(`maven-resolver-provider`)
+    smartImplementation(`maven-resolver-impl`)
+    smartImplementation(`maven-resolver-connector-basic`)
+    smartImplementation(`maven-resolver-transport-http`)
     smartApi(`kotlinx-coroutines-jdk8`)
 
     testApi(project(":mirai-core"))
     testApi(`kotlin-stdlib-jdk8`)
+
+    "consoleRuntimeClasspath"(project)
+    "consoleRuntimeClasspath"(project(":mirai-core"))
 }
 
 tasks {
@@ -71,5 +81,14 @@ tasks {
     }
 }
 
+tasks.getByName("compileKotlin").dependsOn(
+    DependencyDumper.registerDumpTaskKtSrc(
+        project,
+        "consoleRuntimeClasspath",
+        project.file("src/internal/MiraiConsoleBuildDependencies.kt"),
+        "net.mamoe.mirai.console.internal.MiraiConsoleBuildDependencies"
+    )
+)
+
 configurePublishing("mirai-console")
 configureBinaryValidator(null)

mirai-console/backend/mirai-console/src/internal/data/builtins/ConsoleDataScopeImpl.kt

@@ -29,7 +29,10 @@ internal class ConsoleDataScopeImpl(
     override val configHolder: AutoSavePluginDataHolder = ConsoleBuiltInPluginConfigHolder(this.coroutineContext)
 
     private val data: List<PluginData> = mutableListOf()
-    private val configs: MutableList<PluginConfig> = mutableListOf(AutoLoginConfig)
+    private val configs: MutableList<PluginConfig> = mutableListOf(
+        AutoLoginConfig,
+        PluginDependenciesConfig,
+    )
 
     override fun addAndReloadConfig(config: PluginConfig) {
         configs.add(config)

mirai-console/backend/mirai-console/src/internal/data/builtins/PluginDependenciesConfig.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2019-2022 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+package net.mamoe.mirai.console.internal.data.builtins
+
+import net.mamoe.mirai.console.data.ReadOnlyPluginConfig
+import net.mamoe.mirai.console.data.ValueDescription
+import net.mamoe.mirai.console.data.value
+
+internal object PluginDependenciesConfig : ReadOnlyPluginConfig("PluginDependencies") {
+    @ValueDescription("远程仓库, 如无必要无需修改")
+    val repoLoc by value<String>("https://maven.aliyun.com/repository/public")
+}

mirai-console/backend/mirai-console/src/internal/plugin/BuiltInJvmPluginLoaderImpl.kt

@@ -18,15 +18,14 @@ import net.mamoe.mirai.console.data.PluginDataStorage
 import net.mamoe.mirai.console.internal.util.PluginServiceHelper.findServices
 import net.mamoe.mirai.console.internal.util.PluginServiceHelper.loadAllServices
 import net.mamoe.mirai.console.plugin.PluginManager
+import net.mamoe.mirai.console.plugin.id
 import net.mamoe.mirai.console.plugin.jvm.*
 import net.mamoe.mirai.console.plugin.loader.AbstractFilePluginLoader
 import net.mamoe.mirai.console.plugin.loader.PluginLoadException
 import net.mamoe.mirai.console.plugin.name
-import net.mamoe.mirai.utils.MiraiLogger
-import net.mamoe.mirai.utils.castOrNull
-import net.mamoe.mirai.utils.childScope
-import net.mamoe.mirai.utils.verbose
+import net.mamoe.mirai.utils.*
 import java.io.File
+import java.io.InputStream
 import java.nio.file.Path
 import java.util.concurrent.ConcurrentHashMap
 import kotlin.coroutines.CoroutineContext
@@ -51,7 +50,69 @@ internal class BuiltInJvmPluginLoaderImpl(
     override val dataStorage: PluginDataStorage
         get() = MiraiConsoleImplementation.getInstance().dataStorageForJvmPluginLoader
 
-    override val classLoaders: MutableList<JvmPluginClassLoader> = mutableListOf()
+
+    internal val jvmPluginLoadingCtx: JvmPluginsLoadingCtx by lazy {
+        val classLoader = DynLibClassLoader(BuiltInJvmPluginLoaderImpl::class.java.classLoader)
+        val ctx = JvmPluginsLoadingCtx(
+            classLoader,
+            mutableListOf(),
+            JvmPluginDependencyDownloader(logger),
+        )
+        logger.verbose { "Plugin shared libraries: " + PluginManager.pluginSharedLibrariesFolder }
+        PluginManager.pluginSharedLibrariesFolder.listFiles()?.asSequence().orEmpty()
+            .onEach { logger.debug { "Peek $it in shared libraries" } }
+            .filter { file ->
+                if (file.isDirectory) {
+                    return@filter true
+                }
+                if (!file.exists()) {
+                    logger.debug { "Skipped $file because file not exists" }
+                    return@filter false
+                }
+                if (file.isFile) {
+                    if (file.extension == "jar") {
+                        return@filter true
+                    }
+                    logger.debug { "Skipped $file because extension <${file.extension}> != jar" }
+                    return@filter false
+                }
+                logger.debug { "Skipped $file because unknown error" }
+                return@filter false
+            }
+            .filter { it.isDirectory || (it.isFile && it.extension == "jar") }
+            .forEach { pt ->
+                classLoader.addLib(pt)
+                logger.debug { "Linked static shared library: $pt" }
+            }
+        val libraries = PluginManager.pluginSharedLibrariesFolder.resolve("libraries.txt")
+        if (libraries.isFile) {
+            logger.verbose { "Linking static shared libraries...." }
+            val libs = libraries.useLines { lines ->
+                lines.filter { it.isNotBlank() }
+                    .filterNot { it.startsWith("#") }
+                    .onEach { logger.verbose { "static lib queued: $it" } }
+                    .toMutableList()
+            }
+            val staticLibs = ctx.downloader.resolveDependencies(libs)
+            staticLibs.artifactResults.forEach { artifactResult ->
+                if (artifactResult.isResolved) {
+                    ctx.sharedLibrariesLoader.addLib(artifactResult.artifact.file)
+                    ctx.sharedLibrariesDependencies.add(artifactResult.artifact.depId())
+                    logger.debug { "Linked static shared library: ${artifactResult.artifact}" }
+                }
+            }
+        } else {
+            libraries.createNewFile()
+        }
+        ctx
+    }
+
+    override val classLoaders: MutableList<JvmPluginClassLoaderN> get() = jvmPluginLoadingCtx.pluginClassLoaders
+
+    override fun findLoadedClass(name: String): Class<*>? {
+        return classLoaders.firstNotNullOfOrNull { it.loadedClass(name) }
+    }
+
 
     @Suppress("EXTENSION_SHADOWED_BY_MEMBER") // doesn't matter
     override fun getPluginDescription(plugin: JvmPlugin): JvmPluginDescription = plugin.description
@@ -61,7 +122,7 @@ internal class BuiltInJvmPluginLoaderImpl(
     override fun Sequence<File>.extractPlugins(): List<JvmPlugin> {
         ensureActive()
 
-        fun Sequence<Map.Entry<File, JvmPluginClassLoader>>.findAllInstances(): Sequence<Map.Entry<File, JvmPlugin>> {
+        fun Sequence<Map.Entry<File, JvmPluginClassLoaderN>>.findAllInstances(): Sequence<Map.Entry<File, JvmPlugin>> {
             return onEach { (_, pluginClassLoader) ->
                 val exportManagers = pluginClassLoader.findServices(
                     ExportManager::class
@@ -91,7 +152,7 @@ internal class BuiltInJvmPluginLoaderImpl(
         val filePlugins = this.filterNot {
             pluginFileToInstanceMap.containsKey(it)
         }.associateWith {
-            JvmPluginClassLoader(it, MiraiConsole::class.java.classLoader, classLoaders)
+            JvmPluginClassLoaderN.newLoader(it, jvmPluginLoadingCtx)
         }.onEach { (_, classLoader) ->
             classLoaders.add(classLoader)
         }.asSequence().findAllInstances().onEach {
@@ -149,6 +210,36 @@ internal class BuiltInJvmPluginLoaderImpl(
             PluginManager.pluginsDataPath.moveNameFolder(plugin)
             PluginManager.pluginsConfigPath.moveNameFolder(plugin)
             check(plugin is JvmPluginInternal) { "A JvmPlugin must extend AbstractJvmPlugin to be loaded by JvmPluginLoader.BuiltIn" }
+            // region Link dependencies
+            plugin.javaClass.classLoader.safeCast<JvmPluginClassLoaderN>()?.let { jvmPluginClassLoaderN ->
+                // Link plugin dependencies
+                plugin.description.dependencies.asSequence().mapNotNull { dependency ->
+                    plugin.logger.verbose { "Linking dependency: ${dependency.id}" }
+                    PluginManager.plugins.firstOrNull { it.id == dependency.id }
+                }.mapNotNull { it.javaClass.classLoader.safeCast<JvmPluginClassLoaderN>() }.forEach { dependency ->
+                    plugin.logger.debug { "Linked  dependency: $dependency" }
+                    jvmPluginClassLoaderN.dependencies.add(dependency)
+                }
+                // Link jar dependencies
+                fun InputStream?.readDependencies(): Collection<String> {
+                    if (this == null) return emptyList()
+                    return bufferedReader().useLines { lines ->
+                        lines.filterNot { it.isBlank() }
+                            .filterNot { it.startsWith('#') }
+                            .map { it.trim() }
+                            .toMutableList()
+                    }
+                }
+                jvmPluginClassLoaderN.linkPluginSharedLibraries(
+                    plugin.logger,
+                    jvmPluginClassLoaderN.getResourceAsStream("META-INF/mirai-console-plugin/dependencies-shared.txt").readDependencies()
+                )
+                jvmPluginClassLoaderN.linkPluginPrivateLibraries(
+                    plugin.logger,
+                    jvmPluginClassLoaderN.getResourceAsStream("META-INF/mirai-console-plugin/dependencies-private.txt").readDependencies()
+                )
+            }
+            // endregion
             plugin.internalOnLoad()
         }.getOrElse {
             throw PluginLoadException("Exception while loading ${plugin.description.smartToString()}", it)

mirai-console/backend/mirai-console/src/internal/plugin/JvmPluginClassLoader.kt

@@ -11,104 +11,221 @@
 package net.mamoe.mirai.console.internal.plugin
 
 import net.mamoe.mirai.console.plugin.jvm.ExportManager
+import net.mamoe.mirai.utils.MiraiLogger
+import net.mamoe.mirai.utils.debug
+import net.mamoe.mirai.utils.verbose
+import org.eclipse.aether.artifact.Artifact
+import org.eclipse.aether.graph.DependencyFilter
 import java.io.File
 import java.net.URL
 import java.net.URLClassLoader
 import java.util.*
-import java.util.concurrent.ConcurrentHashMap
+import java.util.zip.ZipFile
 
-internal class JvmPluginClassLoader(
-    val file: File,
+/*
+Class resolving:
+
+|
+`- Resolve standard classes: by super class loader.
+`- Resolve classes in shared libraries (Shared in all plugins)
+|
+|-===== SANDBOX =====
+|
+`- Resolve classes in plugin dependency shared libraries (Shared by depend-ed plugins)
+`- Resolve classes in independent libraries (Can only be loaded by current plugin)
+`- Resolve classes in current jar.
+`- Resolve classes from other plugin jar
+
+ */
+
+internal class JvmPluginsLoadingCtx(
+    val sharedLibrariesLoader: DynLibClassLoader,
+    val pluginClassLoaders: MutableList<JvmPluginClassLoaderN>,
+    val downloader: JvmPluginDependencyDownloader,
+) {
+    val sharedLibrariesDependencies = HashSet<String>()
+    val sharedLibrariesFilter: DependencyFilter = DependencyFilter { node, _ ->
+        return@DependencyFilter node.artifact.depId() !in sharedLibrariesDependencies
+    }
+}
+
+internal class DynLibClassLoader(
     parent: ClassLoader?,
-    val classLoaders: Collection<JvmPluginClassLoader>,
-) : URLClassLoader(arrayOf(file.toURI().toURL()), parent) {
-    //// 只允许插件 getResource 时获取插件自身资源, #205
-    override fun getResources(name: String?): Enumeration<URL> = findResources(name)
-    override fun getResource(name: String?): URL? = findResource(name)
-    // getResourceAsStream 在 URLClassLoader 中通过 getResource 确定资源
-    //      因此无需 override getResourceAsStream
+) : URLClassLoader(arrayOf(), parent) {
+    companion object {
+        init {
+            ClassLoader.registerAsParallelCapable()
+        }
+    }
+
+    internal fun addLib(url: URL) {
+        addURL(url)
+    }
+
+    internal fun addLib(file: File) {
+        addURL(file.toURI().toURL())
+    }
 
     override fun toString(): String {
-        return "JvmPluginClassLoader{source=$file}"
+        return "DynLibClassLoader@" + hashCode()
+    }
+}
+
+@Suppress("JoinDeclarationAndAssignment")
+internal class JvmPluginClassLoaderN : URLClassLoader {
+    val file: File
+    val ctx: JvmPluginsLoadingCtx
+
+    val dependencies: MutableCollection<JvmPluginClassLoaderN> = hashSetOf()
+
+    lateinit var pluginSharedCL: DynLibClassLoader
+    lateinit var pluginIndependentCL: DynLibClassLoader
+
+
+    private constructor(file: File, ctx: JvmPluginsLoadingCtx, unused: Unit) : super(
+        arrayOf(), ctx.sharedLibrariesLoader
+    ) {
+        this.file = file
+        this.ctx = ctx
+        init0()
+    }
+
+    private constructor(file: File, ctx: JvmPluginsLoadingCtx) : super(
+        file.name,
+        arrayOf(), ctx.sharedLibrariesLoader
+    ) {
+        this.file = file
+        this.ctx = ctx
+        init0()
+    }
+
+    private fun init0() {
+        ZipFile(file).use { zipFile ->
+            zipFile.entries().asSequence()
+                .filter { it.name.endsWith(".class") }
+                .map { it.name.substringBeforeLast('.') }
+                .map { it.removePrefix("/").replace('/', '.') }
+                .map { it.substringBeforeLast('.') }
+                .forEach { pkg ->
+                    pluginMainPackages.add(pkg)
+                }
+        }
+        pluginSharedCL = DynLibClassLoader(ctx.sharedLibrariesLoader)
+        pluginIndependentCL = DynLibClassLoader(pluginSharedCL)
+        addURL(file.toURI().toURL())
     }
 
-    private val cache = ConcurrentHashMap<String, Class<*>>()
+    private val pluginMainPackages: MutableSet<String> = HashSet()
     internal var declaredFilter: ExportManager? = null
 
+    val sharedClLoadedDependencies = mutableSetOf<String>()
+    internal fun containsSharedDependency(
+        dependency: String
+    ): Boolean {
+        if (dependency in sharedClLoadedDependencies) return true
+        return dependencies.any { it.containsSharedDependency(dependency) }
+    }
+
+    internal fun linkPluginSharedLibraries(logger: MiraiLogger, dependencies: Collection<String>) {
+        linkLibraries(logger, dependencies, true)
+    }
+
+    internal fun linkPluginPrivateLibraries(logger: MiraiLogger, dependencies: Collection<String>) {
+        linkLibraries(logger, dependencies, false)
+    }
+
+    private fun linkLibraries(logger: MiraiLogger, dependencies: Collection<String>, shared: Boolean) {
+        if (dependencies.isEmpty()) return
+        val results = ctx.downloader.resolveDependencies(
+            dependencies, ctx.sharedLibrariesFilter,
+            DependencyFilter { node, _ ->
+                return@DependencyFilter !containsSharedDependency(node.artifact.depId())
+            })
+        val files = results.artifactResults.mapNotNull { result ->
+            result.artifact?.let { it to it.file }
+        }
+        val linkType = if (shared) "(shared)" else "(private)"
+        files.forEach { (artifact, lib) ->
+            logger.verbose { "Linking $lib $linkType" }
+            if (shared) {
+                pluginSharedCL.addLib(lib)
+                sharedClLoadedDependencies.add(artifact.depId())
+            } else {
+                pluginIndependentCL.addLib(lib)
+            }
+            logger.debug { "Linked $artifact $linkType" }
+        }
+    }
+
     companion object {
-        val loadingLock = ConcurrentHashMap<String, Any>()
+        private val java9: Boolean
 
         init {
             ClassLoader.registerAsParallelCapable()
+            java9 = kotlin.runCatching { Class.forName("java.lang.Module") }.isSuccess
+        }
+
+        fun newLoader(file: File, ctx: JvmPluginsLoadingCtx): JvmPluginClassLoaderN {
+            return when {
+                java9 -> JvmPluginClassLoaderN(file, ctx)
+                else -> JvmPluginClassLoaderN(file, ctx, Unit)
+            }
         }
     }
 
-    override fun findClass(name: String): Class<*> {
-        synchronized(kotlin.run {
-            val lock = Any()
-            loadingLock.putIfAbsent(name, lock) ?: lock
-        }) {
-            return findClass(name, false) ?: throw ClassNotFoundException(name)
+    internal fun resolvePluginSharedLibAndPluginClass(name: String): Class<*>? {
+        return try {
+            pluginSharedCL.loadClass(name)
+        } catch (e: ClassNotFoundException) {
+            resolvePluginPublicClass(name)
         }
     }
 
-    internal fun findClass(name: String, disableGlobal: Boolean): Class<*>? {
-        // First. Try direct load in cache.
-        val cachedClass = cache[name]
-        if (cachedClass != null) {
-            if (disableGlobal) {
-                val filter = declaredFilter
-                if (filter != null && !filter.isExported(name)) {
-                    throw LoadingDeniedException(name)
-                }
-            }
-            return cachedClass
+    internal fun resolvePluginPublicClass(name: String): Class<*>? {
+        if (pluginMainPackages.contains(name.pkgName())) {
+            if (declaredFilter?.isExported(name) == false) return null
+            return loadClass(name)
         }
-        if (disableGlobal) {
-            // ==== Process Loading Request From JvmPluginClassLoader ====
-            //
-            // If load from other classloader,
-            // means no other loaders are cached.
-            // direct load
-            return kotlin.runCatching {
-                super.findClass(name).also { cache[name] = it }
-            }.getOrElse {
-                if (it is ClassNotFoundException) null
-                else throw it
-            }?.also {
-                // This request is from other classloader,
-                // so we need to check the class is exported or not.
-                val filter = declaredFilter
-                if (filter != null && !filter.isExported(name)) {
-                    throw LoadingDeniedException(name)
+        return null
+    }
+
+    override fun findClass(name: String): Class<*> {
+        // Search dependencies first
+        dependencies.forEach { dependency ->
+            dependency.resolvePluginSharedLibAndPluginClass(name)?.let { return it }
+        }
+        // Search in independent class loader
+        // @context: pluginIndependentCL.parent = pluinSharedCL
+        try {
+            return pluginIndependentCL.loadClass(name)
+        } catch (ignored: ClassNotFoundException) {
+        }
+
+        try {
+            return super.findClass(name)
+        } catch (error: ClassNotFoundException) {
+            // Finally, try search from other plugins
+            ctx.pluginClassLoaders.forEach { other ->
+                if (other !== this) {
+                    other.resolvePluginPublicClass(name)?.let { return it }
                 }
             }
+            throw error
         }
+    }
 
-        // ==== Process Loading Request From JDK ClassLoading System ====
+    internal fun loadedClass(name: String): Class<*>? = super.findLoadedClass(name)
 
-        // First. scan other classLoaders's caches
-        classLoaders.forEach { otherClassloader ->
-            if (otherClassloader === this) return@forEach
-            val filter = otherClassloader.declaredFilter
-            if (otherClassloader.cache.containsKey(name)) {
-                return if (filter == null || filter.isExported(name)) {
-                    otherClassloader.cache[name]
-                } else throw LoadingDeniedException("$name was not exported by $otherClassloader")
-            }
-        }
-        classLoaders.forEach { otherClassloader ->
-            val other = kotlin.runCatching {
-                if (otherClassloader === this) super.findClass(name).also { cache[name] = it }
-                else otherClassloader.findClass(name, true)
-            }.onFailure { err ->
-                if (err is LoadingDeniedException || err !is ClassNotFoundException)
-                    throw err
-            }.getOrNull()
-            if (other != null) return other
-        }
-        throw ClassNotFoundException(name)
+    //// 只允许插件 getResource 时获取插件自身资源, https://github.com/mamoe/mirai-console/issues/205
+    override fun getResources(name: String?): Enumeration<URL> = findResources(name)
+    override fun getResource(name: String?): URL? = findResource(name)
+    // getResourceAsStream 在 URLClassLoader 中通过 getResource 确定资源
+    //      因此无需 override getResourceAsStream
+
+    override fun toString(): String {
+        return "JvmPluginClassLoader{${file.name}}"
     }
 }
 
-internal class LoadingDeniedException(name: String) : ClassNotFoundException(name)
+private fun String.pkgName(): String = substringBeforeLast('.', "")
+internal fun Artifact.depId(): String = "$groupId:$artifactId"

mirai-console/backend/mirai-console/src/internal/plugin/JvmPluginDependencyDownload.kt

@@ -0,0 +1,137 @@
+/*
+ * Copyright 2019-2022 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+package net.mamoe.mirai.console.internal.plugin
+
+import net.mamoe.mirai.console.internal.MiraiConsoleBuildDependencies
+import net.mamoe.mirai.console.internal.data.builtins.PluginDependenciesConfig
+import net.mamoe.mirai.console.plugin.PluginManager
+import net.mamoe.mirai.utils.MiraiLogger
+import net.mamoe.mirai.utils.debug
+import net.mamoe.mirai.utils.verbose
+import org.apache.maven.repository.internal.MavenRepositorySystemUtils
+import org.eclipse.aether.RepositorySystem
+import org.eclipse.aether.RepositorySystemSession
+import org.eclipse.aether.artifact.DefaultArtifact
+import org.eclipse.aether.collection.CollectRequest
+import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory
+import org.eclipse.aether.graph.Dependency
+import org.eclipse.aether.graph.DependencyFilter
+import org.eclipse.aether.repository.LocalRepository
+import org.eclipse.aether.repository.RemoteRepository
+import org.eclipse.aether.resolution.DependencyRequest
+import org.eclipse.aether.resolution.DependencyResult
+import org.eclipse.aether.spi.connector.RepositoryConnectorFactory
+import org.eclipse.aether.spi.connector.transport.TransporterFactory
+import org.eclipse.aether.spi.locator.ServiceLocator
+import org.eclipse.aether.transfer.AbstractTransferListener
+import org.eclipse.aether.transfer.TransferEvent
+import org.eclipse.aether.transport.http.HttpTransporterFactory
+
+
+@Suppress("DEPRECATION", "MemberVisibilityCanBePrivate")
+internal class JvmPluginDependencyDownloader(
+    val logger: MiraiLogger,
+) {
+    val repositories: MutableList<RemoteRepository>
+    val session: RepositorySystemSession
+    val locator: ServiceLocator
+    val repository: RepositorySystem
+    val dependencyFilter: DependencyFilter = DependencyFilter { node, parents ->
+        if (node == null || node.artifact == null) return@DependencyFilter true
+
+        val artGroup = node.artifact.groupId
+        val artId = node.artifact.artifactId
+
+        // mirai used netty-all
+        if (artGroup == "io.netty") return@DependencyFilter false
+
+        if (artGroup == "net.mamoe") {
+            if (artId in listOf(
+                    "mirai-core",
+                    "mirai-core-jvm",
+                    "mirai-core-android",
+                    "mirai-core-api",
+                    "mirai-core-api-jvm",
+                    "mirai-core-api-android",
+                    "mirai-core-utils",
+                    "mirai-core-utils-jvm",
+                    "mirai-core-utils-android",
+                )
+            ) return@DependencyFilter false
+        }
+
+        // Loaded by console system
+        if ("$artGroup:$artId" in MiraiConsoleBuildDependencies.dependencies)
+            return@DependencyFilter false
+
+        // println("  `- filter: $node")
+        true
+    }
+
+    init {
+        locator = MavenRepositorySystemUtils.newServiceLocator()
+        locator.addService(RepositoryConnectorFactory::class.java, BasicRepositoryConnectorFactory::class.java)
+        locator.addService(TransporterFactory::class.java, HttpTransporterFactory::class.java)
+        repository = locator.getService(RepositorySystem::class.java)
+        session = MavenRepositorySystemUtils.newSession()
+        session.checksumPolicy = "fail"
+        session.localRepositoryManager = repository.newLocalRepositoryManager(
+            session, LocalRepository(PluginManager.pluginLibrariesFolder)
+        )
+        session.transferListener = object : AbstractTransferListener() {
+            override fun transferStarted(event: TransferEvent) {
+                logger.verbose {
+                    "Downloading ${event.resource?.repositoryUrl}${event.resource?.resourceName}"
+                }
+            }
+
+            override fun transferFailed(event: TransferEvent) {
+                logger.warning(event.exception)
+            }
+        }
+        session.setReadOnly()
+        repositories = repository.newResolutionRepositories(
+            session,
+//            listOf(RemoteRepository.Builder("central", "default", "https://repo.maven.apache.org/maven2").build())
+//            listOf(RemoteRepository.Builder("central", "default", "https://maven.aliyun.com/repository/public").build())
+            listOf(RemoteRepository.Builder("central", "default", PluginDependenciesConfig.repoLoc).build())
+        )
+        logger.debug { "Remote server: " + PluginDependenciesConfig.repoLoc }
+    }
+
+    public fun resolveDependencies(deps: Collection<String>, vararg filters: DependencyFilter): DependencyResult {
+
+        val dependencies: MutableList<Dependency> = ArrayList()
+        for (library in deps) {
+            val defaultArtifact = DefaultArtifact(library)
+            val dependency = Dependency(defaultArtifact, null)
+            dependencies.add(dependency)
+        }
+        return repository.resolveDependencies(
+            session as RepositorySystemSession?, DependencyRequest(
+                CollectRequest(
+                    null as Dependency?, dependencies,
+                    repositories
+                ),
+                when {
+                    filters.isEmpty() -> dependencyFilter
+                    else -> DependencyFilter { node, parents ->
+                        if (node == null || node.artifact == null) return@DependencyFilter true
+                        if (!dependencyFilter.accept(node, parents)) return@DependencyFilter false
+                        filters.forEach { filter ->
+                            if (!filter.accept(node, parents)) return@DependencyFilter false
+                        }
+                        return@DependencyFilter true
+                    }
+                }
+            )
+        )
+    }
+}

mirai-console/backend/mirai-console/src/internal/plugin/PluginManagerImpl.kt

@@ -48,6 +48,12 @@ internal class PluginManagerImpl(
     override val pluginsConfigPath: Path = MiraiConsole.rootPath.resolve("config").apply { mkdir() }
     override val pluginsConfigFolder: File = pluginsConfigPath.toFile()
 
+    override val pluginLibrariesPath: Path = MiraiConsole.rootPath.resolve("plugin-libraries").apply { mkdir() }
+    override val pluginLibrariesFolder: File = pluginLibrariesPath.toFile()
+
+    override val pluginSharedLibrariesPath: Path = MiraiConsole.rootPath.resolve("plugin-shared-libraries").apply { mkdir() }
+    override val pluginSharedLibrariesFolder: File = pluginSharedLibrariesPath.toFile()
+
     @Suppress("ObjectPropertyName")
     private val _pluginLoaders: MutableList<PluginLoader<*, *>> by lazy {
         builtInLoaders.toMutableList()

mirai-console/backend/mirai-console/src/internal/util/CommonUtils.kt

@@ -38,7 +38,7 @@ internal inline fun <reified E : Throwable> runIgnoreException(block: () -> Unit
 internal fun StackFrame.findLoader(): ClassLoader? {
     classInstance?.let { return it.classLoader }
     return runCatching {
-        JvmPluginLoader.implOrNull?.classLoaders?.firstOrNull { it.findClass(className, true) != null }
+        JvmPluginLoader.implOrNull?.findLoadedClass(className)?.classLoader
     }.getOrNull()
 }
 

mirai-console/backend/mirai-console/src/plugin/PluginManager.kt

@@ -82,6 +82,44 @@ public interface PluginManager {
      */
     public val pluginsConfigFolder: File
 
+    /**
+     * 插件运行时依赖存放路径 [Path], 插件自动下载的依赖都会存放于此目录
+     *
+     * **实现细节**: 在 terminal 前端实现为 `$rootPath/plugin-libraries`,
+     * 依赖 jar 文件由插件共享, 但是运行时插件加载的类是互相隔离的
+     *
+     * @since 2.11
+     */
+    public val pluginLibrariesPath: Path
+
+    /**
+     * 插件运行时依赖存放路径 [File], 插件自动下载的依赖都会存放于此目录
+     *
+     * **实现细节**: 在 terminal 前端实现为 `$rootPath/plugin-libraries`,
+     * 依赖 jar 文件由插件共享, 但是运行时插件加载的类是互相隔离的
+     *
+     * @since 2.11
+     */
+    public val pluginLibrariesFolder: File
+
+    /**
+     * 插件运行时依赖存放路径 [Path], 该路径下的依赖由全部插件共享
+     *
+     * **实现细节**: 在 terminal 前端实现为 `$rootPath/plugin-shared-libraries`
+     *
+     * @since 2.11
+     */
+    public val pluginSharedLibrariesPath: Path
+
+    /**
+     * 插件运行时依赖存放路径 [File], 该路径下的依赖由全部插件共享
+     *
+     * **实现细节**: 在 terminal 前端实现为 `$rootPath/plugin-shared-libraries`
+     *
+     * @since 2.11
+     */
+    public val pluginSharedLibrariesFolder: File
+
     // endregion
 
 

mirai-console/backend/mirai-console/src/plugin/jvm/JvmPluginLoader.kt

@@ -44,6 +44,9 @@ public interface JvmPluginLoader : CoroutineScope, FilePluginLoader<JvmPlugin, J
     @MiraiInternalApi
     public val classLoaders: List<ClassLoader>
 
+    @MiraiInternalApi
+    public fun findLoadedClass(name: String): Class<*>?
+
     public companion object BuiltIn :
         JvmPluginLoader by (dynamicDelegation { MiraiConsoleImplementation.getInstance().jvmPluginLoader }) {