Improve duplication filtering. Fix #249, fix #580, fix #581, fix #590, fix #542, fix #567

mirai-core-qqandroid/src/commonMain/kotlin/net/mamoe/mirai/qqandroid/network/QQAndroidClient.kt

@@ -14,6 +14,7 @@ package net.mamoe.mirai.qqandroid.network
 import kotlinx.atomicfu.AtomicBoolean
 import kotlinx.atomicfu.AtomicInt
 import kotlinx.atomicfu.atomic
+import kotlinx.coroutines.sync.Mutex
 import kotlinx.io.core.*
 import net.mamoe.mirai.data.OnlineStatus
 import net.mamoe.mirai.network.LoginFailedException
@@ -212,6 +213,16 @@ internal open class QQAndroidClient(
         var syncCookie: ByteArray? = null
         var pubAccountCookie = EMPTY_BYTE_ARRAY
         var msgCtrlBuf: ByteArray = EMPTY_BYTE_ARRAY
+
+
+        internal data class SyncPacketIdentifier(
+            val uid: Long,
+            val sequence: Int,
+            val time: Int
+        )
+
+        val packetIdList = LinkedList<SyncPacketIdentifier>()
+        val packetIdListLock = Mutex()
     }
 
     val c2cMessageSync = C2cMessageSyncData()

mirai-core-qqandroid/src/commonMain/kotlin/net/mamoe/mirai/qqandroid/network/protocol/packet/chat/receive/MessageSvc.PbGetMsg.kt

@@ -17,7 +17,6 @@ import kotlinx.coroutines.FlowPreview
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.flow.*
 import kotlinx.coroutines.launch
-import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import kotlinx.io.core.ByteReadPacket
 import kotlinx.io.core.discardExact
@@ -63,10 +62,6 @@ import kotlin.random.Random
 internal object MessageSvcPbGetMsg : OutgoingPacketFactory<MessageSvcPbGetMsg.Response>("MessageSvc.PbGetMsg") {
 
 
-    private val msgUidQueue = ArrayDeque<Long>()
-    private val msgUidSet = hashSetOf<Long>()
-    private val msgQueueMutex = Mutex()
-
     @Suppress("SpellCheckingInspection")
     operator fun invoke(
         client: QQAndroidClient,
@@ -137,7 +132,7 @@ internal object MessageSvcPbGetMsg : OutgoingPacketFactory<MessageSvcPbGetMsg.Re
         }
     }
 
-    suspend fun QQAndroidBot.createGroupForBot(groupUin: Long): Group? {
+    private suspend fun QQAndroidBot.createGroupForBot(groupUin: Long): Group? {
         val group = getGroupByUinOrNull(groupUin)
         if (group != null) {
             return null
@@ -195,19 +190,21 @@ internal object MessageSvcPbGetMsg : OutgoingPacketFactory<MessageSvcPbGetMsg.Re
             }.also {
                 MessageSvcPbDeleteMsg.delete(bot, it) // 删除消息
             }
-            .mapNotNull<MsgComm.Msg, Packet> { msg ->
-
-                msgQueueMutex.lock()
-                val msgUid = msg.msgHead.msgUid
-                if (msgUidSet.size > 50) {
-                    msgUidSet.remove(msgUidQueue.removeFirst())
-                }
-                if (!msgUidSet.add(msgUid)) {
-                    msgQueueMutex.unlock()
-                    return@mapNotNull null
+            .mapNotNull { msg ->
+
+                bot.client.c2cMessageSync.run {
+                    val identifier = QQAndroidClient.C2cMessageSyncData.SyncPacketIdentifier(
+                        uid = msg.msgHead.msgUid,
+                        sequence = msg.msgHead.msgSeq,
+                        time = msg.msgHead.msgTime
+                    )
+
+                    packetIdListLock.withLock {
+                        if (packetIdList.contains(identifier)) return@mapNotNull null // duplicate
+                        packetIdList.addLast(identifier)
+                        if (packetIdList.size >= 50) packetIdList.removeFirst()
+                    }
                 }
-                msgQueueMutex.unlock()
-                msgUidQueue.addLast(msgUid)
 
                 when (msg.msgHead.msgType) {
                     33 -> bot.groupListModifyLock.withLock {

mirai-core-qqandroid/src/commonMain/kotlin/net/mamoe/mirai/qqandroid/utils/LinkedList.common.kt

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019-2020 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AFFERO GENERAL PUBLIC LICENSE version 3 license that can be found via the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/master/LICENSE
+ */
+
+package net.mamoe.mirai.qqandroid.utils
+
+// We target JVM and Android only.
+internal expect class LinkedList<E>() : List<E>, Queue<E>, Deque<E>
+
+internal interface Queue<E> : MutableCollection<E> {
+    override fun add(element: E): Boolean
+    fun offer(element: E): Boolean
+    fun remove(): E
+    fun poll(): E
+    fun element(): E
+    fun peek(): E
+}
+
+internal interface Deque<E> : Queue<E> {
+    fun addFirst(e: E)
+    fun addLast(e: E)
+    fun offerFirst(e: E): Boolean
+    fun offerLast(e: E): Boolean
+    fun removeFirst(): E
+    fun removeLast(): E
+    fun pollFirst(): E
+    fun pollLast(): E
+    val first: E
+    val last: E
+    fun peekFirst(): E
+    fun peekLast(): E
+    fun removeFirstOccurrence(o: E): Boolean
+    fun removeLastOccurrence(o: E): Boolean
+    fun push(e: E)
+    fun pop(): E
+    fun descendingIterator(): Iterator<E>
+}

mirai-core-qqandroid/src/jvmMain/kotlin/net/mamoe/mirai/qqandroid/utils/LinkedList.kt

@@ -0,0 +1,17 @@
+/*
+ *
+ *  * Copyright 2020 Mamoe Technologies and contributors.
+ *  *
+ *  * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ *  * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *  *
+ *  * https://github.com/mamoe/mirai/blob/master/LICENSE
+ *
+ */
+
+package net.mamoe.mirai.qqandroid.utils
+
+import java.util.LinkedList
+
+@Suppress("ACTUAL_WITHOUT_EXPECT")
+internal actual typealias LinkedList<E> = LinkedList<E>