[F] Fix export import missing fields and database errors (#236)

src/main/java/icu/samnyan/aqua/net/games/ImportController.kt

@@ -1,15 +1,18 @@
 package icu.samnyan.aqua.net.games
 
+import com.fasterxml.jackson.core.JsonProcessingException
 import ext.*
 import icu.samnyan.aqua.net.Fedy
 import icu.samnyan.aqua.net.db.AquaNetUser
 import icu.samnyan.aqua.net.db.AquaUserServices
+import icu.samnyan.aqua.net.utils.ApiException
 import icu.samnyan.aqua.net.utils.AquaNetProps
 import icu.samnyan.aqua.net.utils.SUCCESS
 import icu.samnyan.aqua.sega.general.model.Card
 import icu.samnyan.aqua.sega.general.service.CardService
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.context.annotation.Lazy
+import org.springframework.dao.DataIntegrityViolationException
 import org.springframework.data.jpa.repository.JpaRepository
 import org.springframework.data.repository.NoRepositoryBean
 import org.springframework.transaction.PlatformTransactionManager
@@ -104,11 +107,36 @@ abstract class ImportController<ExportModel: IExportClass<UserModel>, UserModel:
         export(u)
     }
 
+    internal fun replaceInTransaction(existingUserData: UserModel?, auId: Long, insert: () -> Unit) {
+        trans.execute {
+            existingUserData?.also { gu ->
+                // After migration v1000.7, all user-linked entities have ON DELETE CASCADE.
+                log.info("$game Import: Replacing old data for user $auId")
+                userDataRepo.delete(gu)
+                userDataRepo.flush()
+            }
+
+            insert()
+        }
+    }
+
+    private fun parseImport(json: String): ExportModel = try {
+        json.parseJackson(exportClass.java)
+    } catch (e: Exception) {
+        val jsonError = generateSequence<Throwable>(e) { it.cause }
+            .filterIsInstance<JsonProcessingException>()
+            .firstOrNull()
+            ?: throw e
+
+        log.warn("Rejected invalid $game import: ${jsonError.message}")
+        400 - "Invalid import data: ${jsonError.originalMessage}"
+    }
+
     @Suppress("UNCHECKED_CAST")
     @API("import")
     fun importUserData(@RP token: Str, @RB json: Str) = us.jwt.auth(token) { u ->
         try {
-            val export = json.parseJackson(exportClass.java)
+            val export = parseImport(json)
             if (!export.gameId.equals(game, true)) 400 - "Invalid game ID"
 
             val lists = listRepos.toList().associate { (f, r) -> r to f.get(export) as List<IUserEntity<UserModel>> }.vNotNull()
@@ -124,19 +152,15 @@ abstract class ImportController<ExportModel: IExportClass<UserModel>, UserModel:
             // Set user card
             export.userData.card = u.ghostCard
 
-            // Check existing data
-            userDataRepo.findByCard(u.ghostCard)?.also { gu ->
+            // Back up existing data before starting the replacement transaction.
+            val existingUserData = userDataRepo.findByCard(u.ghostCard)
+            existingUserData?.also {
                 // Store a backup of the old data
                 val fl = "${game}-backup-${u.auId}-${LocalDateTime.now().urlSafeStr()}.json"
                 (Path(netProps.importBackupPath) / fl).writeText(export(u).toJson())
-
-                // Delete the old data (After migration v1000.7, all user-linked entities have ON DELETE CASCADE)
-                log.info("$game Import: Deleting old data for user ${u.auId}")
-                userDataRepo.delete(gu)
-                userDataRepo.flush()
             }
 
-            trans.execute {
+            replaceInTransaction(existingUserData, u.auId) {
                 // Insert new data
                 val nu = userDataRepo.save(export.userData)
                 // Set user fields
@@ -155,6 +179,12 @@ abstract class ImportController<ExportModel: IExportClass<UserModel>, UserModel:
 
             SUCCESS
         } catch(e: Exception) {
+            if (e is ApiException) throw e
+            if (e is DataIntegrityViolationException) {
+                log.warn("Rejected conflicting $game import: ${e.message}")
+                400 - "Invalid import data: duplicate or conflicting records"
+            }
+
             log.error(e.message, e)
             500 - "Failed to import user data. More information can be found in the server logs (or contact an administrator for help if you do not have access)."
         }

src/main/java/icu/samnyan/aqua/sega/chusan/model/userdata/Chu3UserData.kt

@@ -17,15 +17,34 @@ import jakarta.persistence.*
 import kotlinx.io.IOException
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
+import java.time.format.DateTimeFormatterBuilder
+import java.time.format.ResolverStyle
+import java.time.temporal.ChronoField
 
 class FlexibleDateTimeDeserializer : JsonDeserializer<LocalDateTime?>() {
     @Throws(IOException::class)
-    public override fun deserialize(p: JsonParser, ctxt: DeserializationContext?): LocalDateTime {
-        return LocalDateTime.parse(p.getText(), FORMATTER)
+    public override fun deserialize(p: JsonParser, ctxt: DeserializationContext): LocalDateTime {
+        val value = p.text
+        return FORMATTERS.firstNotNullOfOrNull { formatter ->
+            runCatching { LocalDateTime.parse(value, formatter) }.getOrNull()
+        } ?: throw ctxt.weirdStringException(
+            value,
+            LocalDateTime::class.java,
+            "Invalid date-time; expected yyyy-MM-dd'T'HH:mm:ss or yyyy-MM-dd HH:mm:ss with an optional fractional second",
+        )
     }
+
     companion object {
-        // Chusan imports previous would fail because the pattern was too strict. Now the incorrect pattern should at least get accepted correctly
-        private val FORMATTER: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd[' ']['T']HH:mm:ss[.S][.SS][.SSS]")
+        private val SPACE_SEPARATED_FORMATTER = DateTimeFormatterBuilder()
+            .parseStrict()
+            .appendPattern("uuuu-MM-dd HH:mm:ss")
+            .optionalStart()
+            .appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, true)
+            .optionalEnd()
+            .toFormatter()
+            .withResolverStyle(ResolverStyle.STRICT)
+
+        private val FORMATTERS = listOf(DateTimeFormatter.ISO_LOCAL_DATE_TIME, SPACE_SEPARATED_FORMATTER)
     }
 }
 

src/main/java/icu/samnyan/aqua/sega/chusan/model/userdata/UserCMissionProgress.kt

@@ -1,6 +1,5 @@
 package icu.samnyan.aqua.sega.chusan.model.userdata
 
-import com.fasterxml.jackson.annotation.JsonIgnore
 import jakarta.persistence.Column
 import jakarta.persistence.Entity
 import jakarta.persistence.Table
@@ -10,7 +9,6 @@ import jakarta.persistence.UniqueConstraint
 @Table(name = "chusan_user_cmission_progress", uniqueConstraints = [UniqueConstraint(columnNames = ["user_id", "mission_id", "order"])])
 class UserCMissionProgress : Chu3UserEntity() {
     @Column(name = "mission_id")
-    @JsonIgnore
     var missionId = 0
     @Column(name = "`order`")
     var order = 0

src/test/kotlin/test/Chu3ImportJsonTest.kt

@@ -0,0 +1,58 @@
+package test
+
+import com.fasterxml.jackson.databind.JsonMappingException
+import ext.JACKSON
+import icu.samnyan.aqua.sega.chusan.model.userdata.Chu3UserData
+import icu.samnyan.aqua.sega.chusan.model.userdata.UserCMissionProgress
+import io.kotest.assertions.throwables.shouldThrow
+import io.kotest.core.spec.style.StringSpec
+import io.kotest.matchers.collections.shouldContainExactly
+import io.kotest.matchers.shouldBe
+import java.time.LocalDateTime
+
+class Chu3ImportJsonTest : StringSpec({
+    "mission progress preserves mission IDs with the same order" {
+        val progress = listOf(
+            UserCMissionProgress().apply {
+                missionId = 100
+                order = 0
+            },
+            UserCMissionProgress().apply {
+                missionId = 200
+                order = 0
+            },
+        )
+
+        val json = JACKSON.writeValueAsString(progress)
+        val imported = JACKSON.readerForListOf(UserCMissionProgress::class.java).readValue<List<UserCMissionProgress>>(json)
+
+        imported.map { it.missionId to it.order } shouldContainExactly listOf(100 to 0, 200 to 0)
+    }
+
+    "CHUNITHM dates accept ISO and database formats" {
+        val imported = JACKSON.readValue(
+            """{
+                "eventWatchedDate":"2026-08-04T01:02:03",
+                "firstPlayDate":"2026-08-04 01:02:03.123456",
+                "lastPlayDate":"2026-08-04 01:02:03.0"
+            }""",
+            Chu3UserData::class.java,
+        )
+
+        imported.eventWatchedDate shouldBe LocalDateTime.of(2026, 8, 4, 1, 2, 3)
+        imported.firstPlayDate shouldBe LocalDateTime.of(2026, 8, 4, 1, 2, 3, 123456000)
+        imported.lastPlayDate shouldBe LocalDateTime.of(2026, 8, 4, 1, 2, 3)
+    }
+
+    "invalid CHUNITHM dates are rejected with a useful message" {
+        val error = shouldThrow<JsonMappingException> {
+            JACKSON.readValue(
+                """{"firstPlayDate":"573-573-573T22:49:23"}""",
+                Chu3UserData::class.java,
+            )
+        }
+
+        error.originalMessage shouldBe
+            "Cannot deserialize value of type `java.time.LocalDateTime` from String \"573-573-573T22:49:23\": Invalid date-time; expected yyyy-MM-dd'T'HH:mm:ss or yyyy-MM-dd HH:mm:ss with an optional fractional second"
+    }
+})

src/test/kotlin/test/ImportTransactionTest.kt

@@ -0,0 +1,82 @@
+package test
+
+import icu.samnyan.aqua.net.games.GenericUserDataRepo
+import icu.samnyan.aqua.net.games.IExportClass
+import icu.samnyan.aqua.net.games.ImportController
+import icu.samnyan.aqua.sega.chusan.model.userdata.Chu3UserData
+import io.kotest.assertions.throwables.shouldThrow
+import io.kotest.core.spec.style.StringSpec
+import io.kotest.matchers.shouldBe
+import org.mockito.Mockito.doAnswer
+import org.mockito.Mockito.mock
+import org.springframework.transaction.PlatformTransactionManager
+import org.springframework.transaction.TransactionDefinition
+import org.springframework.transaction.TransactionStatus
+import org.springframework.transaction.support.SimpleTransactionStatus
+
+private data class TransactionTestExport(
+    override var gameId: String = "TEST",
+    override var userData: Chu3UserData = Chu3UserData(),
+) : IExportClass<Chu3UserData>
+
+private class TransactionTestImport(
+    override val userDataRepo: GenericUserDataRepo<Chu3UserData>,
+) : ImportController<TransactionTestExport, Chu3UserData>(
+    game = "TEST",
+    gameName = "test",
+    exportClass = TransactionTestExport::class,
+    exportFields = emptyMap(),
+    exportRepos = emptyMap(),
+    artemisRenames = emptyMap(),
+) {
+    override fun createEmpty() = TransactionTestExport()
+}
+
+private class TrackingTransactionManager : PlatformTransactionManager {
+    var active = false
+    var committed = false
+    var rolledBack = false
+
+    override fun getTransaction(definition: TransactionDefinition?): TransactionStatus {
+        active = true
+        return SimpleTransactionStatus()
+    }
+
+    override fun commit(status: TransactionStatus) {
+        committed = true
+        active = false
+    }
+
+    override fun rollback(status: TransactionStatus) {
+        rolledBack = true
+        active = false
+    }
+}
+
+class ImportTransactionTest : StringSpec({
+    "failed replacement rolls back deletion and insertion together" {
+        @Suppress("UNCHECKED_CAST")
+        val repo = mock(GenericUserDataRepo::class.java) as GenericUserDataRepo<Chu3UserData>
+        val transactionManager = TrackingTransactionManager()
+        val controller = TransactionTestImport(repo).apply {
+            transManager = transactionManager
+        }
+        val existing = Chu3UserData()
+        var deleteWasTransactional = false
+
+        doAnswer {
+            deleteWasTransactional = transactionManager.active
+            null
+        }.`when`(repo).delete(existing)
+
+        shouldThrow<IllegalStateException> {
+            controller.replaceInTransaction(existing, 123) {
+                throw IllegalStateException("simulated insert failure")
+            }
+        }
+
+        deleteWasTransactional shouldBe true
+        transactionManager.rolledBack shouldBe true
+        transactionManager.committed shouldBe false
+    }
+})