From 503aba863fd987b5e27dc2be49dbc6d3ad975750 Mon Sep 17 00:00:00 2001 From: m57 <41645758+Hy4ri@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:56:37 +0300 Subject: [PATCH] feat(#1367): render agent MEDIA inline at its directive position (#1370) --- .../hermescontrol/data/model/Attachment.kt | 4 + .../ui/chat/ChatMediaDelegate.kt | 1 + .../ui/chat/ChatServerMessageMapper.kt | 1 + .../ui/chat/HostMediaExtractor.kt | 10 ++- .../hermescontrol/ui/chat/MessageSegment.kt | 84 +++++++++++++++++++ .../chat/fullbleed/FullBleedAgentMessage.kt | 63 ++++++++------ .../ui/chat/MediaSegmentsTest.kt | 62 ++++++++++++++ 7 files changed, 200 insertions(+), 25 deletions(-) create mode 100644 app/src/main/java/com/m57/hermescontrol/ui/chat/MessageSegment.kt create mode 100644 app/src/test/java/com/m57/hermescontrol/ui/chat/MediaSegmentsTest.kt diff --git a/app/src/main/java/com/m57/hermescontrol/data/model/Attachment.kt b/app/src/main/java/com/m57/hermescontrol/data/model/Attachment.kt index f9b2bf17..c45078c2 100644 --- a/app/src/main/java/com/m57/hermescontrol/data/model/Attachment.kt +++ b/app/src/main/java/com/m57/hermescontrol/data/model/Attachment.kt @@ -17,6 +17,9 @@ import kotlinx.serialization.Serializable * file that lives on the *gateway host* (agent-delivered `MEDIA:` directive). * Null for locally-picked attachments. * @param source Where the attachment originated. + * @param contentOffset For agent `MEDIA:` attachments, the char offset in the message's + * stripped text where the directive appeared, so the renderer can place it in-flow. + * Null means "append after the text". */ @Serializable data class Attachment( @@ -26,6 +29,7 @@ data class Attachment( val size: Long = 0, val gatewayUrl: String? = null, val source: AttachmentSource = AttachmentSource.LOCAL, + val contentOffset: Int? = null, ) { val mediaKind: MediaKind get() = classifyMedia(mimeType = mimeType, name = name, uri = gatewayUrl ?: uri) diff --git a/app/src/main/java/com/m57/hermescontrol/ui/chat/ChatMediaDelegate.kt b/app/src/main/java/com/m57/hermescontrol/ui/chat/ChatMediaDelegate.kt index bb8f0afe..a4f8b891 100644 --- a/app/src/main/java/com/m57/hermescontrol/ui/chat/ChatMediaDelegate.kt +++ b/app/src/main/java/com/m57/hermescontrol/ui/chat/ChatMediaDelegate.kt @@ -60,6 +60,7 @@ class ChatMediaDelegate( size = 0, gatewayUrl = url, source = AttachmentSource.GATEWAY, + contentOffset = item.offset, ) } if (newAttachments.isEmpty()) return diff --git a/app/src/main/java/com/m57/hermescontrol/ui/chat/ChatServerMessageMapper.kt b/app/src/main/java/com/m57/hermescontrol/ui/chat/ChatServerMessageMapper.kt index 53bfd0d8..a620b5bd 100644 --- a/app/src/main/java/com/m57/hermescontrol/ui/chat/ChatServerMessageMapper.kt +++ b/app/src/main/java/com/m57/hermescontrol/ui/chat/ChatServerMessageMapper.kt @@ -247,6 +247,7 @@ internal fun mapServerMessages( size = 0, gatewayUrl = url, source = AttachmentSource.GATEWAY, + contentOffset = item.offset, ) }.takeIf { it.isNotEmpty() } } diff --git a/app/src/main/java/com/m57/hermescontrol/ui/chat/HostMediaExtractor.kt b/app/src/main/java/com/m57/hermescontrol/ui/chat/HostMediaExtractor.kt index d969dc58..c47d67ac 100644 --- a/app/src/main/java/com/m57/hermescontrol/ui/chat/HostMediaExtractor.kt +++ b/app/src/main/java/com/m57/hermescontrol/ui/chat/HostMediaExtractor.kt @@ -21,6 +21,8 @@ internal object HostMediaExtractor { data class Item( val match: String, // the full matched directive (for stripping) val path: String, // normalized absolute path + // Issue #1367: char offset into strip(text) where the directive sat. + val offset: Int = 0, ) /** Matches `MEDIA:` (any extension), supporting quotes/backticks around the path. */ @@ -33,12 +35,18 @@ internal object HostMediaExtractor { /** Extract every `MEDIA:` directive from [text] as a normalized item. */ fun extract(text: String): List { if (!text.contains("MEDIA:")) return emptyList() + val total = strip(text).length + // Text kept so far, with every directive (valid or not) removed exactly as strip() does. + val kept = StringBuilder() + var cursor = 0 return MEDIA_TAG_RE .findAll(text) .mapNotNull { m -> + kept.append(text, cursor, m.range.first) + cursor = m.range.last + 1 val raw = m.groupValues.drop(1).firstOrNull { it.isNotBlank() } ?: return@mapNotNull null val path = normalizePath(raw) ?: return@mapNotNull null - Item(match = m.value, path = path) + Item(match = m.value, path = path, offset = strip(kept.toString()).length.coerceAtMost(total)) }.toList() } diff --git a/app/src/main/java/com/m57/hermescontrol/ui/chat/MessageSegment.kt b/app/src/main/java/com/m57/hermescontrol/ui/chat/MessageSegment.kt new file mode 100644 index 00000000..95bd57b4 --- /dev/null +++ b/app/src/main/java/com/m57/hermescontrol/ui/chat/MessageSegment.kt @@ -0,0 +1,84 @@ +package com.m57.hermescontrol.ui.chat + +import com.m57.hermescontrol.data.model.Attachment + +/** Issue #1367: one in-order piece of an agent message body. */ +internal sealed interface MessageSegment { + data class Text( + val text: String, + ) : MessageSegment + + data class Media( + val attachment: Attachment, + ) : MessageSegment +} + +/** + * Interleave [attachments] into [content] at the position their `MEDIA:` directive occupied + * ([Attachment.contentOffset]). Attachments without an offset trail the text, as before. + * + * Offsets snap forward to the end of their line, and past a still-open code fence, so media + * never splits a sentence's markdown, list item, or fenced block. + */ +internal fun splitByMedia( + content: String, + attachments: List?, +): List { + if (attachments.isNullOrEmpty()) { + return if (content.isBlank()) emptyList() else listOf(MessageSegment.Text(content)) + } + val inline = attachments.filter { it.contentOffset != null }.sortedBy { it.contentOffset } + val trailing = attachments.filter { it.contentOffset == null } + val out = mutableListOf() + var cursor = 0 + inline.forEach { attachment -> + val pos = snapToBlockEnd(content, attachment.contentOffset ?: 0).coerceAtLeast(cursor) + addText(out, content.substring(cursor, pos)) + out += MessageSegment.Media(attachment) + cursor = pos + } + addText(out, content.substring(cursor)) + trailing.forEach { out += MessageSegment.Media(it) } + return out +} + +private fun addText( + out: MutableList, + text: String, +) { + if (text.isNotBlank()) out += MessageSegment.Text(text.trim('\n')) +} + +private fun snapToBlockEnd( + content: String, + offset: Int, +): Int { + var pos = offset.coerceIn(0, content.length) + pos = lineEnd(content, pos) + // Inside an unclosed fence? Advance to the end of the closing fence line. + while (fenceOpenBefore(content, pos)) { + val next = pos + 1 + if (next > content.length) return content.length + pos = lineEnd(content, next) + } + return pos +} + +private fun lineEnd( + content: String, + from: Int, +): Int { + val idx = content.indexOf('\n', from) + return if (idx < 0) content.length else idx +} + +private fun fenceOpenBefore( + content: String, + pos: Int, +): Boolean { + var open = false + content.substring(0, pos).lineSequence().forEach { line -> + if (line.trimStart().startsWith("```")) open = !open + } + return open +} diff --git a/app/src/main/java/com/m57/hermescontrol/ui/chat/fullbleed/FullBleedAgentMessage.kt b/app/src/main/java/com/m57/hermescontrol/ui/chat/fullbleed/FullBleedAgentMessage.kt index 318f3806..ec21135c 100644 --- a/app/src/main/java/com/m57/hermescontrol/ui/chat/fullbleed/FullBleedAgentMessage.kt +++ b/app/src/main/java/com/m57/hermescontrol/ui/chat/fullbleed/FullBleedAgentMessage.kt @@ -23,6 +23,7 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment @@ -36,11 +37,13 @@ import com.m57.hermescontrol.R import com.m57.hermescontrol.data.model.Attachment import com.m57.hermescontrol.ui.chat.ChatMessage import com.m57.hermescontrol.ui.chat.ImageViewerModel -import com.m57.hermescontrol.ui.chat.InlineAttachmentList +import com.m57.hermescontrol.ui.chat.InlineAttachment import com.m57.hermescontrol.ui.chat.MarkdownText +import com.m57.hermescontrol.ui.chat.MessageSegment import com.m57.hermescontrol.ui.chat.TokenEstimator import com.m57.hermescontrol.ui.chat.components.ReasoningCard import com.m57.hermescontrol.ui.chat.components.rememberCopyFeedback +import com.m57.hermescontrol.ui.chat.splitByMedia import kotlinx.coroutines.launch /** @@ -96,32 +99,44 @@ internal fun FullBleedAgentMessage( // lone Copy button). Blank rows are tool-call placeholders that slipped // through upstream mapping; the parent list renders the live status // indicator until the first visible delta lands. - if (message.content.isNotBlank()) { - SelectionContainer { - MarkdownText( - text = message.content, - textColor = textColor, - isStreaming = message.isStreaming, - searchQuery = searchQuery, - isCurrentMatch = isCurrentMatch, - onImageClick = onImageClick, - ) + // Issue #1367: agent media renders where its `MEDIA:` directive sat in the + // prose; attachments without a recorded position trail the text. + val segments = + remember(message.content, message.attachments) { + splitByMedia(message.content, message.attachments) + } + segments.forEach { segment -> + when (segment) { + is MessageSegment.Text -> { + SelectionContainer { + MarkdownText( + text = segment.text, + textColor = textColor, + isStreaming = message.isStreaming, + searchQuery = searchQuery, + isCurrentMatch = isCurrentMatch, + onImageClick = onImageClick, + ) + } + } + + is MessageSegment.Media -> { + Spacer(modifier = Modifier.height(6.dp)) + InlineAttachment( + attachment = segment.attachment, + textColor = textColor, + onOpen = onOpenAttachment, + onSave = onSaveAttachment, + savingPath = savingAttachmentPath, + openingPath = openingAttachmentPath, + canSave = canSaveAttachment, + onImageClick = onImageClick, + ) + Spacer(modifier = Modifier.height(6.dp)) + } } } - // Inline attachments (shared with UserBubble so agent-delivered - // media — images, files — shows in full-bleed mode too). - InlineAttachmentList( - attachments = message.attachments, - textColor = textColor, - onOpen = onOpenAttachment, - onSave = onSaveAttachment, - savingPath = savingAttachmentPath, - openingPath = openingAttachmentPath, - canSave = canSaveAttachment, - onImageClick = onImageClick, - ) - if (!message.isStreaming && message.content.isNotBlank()) { val showTokenStat = messageStatsEnabled && showAssistantMessageTokens && diff --git a/app/src/test/java/com/m57/hermescontrol/ui/chat/MediaSegmentsTest.kt b/app/src/test/java/com/m57/hermescontrol/ui/chat/MediaSegmentsTest.kt new file mode 100644 index 00000000..fd1ce63f --- /dev/null +++ b/app/src/test/java/com/m57/hermescontrol/ui/chat/MediaSegmentsTest.kt @@ -0,0 +1,62 @@ +package com.m57.hermescontrol.ui.chat + +import com.m57.hermescontrol.data.model.Attachment +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class MediaSegmentsTest { + private fun att( + name: String, + offset: Int?, + ) = Attachment(uri = "u/$name", name = name, mimeType = "image/png", contentOffset = offset) + + private fun kinds(segments: List) = + segments.map { + when (it) { + is MessageSegment.Text -> "T:${it.text}" + is MessageSegment.Media -> "M:${it.attachment.name}" + } + } + + @Test + fun extractRecordsOffsetsIntoStrippedText() { + val text = "First: MEDIA:/a/1.png\nthen second: MEDIA:/a/2.png\nend" + val stripped = HostMediaExtractor.strip(text) + val items = HostMediaExtractor.extract(text) + assertEquals(2, items.size) + assertTrue(items[0].offset <= items[1].offset) + assertTrue(items[1].offset <= stripped.length) + } + + @Test + fun mediaLandsBetweenTheParagraphsItSatBetween() { + val text = "Chart one:\nMEDIA:/a/1.png\nChart two:\nMEDIA:/a/2.png\nDone." + val stripped = HostMediaExtractor.strip(text) + val items = HostMediaExtractor.extract(text) + val atts = items.map { att(it.path.substringAfterLast('/'), it.offset) } + val out = kinds(splitByMedia(stripped, atts)) + assertEquals(listOf("T:Chart one:", "M:1.png", "T:Chart two:", "M:2.png", "T:Done."), out) + } + + @Test + fun noOffsetAppendsAfterText() { + val out = kinds(splitByMedia("Hello", listOf(att("x.png", null)))) + assertEquals(listOf("T:Hello", "M:x.png"), out) + } + + @Test + fun mediaOnlyMessageHasNoBlankText() { + val out = kinds(splitByMedia("", listOf(att("x.png", 0)))) + assertEquals(listOf("M:x.png"), out) + } + + @Test + fun mediaDoesNotSplitAFencedCodeBlock() { + val content = "Look:\n```\ncode line\nmore\n```\nafter" + val out = kinds(splitByMedia(content, listOf(att("x.png", content.indexOf("code line"))))) + assertEquals(2, out.count { it.startsWith("T:") }) + assertTrue(out[0].contains("more") && out[0].endsWith("```")) + assertEquals("M:x.png", out[1]) + } +}