feat(#1367): render agent MEDIA inline at its directive position (#1370)

This commit is contained in:
m57 authored and GitHub committed 2026-09-28 23:56:37 +03:00
1 parent 502882443e
commit 503aba863f
7 files changed
+200 -25

No files matched your search

@@ -17,6 +17,9 @@ import kotlinx.serialization.Serializable
* file that lives on the *gateway host* (agent-delivered `MEDIA:` directive).
* Null for locally-picked attachments.
* @param source Where the attachment originated.
* @param contentOffset For agent `MEDIA:` attachments, the char offset in the message's
* stripped text where the directive appeared, so the renderer can place it in-flow.
* Null means "append after the text".
*/
@Serializable
data class Attachment(
@@ -26,6 +29,7 @@ data class Attachment(
val size: Long = 0,
val gatewayUrl: String? = null,
val source: AttachmentSource = AttachmentSource.LOCAL,
val contentOffset: Int? = null,
) {
val mediaKind: MediaKind
get() = classifyMedia(mimeType = mimeType, name = name, uri = gatewayUrl ?: uri)
@@ -60,6 +60,7 @@ class ChatMediaDelegate(
size = 0,
gatewayUrl = url,
source = AttachmentSource.GATEWAY,
contentOffset = item.offset,
)
}
if (newAttachments.isEmpty()) return
@@ -247,6 +247,7 @@ internal fun mapServerMessages(
size = 0,
gatewayUrl = url,
source = AttachmentSource.GATEWAY,
contentOffset = item.offset,
)
}.takeIf { it.isNotEmpty() }
}
@@ -21,6 +21,8 @@ internal object HostMediaExtractor {
data class Item(
val match: String, // the full matched directive (for stripping)
val path: String, // normalized absolute path
// Issue #1367: char offset into strip(text) where the directive sat.
val offset: Int = 0,
)
/** Matches `MEDIA:<path>` (any extension), supporting quotes/backticks around the path. */
@@ -33,12 +35,18 @@ internal object HostMediaExtractor {
/** Extract every `MEDIA:<path>` directive from [text] as a normalized item. */
fun extract(text: String): List<Item> {
if (!text.contains("MEDIA:")) return emptyList()
val total = strip(text).length
// Text kept so far, with every directive (valid or not) removed exactly as strip() does.
val kept = StringBuilder()
var cursor = 0
return MEDIA_TAG_RE
.findAll(text)
.mapNotNull { m ->
kept.append(text, cursor, m.range.first)
cursor = m.range.last + 1
val raw = m.groupValues.drop(1).firstOrNull { it.isNotBlank() } ?: return@mapNotNull null
val path = normalizePath(raw) ?: return@mapNotNull null
Item(match = m.value, path = path)
Item(match = m.value, path = path, offset = strip(kept.toString()).length.coerceAtMost(total))
}.toList()
}
@@ -0,0 +1,84 @@
package com.m57.hermescontrol.ui.chat
import com.m57.hermescontrol.data.model.Attachment
/** Issue #1367: one in-order piece of an agent message body. */
internal sealed interface MessageSegment {
data class Text(
val text: String,
) : MessageSegment
data class Media(
val attachment: Attachment,
) : MessageSegment
}
/**
* Interleave [attachments] into [content] at the position their `MEDIA:` directive occupied
* ([Attachment.contentOffset]). Attachments without an offset trail the text, as before.
*
* Offsets snap forward to the end of their line, and past a still-open code fence, so media
* never splits a sentence's markdown, list item, or fenced block.
*/
internal fun splitByMedia(
content: String,
attachments: List<Attachment>?,
): List<MessageSegment> {
if (attachments.isNullOrEmpty()) {
return if (content.isBlank()) emptyList() else listOf(MessageSegment.Text(content))
}
val inline = attachments.filter { it.contentOffset != null }.sortedBy { it.contentOffset }
val trailing = attachments.filter { it.contentOffset == null }
val out = mutableListOf<MessageSegment>()
var cursor = 0
inline.forEach { attachment ->
val pos = snapToBlockEnd(content, attachment.contentOffset ?: 0).coerceAtLeast(cursor)
addText(out, content.substring(cursor, pos))
out += MessageSegment.Media(attachment)
cursor = pos
}
addText(out, content.substring(cursor))
trailing.forEach { out += MessageSegment.Media(it) }
return out
}
private fun addText(
out: MutableList<MessageSegment>,
text: String,
) {
if (text.isNotBlank()) out += MessageSegment.Text(text.trim('\n'))
}
private fun snapToBlockEnd(
content: String,
offset: Int,
): Int {
var pos = offset.coerceIn(0, content.length)
pos = lineEnd(content, pos)
// Inside an unclosed fence? Advance to the end of the closing fence line.
while (fenceOpenBefore(content, pos)) {
val next = pos + 1
if (next > content.length) return content.length
pos = lineEnd(content, next)
}
return pos
}
private fun lineEnd(
content: String,
from: Int,
): Int {
val idx = content.indexOf('\n', from)
return if (idx < 0) content.length else idx
}
private fun fenceOpenBefore(
content: String,
pos: Int,
): Boolean {
var open = false
content.substring(0, pos).lineSequence().forEach { line ->
if (line.trimStart().startsWith("```")) open = !open
}
return open
}
@@ -23,6 +23,7 @@ import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
@@ -36,11 +37,13 @@ import com.m57.hermescontrol.R
import com.m57.hermescontrol.data.model.Attachment
import com.m57.hermescontrol.ui.chat.ChatMessage
import com.m57.hermescontrol.ui.chat.ImageViewerModel
import com.m57.hermescontrol.ui.chat.InlineAttachmentList
import com.m57.hermescontrol.ui.chat.InlineAttachment
import com.m57.hermescontrol.ui.chat.MarkdownText
import com.m57.hermescontrol.ui.chat.MessageSegment
import com.m57.hermescontrol.ui.chat.TokenEstimator
import com.m57.hermescontrol.ui.chat.components.ReasoningCard
import com.m57.hermescontrol.ui.chat.components.rememberCopyFeedback
import com.m57.hermescontrol.ui.chat.splitByMedia
import kotlinx.coroutines.launch
/**
@@ -96,32 +99,44 @@ internal fun FullBleedAgentMessage(
// lone Copy button). Blank rows are tool-call placeholders that slipped
// through upstream mapping; the parent list renders the live status
// indicator until the first visible delta lands.
if (message.content.isNotBlank()) {
SelectionContainer {
MarkdownText(
text = message.content,
textColor = textColor,
isStreaming = message.isStreaming,
searchQuery = searchQuery,
isCurrentMatch = isCurrentMatch,
onImageClick = onImageClick,
)
// Issue #1367: agent media renders where its `MEDIA:` directive sat in the
// prose; attachments without a recorded position trail the text.
val segments =
remember(message.content, message.attachments) {
splitByMedia(message.content, message.attachments)
}
segments.forEach { segment ->
when (segment) {
is MessageSegment.Text -> {
SelectionContainer {
MarkdownText(
text = segment.text,
textColor = textColor,
isStreaming = message.isStreaming,
searchQuery = searchQuery,
isCurrentMatch = isCurrentMatch,
onImageClick = onImageClick,
)
}
}
is MessageSegment.Media -> {
Spacer(modifier = Modifier.height(6.dp))
InlineAttachment(
attachment = segment.attachment,
textColor = textColor,
onOpen = onOpenAttachment,
onSave = onSaveAttachment,
savingPath = savingAttachmentPath,
openingPath = openingAttachmentPath,
canSave = canSaveAttachment,
onImageClick = onImageClick,
)
Spacer(modifier = Modifier.height(6.dp))
}
}
}
// Inline attachments (shared with UserBubble so agent-delivered
// media — images, files — shows in full-bleed mode too).
InlineAttachmentList(
attachments = message.attachments,
textColor = textColor,
onOpen = onOpenAttachment,
onSave = onSaveAttachment,
savingPath = savingAttachmentPath,
openingPath = openingAttachmentPath,
canSave = canSaveAttachment,
onImageClick = onImageClick,
)
if (!message.isStreaming && message.content.isNotBlank()) {
val showTokenStat =
messageStatsEnabled && showAssistantMessageTokens &&