adamlamers/zep
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix screen sharing

Browse Source
This commit is contained in:
Adam Lamers
2026-04-21 02:29:33 -04:00
parent 43c44ec05e
commit 466678052e
5 changed files with 155 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/chat/components/MemberList.svelte
+56 -16
View File
@@ -1,25 +1,34 @@
<script lang="ts">
import { getContext } from "svelte";
import type { ChatService } from "../services/chat.svelte";
import type { WebRTCService } from "../services/webrtc/webrtc.svelte";
import type * as Types from "../../module_bindings/types";
import Avatar from "./Avatar.svelte";
import Skeleton from "./Skeleton.svelte";
const chat = getContext<ChatService>("chat");
const webrtc = getContext<WebRTCService>("webrtc");
let onlineMembers = $derived(chat.activeServerMembers.filter((m) => m.online));
let offlineMembers = $derived(chat.activeServerMembers.filter((m) => !m.online));
function isTalking(member: Types.ServerMember) {
return chat.userStates.find((s) => s.identity.isEqual(member.identity))?.isTalking || false;
function getVoiceState(member: Types.ServerMember) {
return chat.userStates.find((s) => s.identity.isEqual(member.identity));
}
function isSharing(member: Types.ServerMember) {
return chat.userStates.find((s) => s.identity.isEqual(member.identity))?.isSharingScreen || false;
}
function getVoiceStatusColor(member: Types.ServerMember) {
const state = getVoiceState(member);
if (!state || state.channelId === undefined) return null;
function isMe(member: Types.ServerMember) {
return chat.identity?.isEqual(member.identity);
if (chat.identity?.isEqual(member.identity)) return "green";
const status = webrtc.peerStatuses.get(member.identity.toHexString());
if (!status) return "yellow";
const [iceState] = status.toLowerCase().split("/");
if (iceState.includes("connected") || iceState.includes("completed")) return "green";
if (iceState.includes("connecting") || iceState.includes("checking") || iceState.includes("new")) return "yellow";
return "red";
}
function getStatus(member: Types.ServerMember) {
@@ -57,13 +66,18 @@
</div>
{#each onlineMembers as member (member.identity.toHexString())}
{@const status = getStatus(member)}
{@const voiceState = getVoiceState(member)}
{@const voiceColor = getVoiceStatusColor(member)}
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div class="member-item" oncontextmenu={(e) => handleContextMenu(e, member)}>
<Avatar user={member} size="tiny" isTalking={isTalking(member)} />
<div class="avatar-container">
<Avatar user={member} size="small" isTalking={voiceState?.isTalking} />
<div class="status-dot green overlay"></div>
</div>
<div class="member-details">
<span class="member-name {isTalking(member) ? 'talking' : ''}">
<span class="member-name {voiceState?.isTalking ? 'talking' : ''}">
{member.name || member.identity.toHexString().substring(0, 8)}
{#if isMe(member)}
{#if chat.identity?.isEqual(member.identity)}
<span class="me-badge">(You)</span>
{/if}
</span>
@@ -72,10 +86,14 @@
{/if}
</div>
<div style="flex: 1;"></div>
{#if isSharing(member)}
<div class="member-indicators">
{#if voiceState?.isSharingScreen}
<span class="sharing-badge">LIVE</span>
{/if}
<div class="status-dot green"></div>
{#if voiceColor}
<div class="status-dot {voiceColor}" title="Voice connection status"></div>
{/if}
</div>
</div>
{/each}
{/if}
@@ -88,11 +106,14 @@
{@const status = getStatus(member)}
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div class="member-item offline" oncontextmenu={(e) => handleContextMenu(e, member)}>
<Avatar user={member} size="tiny" />
<div class="avatar-container">
<Avatar user={member} size="small" />
<div class="status-dot grey overlay"></div>
</div>
<div class="member-details">
<span class="member-name">
{member.name || member.identity.toHexString().substring(0, 8)}
{#if isMe(member)}
{#if chat.identity?.isEqual(member.identity)}
<span class="me-badge">(You)</span>
{/if}
</span>
@@ -100,8 +121,6 @@
<div class="member-status" title={status}>{status}</div>
{/if}
</div>
<div style="flex: 1;"></div>
<div class="status-dot grey"></div>
</div>
{/each}
{/if}
@@ -110,6 +129,27 @@
</div>
<style>
.avatar-container {
position: relative;
flex-shrink: 0;
}
.status-dot.overlay {
position: absolute;
bottom: -2px;
right: -2px;
border: 3px solid var(--background-secondary);
width: 14px;
height: 14px;
z-index: 2;
}
.member-indicators {
display: flex;
align-items: center;
gap: 4px;
}
.me-badge {
font-size: 0.7rem;
opacity: 0.6;
src/chat/components/channels/VoiceChannelGroup.svelte
+1 -1
View File
@@ -146,7 +146,7 @@
<span class="voice-indicator-icon" title="Muted"><i class="fas fa-microphone-slash"></i></span>
{/if}
{#if isLocalUserInThisChannel && (s.isDeafened || s.isMuted)}
{#if isLocalUserInThisChannel}
<div class="status-dot {voiceStatusColor}"></div>
{/if}
</div>
src/chat/services/webrtc/channel-audio-webrtc.svelte.ts
+25 -12
View File
@@ -63,16 +63,13 @@ export class ChannelAudioWebRTCService {
void this.peerManager.peerStatuses;
this.peerManager.peers.forEach(async (peer, peerIdHex) => {
const transceivers = peer.pc.getTransceivers();
if (transceivers[0] && transceivers[0].sender.track !== audioTrack) {
try {
console.log(
`[WebRTC][voice] Syncing track for ${peerIdHex} (track: ${audioTrack?.id})`,
);
await transceivers[0].sender.replaceTrack(audioTrack);
const audioTransceiver = transceivers.find(t => t.receiver.track.kind === 'audio' || t.sender.track?.kind === 'audio');
if (audioTransceiver && audioTransceiver.sender.track !== audioTrack) {
try {
console.log(`[WebRTC][voice] Syncing track for ${peerIdHex}`);
await audioTransceiver.sender.replaceTrack(audioTrack);
// If we just attached a real track and we are stable,
// we might need to negotiate if the remote side didn't know we have a track.
// But replaceTrack usually doesn't need negotiation if the transceiver was already sendrecv.
if (audioTrack && peer.pc.signalingState === "stable") {
this.onNegotiationNeeded(peerIdHex, peer.pc);
}
@@ -129,9 +126,14 @@ export class ChannelAudioWebRTCService {
peersToConnect.forEach((id) => {
if (!this.peerManager.peers.has(id)) {
console.log(`[WebRTC][voice] Initiating mesh connection to ${id}`);
this.peerManager.createPeerConnection(id, [
this.localStream?.getAudioTracks()[0] || null,
]);
const pc = this.peerManager.createPeerConnection(id);
if (pc) {
const audioTrack = this.localStream?.getAudioTracks()[0] || null;
const transceiver = pc.addTransceiver('audio', { direction: 'sendrecv' });
if (audioTrack) {
transceiver.sender.replaceTrack(audioTrack);
}
}
}
});
@@ -270,6 +272,17 @@ export class ChannelAudioWebRTCService {
await pc.setRemoteDescription(
new RTCSessionDescription(JSON.parse(signal.data)),
);
// Map local track to the transceiver created by the offer
const transceivers = pc.getTransceivers();
const audioTrack = this.localStream?.getAudioTracks()[0];
const audioTransceiver = transceivers.find(t => t.receiver.track.kind === 'audio');
if (audioTransceiver && audioTrack) {
await audioTransceiver.sender.replaceTrack(audioTrack);
audioTransceiver.direction = 'sendrecv';
}
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
console.log(`[WebRTC][voice] Sending Answer to ${peerIdHex.substring(0,8)}`);
src/chat/services/webrtc/peer-manager.svelte.ts
+28 -37
View File
@@ -196,7 +196,7 @@ export class PeerManagerService {
createPeerConnection = (
peerIdHex: string,
initialTracks: (MediaStreamTrack | null)[] = [],
_initialTracks: (MediaStreamTrack | null)[] = [],
) => {
if (this.peers.has(peerIdHex)) return this.peers.get(peerIdHex)!.pc;
@@ -276,7 +276,10 @@ export class PeerManagerService {
`[WebRTC][${this.mediaType}] Received track from ${peerIdHex}: ${event.track.kind} (id: ${event.track.id})`,
);
const nextPeers = new Map(this.peers);
const existingPeer = { ...(nextPeers.get(peerIdHex) || { pc }) };
const existingPeer = nextPeers.get(peerIdHex);
if (!existingPeer) return;
const peerUpdate = { ...existingPeer };
if (event.track.kind === "audio") {
if (this.mediaType === "voice") {
@@ -289,12 +292,12 @@ export class PeerManagerService {
const ctx = this.getAudioContext();
const stream = new MediaStream([event.track]);
if (!existingPeer.audio) {
existingPeer.audio = new Audio();
existingPeer.audio.muted = true;
if (!peerUpdate.audio) {
peerUpdate.audio = new Audio();
peerUpdate.audio.muted = true;
}
existingPeer.audio.srcObject = stream;
existingPeer.audio.play().catch((err) => {
peerUpdate.audio.srcObject = stream;
peerUpdate.audio.play().catch((err) => {
if (err.name !== "AbortError")
console.warn(
`[WebRTC][voice] Dummy audio play failed for ${peerIdHex}`,
@@ -310,7 +313,7 @@ export class PeerManagerService {
source.connect(gainNode);
gainNode.connect(ctx.destination);
existingPeer.gainNode = gainNode;
peerUpdate.gainNode = gainNode;
console.log(
`[WebRTC][voice] Web Audio graph connected for ${peerIdHex} (volume: ${pref.volume})`,
);
@@ -319,21 +322,21 @@ export class PeerManagerService {
`[WebRTC][voice] Failed to setup Web Audio for ${peerIdHex}, falling back to HTMLAudioElement`,
e,
);
if (!existingPeer.audio) {
existingPeer.audio = new Audio();
existingPeer.audio.autoplay = true;
if (!peerUpdate.audio) {
peerUpdate.audio = new Audio();
peerUpdate.audio.autoplay = true;
const pref = this.peerAudioPreferences.get(peerIdHex) || {
volume: 1,
muted: false,
};
existingPeer.audio.volume = Math.min(1, pref.volume);
existingPeer.audio.muted = this.isDeafened || pref.muted;
peerUpdate.audio.volume = Math.min(1, pref.volume);
peerUpdate.audio.muted = this.isDeafened || pref.muted;
}
const stream = new MediaStream([event.track]);
existingPeer.audio.srcObject = stream;
existingPeer.audio.play().catch((err) => {
peerUpdate.audio.srcObject = stream;
peerUpdate.audio.play().catch((err) => {
if (err.name !== "AbortError")
console.error(
`[WebRTC][voice] Error playing audio for ${peerIdHex}`,
@@ -342,44 +345,32 @@ export class PeerManagerService {
});
}
} else {
const currentStream = existingPeer.videoStream || new MediaStream();
const currentStream = peerUpdate.videoStream || new MediaStream();
if (!currentStream.getTracks().find((t) => t.id === event.track.id)) {
currentStream.addTrack(event.track);
}
existingPeer.videoStream = new MediaStream(currentStream.getTracks());
peerUpdate.videoStream = new MediaStream(currentStream.getTracks());
}
} else if (event.track.kind === "video") {
const currentStream = existingPeer.videoStream || new MediaStream();
const currentStream = peerUpdate.videoStream || new MediaStream();
if (!currentStream.getTracks().find((t) => t.id === event.track.id)) {
currentStream.addTrack(event.track);
}
existingPeer.videoStream = new MediaStream(currentStream.getTracks());
peerUpdate.videoStream = new MediaStream(currentStream.getTracks());
}
nextPeers.set(peerIdHex, existingPeer);
nextPeers.set(peerIdHex, peerUpdate);
this.peers = nextPeers;
};
if (this.mediaType === "voice") {
pc.addTransceiver("audio", { direction: "sendrecv" });
} else {
pc.addTransceiver("video", { direction: "sendrecv" });
pc.addTransceiver("audio", { direction: "sendrecv" });
}
const transceivers = pc.getTransceivers();
initialTracks.forEach((track, i) => {
if (track && transceivers[i]) {
console.log(
`[WebRTC][${this.mediaType}] Attaching initial track ${i} to ${peerIdHex}`,
);
transceivers[i].sender.replaceTrack(track);
}
});
const nextPeers = new Map(this.peers);
nextPeers.set(peerIdHex, { pc });
this.peers = nextPeers;
// We no longer add transceivers here.
// The calling service is responsible for adding them if it's the offerer,
// or setRemoteDescription will create them if it's the answerer.
return pc;
};
src/chat/services/webrtc/screen-sharing-webrtc.svelte.ts
+43 -14
View File
@@ -61,20 +61,21 @@ export class ScreenSharingWebRTCService {
this.peerManager.peers.forEach(async (peer, peerIdHex) => {
const transceivers = peer.pc.getTransceivers();
let changed = false;
if (transceivers[0] && transceivers[0].sender.track !== videoTrack) {
console.log(
`[WebRTC][screen] Syncing video track for ${peerIdHex} (track: ${videoTrack?.id})`,
);
await transceivers[0].sender.replaceTrack(videoTrack);
const videoTransceiver = transceivers.find(t => t.receiver.track.kind === 'video' || t.sender.track?.kind === 'video');
const audioTransceiver = transceivers.find(t => t.receiver.track.kind === 'audio' || t.sender.track?.kind === 'audio');
if (videoTransceiver && videoTransceiver.sender.track !== videoTrack) {
console.log(`[WebRTC][screen] Syncing video track for ${peerIdHex}`);
await videoTransceiver.sender.replaceTrack(videoTrack);
changed = true;
}
if (transceivers[1] && transceivers[1].sender.track !== audioTrack) {
console.log(
`[WebRTC][screen] Syncing audio track for ${peerIdHex} (track: ${audioTrack?.id})`,
);
await transceivers[1].sender.replaceTrack(audioTrack);
if (audioTransceiver && audioTransceiver.sender.track !== audioTrack) {
console.log(`[WebRTC][screen] Syncing audio track for ${peerIdHex}`);
await audioTransceiver.sender.replaceTrack(audioTrack);
changed = true;
}
if (changed && peer.pc.signalingState === "stable") {
this.onNegotiationNeeded(peerIdHex, peer.pc);
}
@@ -128,10 +129,19 @@ export class ScreenSharingWebRTCService {
screenPeersToConnect.forEach((id) => {
if (!this.peerManager.peers.has(id)) {
console.log(`[WebRTC][screen] Connecting to watched peer ${id.substring(0,8)}`);
this.peerManager.createPeerConnection(id, [
this.localScreenStream?.getVideoTracks()[0] || null,
this.localScreenStream?.getAudioTracks()[0] || null,
]);
const pc = this.peerManager.createPeerConnection(id);
if (pc) {
// Initiate with desired transceivers
pc.addTransceiver('video', { direction: 'sendrecv' });
pc.addTransceiver('audio', { direction: 'sendrecv' });
// Attach initial tracks if available
const transceivers = pc.getTransceivers();
const videoTrack = this.localScreenStream?.getVideoTracks()[0];
const audioTrack = this.localScreenStream?.getAudioTracks()[0];
if (videoTrack) transceivers[0].sender.replaceTrack(videoTrack);
if (audioTrack) transceivers[1].sender.replaceTrack(audioTrack);
}
}
});
@@ -271,6 +281,25 @@ export class ScreenSharingWebRTCService {
await pc.setRemoteDescription(
new RTCSessionDescription(JSON.parse(signal.data)),
);
// After setting remote offer, the transceivers exist.
// We should attach our local tracks if we have them.
const transceivers = pc.getTransceivers();
const videoTrack = this.localScreenStream?.getVideoTracks()[0];
const audioTrack = this.localScreenStream?.getAudioTracks()[0];
const videoTransceiver = transceivers.find(t => t.receiver.track.kind === 'video');
const audioTransceiver = transceivers.find(t => t.receiver.track.kind === 'audio');
if (videoTransceiver && videoTrack) {
await videoTransceiver.sender.replaceTrack(videoTrack);
videoTransceiver.direction = 'sendrecv';
}
if (audioTransceiver && audioTrack) {
await audioTransceiver.sender.replaceTrack(audioTrack);
audioTransceiver.direction = 'sendrecv';
}
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
console.log(`[WebRTC][screen] Sending Answer to ${peerIdHex.substring(0,8)}`);