using System;
using System.Collections.Generic;
using System.IO;
///
/// Remuxes one HLS MPEG-TS segment (H.264 video + AAC ADTS audio) into a standalone MP4, without
/// re-encoding. Windows Media Foundation (Unity's VideoPlayer on Windows) can't read MPEG-TS, but plays
/// H.264/AAC in MP4. Checked against ffmpeg: decoded video and audio match the source segment exactly.
///
public static class TsToMp4
{
static readonly int[] AacRates = { 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350 };
struct Pes { public long Pts; public byte[] Data; }
struct VSample { public long Pts; public byte[] Data; public bool Key; }
/// Returns the MP4 bytes, or null if the segment has no usable video.
public static byte[] Convert(byte[] ts)
{
Demux(ts, out var videoPes, out var audioPes);
var video = VideoSamples(videoPes, out var sps, out var pps);
if (video.Count == 0 || sps == null || pps == null) return null;
var audio = AudioFrames(audioPes, out int aacObject, out int sfIndex, out int channels);
ParseSpsSize(sps, out int width, out int height);
// Durations: video from PTS deltas (90 kHz), audio is 1024 samples per AAC frame
var vDur = new List();
for (int i = 0; i < video.Count - 1; i++) vDur.Add((uint)Math.Max(1, video[i + 1].Pts - video[i].Pts));
uint avg = 3750;
if (vDur.Count > 0) { long sum = 0; foreach (var d in vDur) sum += d; avg = (uint)(sum / vDur.Count); }
vDur.Add(avg);
long vTotal = 0; foreach (var d in vDur) vTotal += d;
bool hasAudio = audio.Count > 0 && sfIndex >= 0 && sfIndex < AacRates.Length;
int rate = hasAudio ? AacRates[sfIndex] : 48000;
long aTotal = hasAudio ? audio.Count * 1024L : 0;
var ftyp = Box("ftyp", Cat(Ascii("isom"), U32(512), Ascii("isomiso2avc1mp41")));
long vBytes = 0; foreach (var s in video) vBytes += s.Data.Length;
byte[] moov = BuildMoov(video, vDur, vTotal, sps, pps, width, height, audio, hasAudio, aacObject, sfIndex, channels, rate, aTotal, 0, 0);
uint baseOffset = (uint)(ftyp.Length + moov.Length + 8);
moov = BuildMoov(video, vDur, vTotal, sps, pps, width, height, audio, hasAudio, aacObject, sfIndex, channels, rate, aTotal, baseOffset, baseOffset + (uint)vBytes);
using (var ms = new MemoryStream())
{
ms.Write(ftyp, 0, ftyp.Length);
ms.Write(moov, 0, moov.Length);
long mdatLen = 8 + vBytes;
if (hasAudio) foreach (var a in audio) mdatLen += a.Length;
var hdr = Cat(U32((uint)mdatLen), Ascii("mdat"));
ms.Write(hdr, 0, hdr.Length);
foreach (var s in video) ms.Write(s.Data, 0, s.Data.Length);
if (hasAudio) foreach (var a in audio) ms.Write(a, 0, a.Length);
return ms.ToArray();
}
}
// ---------------- MPEG-TS demux ----------------
static void Demux(byte[] data, out List video, out List audio)
{
video = new List(); audio = new List();
int pmtPid = -1, vPid = -1, aPid = -1;
var open = new Dictionary();
void Flush(int pid, List v, List a)
{
if (!open.TryGetValue(pid, out var p)) return;
if (p.buf.Length > 0)
{
var pes = new Pes { Pts = p.pts, Data = p.buf.ToArray() };
if (pid == vPid) v.Add(pes); else a.Add(pes);
}
open.Remove(pid);
}
for (int off = 0; off + 188 <= data.Length; off += 188)
{
if (data[off] != 0x47) continue;
bool pusi = (data[off + 1] & 0x40) != 0;
int pid = ((data[off + 1] & 0x1F) << 8) | data[off + 2];
int afc = (data[off + 3] >> 4) & 3;
int i = off + 4;
if (afc == 2 || afc == 3) i += 1 + data[off + 4];
if (afc == 0 || afc == 2 || i >= off + 188) continue;
int end = off + 188;
if (pid == 0 && pusi)
{
int t = i + 1 + data[i];
int secLen = ((data[t + 1] & 0x0F) << 8) | data[t + 2];
for (int j = t + 8; j < t + 3 + secLen - 4; j += 4)
{
int prog = (data[j] << 8) | data[j + 1];
if (prog != 0) pmtPid = ((data[j + 2] & 0x1F) << 8) | data[j + 3];
}
continue;
}
if (pid == pmtPid && pusi)
{
int t = i + 1 + data[i];
int secLen = ((data[t + 1] & 0x0F) << 8) | data[t + 2];
int progInfo = ((data[t + 10] & 0x0F) << 8) | data[t + 11];
int j = t + 12 + progInfo;
while (j < t + 3 + secLen - 4)
{
int streamType = data[j];
int esPid = ((data[j + 1] & 0x1F) << 8) | data[j + 2];
int esInfo = ((data[j + 3] & 0x0F) << 8) | data[j + 4];
if (streamType == 0x1B && vPid < 0) vPid = esPid;
if (streamType == 0x0F && aPid < 0) aPid = esPid;
j += 5 + esInfo;
}
continue;
}
if (pid != vPid && pid != aPid) continue;
if (pusi)
{
Flush(pid, video, audio);
if (end - i < 9 || data[i] != 0 || data[i + 1] != 0 || data[i + 2] != 1) continue;
int flags = data[i + 7], hlen = data[i + 8];
long pts = -1;
if ((flags & 0x80) != 0)
{
int b = i + 9;
pts = ((long)(data[b] & 0x0E) << 29) | ((long)data[b + 1] << 22) | ((long)(data[b + 2] & 0xFE) << 14) | ((long)data[b + 3] << 7) | ((long)data[b + 4] >> 1);
}
var ms = new MemoryStream();
int start = i + 9 + hlen;
if (start < end) ms.Write(data, start, end - start);
open[pid] = (pts, ms);
}
else if (open.TryGetValue(pid, out var cur))
{
cur.buf.Write(data, i, end - i);
}
}
Flush(vPid, video, audio);
Flush(aPid, video, audio);
}
// ---------------- H.264 ----------------
static List VideoSamples(List pes, out byte[] sps, out byte[] pps)
{
sps = null; pps = null;
var samples = new List();
foreach (var p in pes)
{
var body = new MemoryStream();
bool key = false;
foreach (var nal in NalUnits(p.Data))
{
int type = nal[0] & 0x1F;
if (type == 9) continue; // access unit delimiter
if (type == 7) sps = nal;
if (type == 8) pps = nal;
if (type == 5) key = true;
var len = U32((uint)nal.Length);
body.Write(len, 0, 4);
body.Write(nal, 0, nal.Length);
}
if (body.Length > 0) samples.Add(new VSample { Pts = p.Pts, Data = body.ToArray(), Key = key });
}
return samples;
}
static List NalUnits(byte[] es)
{
var units = new List();
int n = es.Length, i = 0, start = -1;
while (i + 3 <= n)
{
bool sc3 = es[i] == 0 && es[i + 1] == 0 && es[i + 2] == 1;
bool sc4 = i + 3 < n && es[i] == 0 && es[i + 1] == 0 && es[i + 2] == 0 && es[i + 3] == 1;
if (sc3 || sc4)
{
int len = sc3 ? 3 : 4;
if (start >= 0 && i > start) units.Add(Slice(es, start, i - start));
start = i + len; i += len;
}
else i++;
}
if (start >= 0 && start < n) units.Add(Slice(es, start, n - start));
return units;
}
/// Width and height from an SPS (with emulation prevention removed and frame cropping applied).
static void ParseSpsSize(byte[] spsNal, out int width, out int height)
{
width = 1280; height = 720;
try
{
var rbsp = new List();
for (int i = 1; i < spsNal.Length; i++)
{
if (i + 2 < spsNal.Length && spsNal[i] == 0 && spsNal[i + 1] == 0 && spsNal[i + 2] == 3) { rbsp.Add(0); rbsp.Add(0); i += 2; continue; }
rbsp.Add(spsNal[i]);
}
var r = new BitReader(rbsp.ToArray());
int profile = r.Bits(8); r.Bits(16); // constraint flags + level
r.Ue(); // seq_parameter_set_id
int chroma = 1;
if (profile == 100 || profile == 110 || profile == 122 || profile == 244 || profile == 44 || profile == 83 || profile == 86 || profile == 118 || profile == 128 || profile == 138 || profile == 139 || profile == 134 || profile == 135)
{
chroma = r.Ue();
if (chroma == 3) r.Bits(1);
r.Ue(); r.Ue(); r.Bits(1);
if (r.Bits(1) == 1)
for (int k = 0; k < (chroma != 3 ? 8 : 12); k++)
if (r.Bits(1) == 1)
{
int size = k < 6 ? 16 : 64, last = 8, next = 8;
for (int j = 0; j < size; j++) { if (next != 0) next = (last + r.Se() + 256) % 256; last = next == 0 ? last : next; }
}
}
r.Ue(); // log2_max_frame_num_minus4
int pocType = r.Ue();
if (pocType == 0) r.Ue();
else if (pocType == 1) { r.Bits(1); r.Se(); r.Se(); int cycle = r.Ue(); for (int k = 0; k < cycle; k++) r.Se(); }
r.Ue(); r.Bits(1); // max_num_ref_frames, gaps
int wMbs = r.Ue() + 1, hMapUnits = r.Ue() + 1;
int frameMbsOnly = r.Bits(1);
if (frameMbsOnly == 0) r.Bits(1);
r.Bits(1); // direct_8x8_inference
int cl = 0, cr = 0, ct = 0, cb = 0;
if (r.Bits(1) == 1) { cl = r.Ue(); cr = r.Ue(); ct = r.Ue(); cb = r.Ue(); }
int cropX = chroma == 0 ? 1 : 2, cropY = (chroma == 1 ? 2 : 1) * (2 - frameMbsOnly);
width = wMbs * 16 - (cl + cr) * cropX;
height = (2 - frameMbsOnly) * hMapUnits * 16 - (ct + cb) * cropY;
}
catch { width = 1280; height = 720; }
}
class BitReader
{
readonly byte[] d; int pos;
public BitReader(byte[] data) { d = data; }
public int Bits(int n) { int v = 0; for (int k = 0; k < n; k++) { v = (v << 1) | ((d[pos >> 3] >> (7 - (pos & 7))) & 1); pos++; } return v; }
public int Ue() { int zeros = 0; while (Bits(1) == 0) zeros++; return (1 << zeros) - 1 + (zeros > 0 ? Bits(zeros) : 0); }
public int Se() { int k = Ue(); return (k & 1) == 1 ? (k + 1) / 2 : -(k / 2); }
}
// ---------------- AAC ----------------
static List AudioFrames(List pes, out int objectType, out int sfIndex, out int channels)
{
objectType = 2; sfIndex = -1; channels = 2;
var frames = new List();
foreach (var p in pes)
{
var es = p.Data; int i = 0;
while (i + 7 <= es.Length)
{
if (es[i] != 0xFF || (es[i + 1] & 0xF0) != 0xF0) { i++; continue; }
bool protectionAbsent = (es[i + 1] & 1) == 1;
int profile = (es[i + 2] >> 6) & 3;
int sfi = (es[i + 2] >> 2) & 0xF;
int chan = ((es[i + 2] & 1) << 2) | (es[i + 3] >> 6);
int flen = ((es[i + 3] & 3) << 11) | (es[i + 4] << 3) | (es[i + 5] >> 5);
int hdr = protectionAbsent ? 7 : 9;
if (flen < hdr || i + flen > es.Length) break;
if (sfIndex < 0) { objectType = profile + 1; sfIndex = sfi; channels = chan; }
frames.Add(Slice(es, i + hdr, flen - hdr));
i += flen;
}
}
return frames;
}
// ---------------- MP4 boxes ----------------
static byte[] BuildMoov(List video, List vDur, long vTotal, byte[] sps, byte[] pps, int w, int h,
List audio, bool hasAudio, int aacObject, int sfIndex, int channels, int rate, long aTotal,
uint vOffset, uint aOffset)
{
uint vMs = (uint)(vTotal * 1000 / 90000), aMs = hasAudio ? (uint)(aTotal * 1000 / rate) : 0;
var sync = new List();
for (int i = 0; i < video.Count; i++) if (video[i].Key) sync.Add((uint)(i + 1));
var vSizes = new List(); foreach (var s in video) vSizes.Add((uint)s.Data.Length);
var vStbl = Box("stbl", Cat(StsdAvc1(sps, pps, w, h), Stts(vDur),
sync.Count > 0 && sync.Count < video.Count ? FullBox("stss", 0, 0, Cat(U32((uint)sync.Count), U32s(sync))) : new byte[0],
StscOne((uint)video.Count), Stsz(vSizes), Stco(vOffset)));
var vTrak = Box("trak", Cat(Tkhd(1, vMs, w, h, false),
Box("mdia", Cat(Mdhd(90000, (uint)vTotal), Hdlr("vide", "VideoHandler"),
Box("minf", Cat(FullBox("vmhd", 0, 1, new byte[8]), Dinf(), vStbl))))));
byte[] aTrak = new byte[0];
if (hasAudio)
{
var aDur = new List(); var aSizes = new List();
foreach (var f in audio) { aDur.Add(1024); aSizes.Add((uint)f.Length); }
var aStbl = Box("stbl", Cat(StsdMp4a(aacObject, sfIndex, channels, rate), Stts(aDur), StscOne((uint)audio.Count), Stsz(aSizes), Stco(aOffset)));
aTrak = Box("trak", Cat(Tkhd(2, aMs, 0, 0, true),
Box("mdia", Cat(Mdhd((uint)rate, (uint)aTotal), Hdlr("soun", "SoundHandler"),
Box("minf", Cat(FullBox("smhd", 0, 0, new byte[4]), Dinf(), aStbl))))));
}
return Box("moov", Cat(Mvhd(Math.Max(vMs, aMs)), vTrak, aTrak));
}
static readonly uint[] Matrix = { 0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000 };
static byte[] Mvhd(uint durMs) =>
FullBox("mvhd", 0, 0, Cat(U32(0), U32(0), U32(1000), U32(durMs), U32(0x00010000), U16(0x0100), new byte[10], U32s(Matrix), new byte[24], U32(3)));
static byte[] Tkhd(uint id, uint durMs, int w, int h, bool audio) =>
FullBox("tkhd", 0, 3, Cat(U32(0), U32(0), U32(id), U32(0), U32(durMs), new byte[8], U16(0), U16(0), U16((ushort)(audio ? 0x0100 : 0)), U16(0),
U32s(Matrix), U32((uint)w << 16), U32((uint)h << 16)));
static byte[] Mdhd(uint timescale, uint dur) => FullBox("mdhd", 0, 0, Cat(U32(0), U32(0), U32(timescale), U32(dur), U16(0x55C4), U16(0)));
static byte[] Hdlr(string type, string name) => FullBox("hdlr", 0, 0, Cat(U32(0), Ascii(type), new byte[12], Ascii(name), new byte[1]));
static byte[] Dinf() => Box("dinf", FullBox("dref", 0, 0, Cat(U32(1), FullBox("url ", 0, 1, new byte[0]))));
static byte[] Stts(List deltas)
{
var runs = new List<(uint count, uint delta)>();
foreach (var d in deltas)
{
if (runs.Count > 0 && runs[runs.Count - 1].delta == d) runs[runs.Count - 1] = (runs[runs.Count - 1].count + 1, d);
else runs.Add((1, d));
}
var body = new MemoryStream();
var n = U32((uint)runs.Count); body.Write(n, 0, 4);
foreach (var r in runs) { var c = U32(r.count); var dd = U32(r.delta); body.Write(c, 0, 4); body.Write(dd, 0, 4); }
return FullBox("stts", 0, 0, body.ToArray());
}
static byte[] StscOne(uint count) => FullBox("stsc", 0, 0, Cat(U32(1), U32(1), U32(count), U32(1)));
static byte[] Stsz(List sizes) => FullBox("stsz", 0, 0, Cat(U32(0), U32((uint)sizes.Count), U32s(sizes)));
static byte[] Stco(uint offset) => FullBox("stco", 0, 0, Cat(U32(1), U32(offset)));
static byte[] StsdAvc1(byte[] sps, byte[] pps, int w, int h)
{
var avcC = Cat(new byte[] { 1, sps[1], sps[2], sps[3], 0xFF, 0xE1 }, U16((ushort)sps.Length), sps, new byte[] { 1 }, U16((ushort)pps.Length), pps);
var entry = Cat(new byte[6], U16(1), new byte[16], U16((ushort)w), U16((ushort)h), U32(0x00480000), U32(0x00480000), U32(0), U16(1),
new byte[32], U16(0x0018), U16(0xFFFF), Box("avcC", avcC));
return FullBox("stsd", 0, 0, Cat(U32(1), Box("avc1", entry)));
}
static byte[] StsdMp4a(int objectType, int sfIndex, int channels, int rate)
{
var asc = U16((ushort)((objectType << 11) | (sfIndex << 7) | (channels << 3)));
byte[] Desc(byte tag, byte[] body) => Cat(new byte[] { tag, 0x80, 0x80, 0x80, (byte)body.Length }, body);
var dec = Desc(0x04, Cat(new byte[] { 0x40, 0x15, 0, 0, 0 }, U32(0), U32(0), Desc(0x05, asc)));
var es = Desc(0x03, Cat(U16(2), new byte[] { 0 }, dec, Desc(0x06, new byte[] { 0x02 })));
var entry = Cat(new byte[6], U16(1), new byte[8], U16((ushort)channels), U16(16), U16(0), U16(0), U32((uint)rate << 16), FullBox("esds", 0, 0, es));
return FullBox("stsd", 0, 0, Cat(U32(1), Box("mp4a", entry)));
}
// ---------------- byte helpers ----------------
static byte[] Box(string type, byte[] payload) => Cat(U32((uint)(8 + payload.Length)), Ascii(type), payload);
static byte[] FullBox(string type, int version, int flags, byte[] payload) => Box(type, Cat(U32((uint)((version << 24) | flags)), payload));
static byte[] U32(uint v) => new[] { (byte)(v >> 24), (byte)(v >> 16), (byte)(v >> 8), (byte)v };
static byte[] U16(ushort v) => new[] { (byte)(v >> 8), (byte)v };
static byte[] U16(int v) => U16((ushort)v);
static byte[] U32s(IList values) { var b = new byte[values.Count * 4]; for (int i = 0; i < values.Count; i++) Buffer.BlockCopy(U32(values[i]), 0, b, i * 4, 4); return b; }
static byte[] Ascii(string s) => System.Text.Encoding.ASCII.GetBytes(s);
static byte[] Slice(byte[] a, int start, int len) { var b = new byte[len]; Buffer.BlockCopy(a, start, b, 0, len); return b; }
static byte[] Cat(params byte[][] parts)
{
int n = 0; foreach (var p in parts) n += p.Length;
var r = new byte[n]; int o = 0;
foreach (var p in parts) { Buffer.BlockCopy(p, 0, r, o, p.Length); o += p.Length; }
return r;
}
}