Segment an mp4 file to HLS streaming files

Create_Iframe_Index_M3U8.bash

#!/bin/bash
# Create an Iframe index from HLS segmented streams
# $1: Filename to be created
# $2: Location of segmented ts files

# Check how many arguments
if [ $# != 2 ]; then
    echo "Usage: $0 [Input filename] [Location of segmented streams]"
    exit 1;
fi

# Check if parameters have values
if [ -z "$1" ]; then
    echo "No filename was given."
    exit 1;
fi

if [ -z "$2" ]; then
    echo "No location was given."
    exit 1;
fi

# Check if location exists
if [ ! -d "${2}" ]; then
    echo "Location '${2}' doesn't exist."
    exit 1;
fi

IFRAME_FILE="IFRAME_${1}.m3u8"
rm -f ${IFRAME_FILE}
touch ${IFRAME_FILE}
echo "#EXTM3U" > ${IFRAME_FILE}
echo "#EXT-X-TARGETDURATION:" >> ${IFRAME_FILE}
echo "#EXT-X-VERSION:4" >> ${IFRAME_FILE}
echo "#EXT-X-I-FRAMES-ONLY" >> ${IFRAME_FILE}

# Find all segmented files and sort by number
SEGMENTED_FILES=$(find "$2" -name \*.ts | sort -V)

# Source http://www.cyberciti.biz/tips/handling-filenames-with-spaces-in-bash.html
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")

for FILE in ${SEGMENTED_FILES}
do
    # Put into parenthesis to make arrays
    GET_NUM_IFRAMES=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I'))
    GET_PTS_TIME=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_pts_time=\([0-9]*.[0-9]*\).*/\1/p"))
    GET_PKT_SIZE=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_size=\([0-9]*\).*/\1/p"))
    GET_PKT_POS=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_pos=\([0-9]*\).*/\1/p"))
    GET_LAST_PTS_TIME=$(ffprobe -show_frames ${FILE} -of compact | tail -1 | sed -n "s/.*pkt_pts_time=\([0-9]*.[0-9]*\).*/\1/p")

    for (( i = 0 ; i < ${#GET_NUM_IFRAMES[@]} ; i++ ))
    do
	RESULT=
	if [ $(( $i + 1 )) -lt "${#GET_NUM_IFRAMES[@]}" ]; then
	    RESULT=$(echo "${GET_PTS_TIME[ (( $i + 1 )) ]} - ${GET_PTS_TIME[$i]}" | bc -l)
	else
	    RESULT=$(echo "${GET_LAST_PTS_TIME} - ${GET_PTS_TIME[$i]}" | bc -l)
	fi

	echo "#EXTINF:${RESULT}" >> ${IFRAME_FILE}
	echo "#EXT-X-BYTERANGE:${GET_PKT_SIZE[$i]}@${GET_PKT_POS[$i]}" >> ${IFRAME_FILE}
	echo "${FILE}" >> ${IFRAME_FILE}
    done

done

IFS=$SAVEIFS

echo "#EXT-X-ENDLIST" >> ${IFRAME_FILE}

Get_Duration_M3U8.bash

#!/bin/bash
# Get the total duration of the video,
# reading the durations from the M3U8 list
# $1: Location of the M3U8 list

# Check how many arguments
if [ $# != 1 ]; then
    echo "Usage: $0 [Location of M3U8 list]"
    exit 1;
fi

# Check if parameters have values

if [ -z "${1}" ]; then
    echo "No list was given."
    exit 1;
fi

# Check if list exists
if [ ! -f "${1}" ]; then
    echo "List '${1}' doesn't exist."
    exit 1;
fi

# Get all the durations of the segmented streams and create an array
GET_DURATIONS=($(cat "${1}" | sed -n "s/^#EXTINF:\([0-9]*.[0-9]*\).*/\1/p"))

TOTAL_DURATION="0"
for (( i = 0 ; i < ${#GET_DURATIONS[@]} ; i++ ))
do
    GET_DURATION_MSEC=$(echo "${GET_DURATIONS[$i]} * 1000" | bc -l)

    TOTAL_DURATION=$(echo "${GET_DURATION_MSEC} + ${TOTAL_DURATION}" | bc -l)
done

echo "M3U8 List: "${1}" Duration ${TOTAL_DURATION} ms, $(echo "${TOTAL_DURATION} / 1000" | bc -l) sec, $(echo "${TOTAL_DURATION} / 1000 / 60" | bc -l) min"

Get_Duration_Segmented_Streams.bash

#!/bin/bash 
# Get the total duration of the stream 
# by adding the duration of the segmented 
# streams.
# $1: Location of the M3U8 list, where the segmented 
# streams will read

# Check how many arguments
if [ $# != 1 ]; then
    echo "Usage: $0 [Location of M3U8 list of segmented streams]"
    exit 1;
fi

# Check if parameters have values

if [ -z "${1}" ]; then
    echo "No location was given."
    exit 1;
fi

# Check if file exists
if [ ! -f "${1}" ]; then
    echo "Location of list '${1}' doesn't exist."
    exit 1;
fi

DIRECTORY=$(dirname "${1}")

# Find all segmented files from the list and sort by number,
# if they are not sorted.
SEGMENTED_FILES=$(cat "${1}" | grep  \.*.ts | sort -V)

# Source http://www.cyberciti.biz/tips/handling-filenames-with-spaces-in-bash.html
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")

TOTAL_DURATION="0"
for FILE in ${SEGMENTED_FILES}
do
    GET_DURATION_SEC=$(ffprobe -show_format "${DIRECTORY}"/"${FILE}" -of compact | sed -n "s/.*duration=\([0-9]*.[0-9]*\).*/\1/p")
    GET_DURATION_MSEC=$(echo "${GET_DURATION_SEC} * 1000" | bc -l)

    TOTAL_DURATION=$(echo "${GET_DURATION_MSEC} + ${TOTAL_DURATION}" | bc -l)
done

IFS=$SAVEIFS

echo "M3U8 List: "${1}" Duration ${TOTAL_DURATION} ms, $(echo "${TOTAL_DURATION} / 1000" | bc -l) sec, $(echo "${TOTAL_DURATION} / 1000 / 60" | bc -l) min"

Get_M3U8_Lists.bash

#!/bin/bash 
# Get the m3u8 lists from a file 
# $1: A file which contains m3u8 lists, this
# could be a json or csv or anything else in the
# format "test.m3u8":
# $2: A prefix directory to the location of streams e.g /tmp/streams or /home/$USER/Downloads

# Check how many arguments
if [ $# != 2 ]; then
    echo "Usage: $0 [Location of the file containing the M3U8 lists] [prefix to the location of streams. e.g /tmp/streams]"
    exit 1;
fi

# Check if parameters have values

if [ -z "${1}" ]; then
    echo "No file was given."
    exit 1;
fi

if [ -z "${2}" ]; then
    echo "No prefix was given."
    exit 1;
fi

# Check if file exists
if [ ! -f "${1}" ]; then
    echo "File '${1}' doesn't exist."
    exit 1;
fi

# Find all segmented files from the list and sort by number,
# if they are not sorted.
M3U8_LISTS=$(cat "${1}" | sed -n "s/.*\"\(.*\.m3u8\)\"\:.*/\1/p")

# Source http://www.cyberciti.biz/tips/handling-filenames-with-spaces-in-bash.html
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")

for list in $M3U8_LISTS
do
    echo ""${2}"/"$list""
done

IFS=$SAVEIFS

Segment_MP4_to_HLS.bash

#!/bin/bash
# This script segment a mp4 video file to HLS compatible streams using ffmpeg.
# See below for the parameters used in ffmpeg.

CPUs=$(grep -c ^processor /proc/cpuinfo)

# $1: Filename to be created
# $2: Bitrate
# $3: Resolution
# $4: M3U8 playlist
function CREATE_BASIC_VARIANT_M3U8_LIST {
    BASIC_VARIANT_FILE="BASIC_VARIANT_${1}.m3u8"
    rm -f ${BASIC_VARIANT_FILE}
    touch ${BASIC_VARIANT_FILE}
    echo "#EXTM3U" > ${BASIC_VARIANT_FILE}
    echo "#EXT-X-VERSION:3" >> ${BASIC_VARIANT_FILE}
    echo "#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=${2},CODECS=\"avc1.4d001f,mp4a.40.2\",RESOLUTION=${3}" >> ${BASIC_VARIANT_FILE}
    echo "${4}" >> ${BASIC_VARIANT_FILE}
    echo "#EXT-X-ENDLIST" >> ${BASIC_VARIANT_FILE}
}

# $1: Filename to be created
# $2: Location of segmented ts files
function CREATE_IFRAME_M3U8_LIST {
    IFRAME_FILE="IFRAME_${1}.m3u8"
    rm -f ${IFRAME_FILE}
    touch ${IFRAME_FILE}
    echo "#EXTM3U" > ${IFRAME_FILE}
    echo "#EXT-X-TARGETDURATION:" >> ${IFRAME_FILE}
    echo "#EXT-X-VERSION:4" >> ${IFRAME_FILE}
    echo "#EXT-X-I-FRAMES-ONLY" >> ${IFRAME_FILE}

    # Find all segmented files and sort by number
    SEGMENTED_FILES=$(find "$2" -name \*.ts | sort -n)

    # Source http://www.cyberciti.biz/tips/handling-filenames-with-spaces-in-bash.html
    SAVEIFS=$IFS
    IFS=$(echo -en "\n\b")

    for FILE in ${SEGMENTED_FILES}
    do
	# Put into parenthesis to make arrays
	GET_NUM_IFRAMES=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I'))
	GET_PTS_TIME=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_pts_time=\([0-9]*.[0-9]*\).*/\1/p"))
	GET_PKT_SIZE=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_size=\([0-9]*\).*/\1/p"))
	GET_PKT_POS=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_pos=\([0-9]*\).*/\1/p"))
	GET_LAST_PTS_TIME=$(ffprobe -show_frames ${FILE} -of compact | tail -1 | sed -n "s/.*pkt_pts_time=\([0-9]*.[0-9]*\).*/\1/p")

	for (( i = 0 ; i < ${#GET_NUM_IFRAMES[@]} ; i++ ))
	do
	    RESULT=
	    if [ $(( $i + 1 )) -lt "${#GET_NUM_IFRAMES[@]}" ]; then
		RESULT=$(echo "${GET_PTS_TIME[ (( $i + 1 )) ]} - ${GET_PTS_TIME[$i]}" | bc -l)
	    else
		RESULT=$(echo "${GET_LAST_PTS_TIME} - ${GET_PTS_TIME[$i]}" | bc -l)
	    fi

	    echo "#EXTINF:${RESULT}" >> ${IFRAME_FILE}
	    echo "#EXT-X-BYTERANGE:${GET_PKT_SIZE[$i]}@${GET_PKT_POS[$i]}" >> ${IFRAME_FILE}
	    echo "${FILE}" >> ${IFRAME_FILE}
	done
    done

    IFS=$SAVEIFS

    echo "#EXT-X-ENDLIST" >> ${IFRAME_FILE}
}

# Check how many arguments
if [ $# != 5 ]; then
    echo "Usage: $0 [mp4 video] [demux: segment or hls] [output directory prefix] [hls duration of each segment stream] [single ts file(single_file) or not(multiple_files), byterange to .m3u8]"
    exit 1;
fi

# Check if parameters have values
if [ -z "$1" ]; then
    echo "No input video file was given."
    exit 1;
fi

if [ -z "$2" ]; then
    echo "No demux was given."
    exit 1;
fi

if [ -z "$3" ]; then
    echo "No output directory prefix was given."
    exit 1;
fi

if [ -z "$4" ]; then
    echo "No HLS duration was given."
    exit 1;
fi

if [ -z "$5" ]; then
    echo "Single file or not was not specified."
    exit 1;
fi



FILE_INPUT="$1"

# Check if the file exists
if [ ! -f "${FILE_INPUT}" ]; then
    echo "File '${FILE_INPUT}' doesn't exist."
    exit 1;
fi

# Check if the file is MP4 by extention
if [ "${FILE_INPUT##*.}" != 'mp4' ]; then
    echo "File '${MP4_FILE}' is not an mp4 video."
    exit 1;
fi

DEMUX="$2"

# Check if the correct demux used.
if [ "$DEMUX" != 'segment' -a "$DEMUX" != 'hls' ]; then
    echo "Demux '$DEMUX' is not correct. Use segment or hls."
    exit 1;
fi

DIR_OUTPUT_PREFIX="$3"
HLS_DURATION="$4"

SINGLE_FILE="$5"
if [ "$SINGLE_FILE" != 'single_file' -a "$SINGLE_FILE" != 'multiple_files' ]; then
    echo "Single file option '$SINGLE_FILE' is not correct. Use single_file or multiple_files."
    exit 1;
fi



DIR_OUTPUT=""${DIR_OUTPUT_PREFIX}"-"${DEMUX}"-"${HLS_DURATION}"-"${SINGLE_FILE}""

# Check if directory exists
if [ ! -f "${DIR_OUTPUT}" ]; then
    echo "Directory '${DIR_OUTPUT}' doesn't exist. Creating..."
    mkdir -p "${DIR_OUTPUT}"
fi

FFPROBE_FORMAT_COMPACT=$(ffprobe -show_format "${FILE_INPUT}" -of compact)
GET_BITRATE=$(echo "${FFPROBE_FORMAT_COMPACT}" | sed -n "s/.*bit_rate=\([0-9]*\).*/\1/p")
GET_DURATION_SEC=$(echo "${FFPROBE_FORMAT_COMPACT}" | sed -n "s/.*duration=\([0-9]*.[0-9]*\).*/\1/p")
GET_DURATION_MSEC=$(echo "${GET_DURATION_SEC} * 1000" | bc -l)

FFPROBE_STREAMS_COMPACT=$(ffprobe -show_streams "${FILE_INPUT}" -of compact)
GET_WIDTH=$(echo "${FFPROBE_STREAMS_COMPACT}" | grep video | sed -n "s/.*width=\([0-9]*\).*/\1/p")
GET_HEIGHT=$(echo "${FFPROBE_STREAMS_COMPACT}" | grep video | sed -n "s/.*height=\([0-9]*\).*/\1/p")
GET_RESOLUTION="${GET_WIDTH}x${GET_HEIGHT}"
GET_VIDEO_CODEC_TAG_STRING=$(echo "${FFPROBE_STREAMS_COMPACT}" | grep video | sed -n "s/.*codec_tag_string=\([0-9A-Za-z]*\).*/\1/p")
GET_AUDIO_CODEC_TAG_STRING=$(echo "${FFPROBE_STREAMS_COMPACT}" | grep audio | sed -n "s/.*codec_tag_string=\([0-9A-Za-z]*\).*/\1/p")

MP4_FILE_NOPATH="${FILE_INPUT##*/}"
FILENAME="${MP4_FILE_NOPATH%.*}"

COMMAND_OUTPUT=''
COMMAND_EXECUTED=''

if [ "${DEMUX}" == "segment" ]; then
    COMMAND_OUTPUT=$(ffmpeg -i "${FILE_INPUT}" -threads "${CPUs}" -codec copy -map 0 -f "${DEMUX}" -vbsf h264_mp4toannexb -flags -global_header -segment_format mpegts -segment_list "${DIR_OUTPUT}"/"${FILENAME}".m3u8 -segment_list_flags +live -segment_time "${HLS_DURATION}" "${DIR_OUTPUT}"/"${FILENAME}"-%03d.ts)
elif [ "${DEMUX}" == "hls" ]; then
    if [ "${SINGLE_FILE}" == "multiple_files" ]; then
	COMMAND_OUTPUT=$(ffmpeg -i "${FILE_INPUT}" -threads "${CPUs}" -codec copy -map 0 -f "${DEMUX}" -vbsf h264_mp4toannexb -flags -global_header -hls_time "${HLS_DURATION}" -hls_list_size 0 "${DIR_OUTPUT}"/"${FILENAME}".m3u8)
    elif [ "${SINGLE_FILE}" == "single_file" ]; then
	COMMAND_OUTPUT=$(ffmpeg -i "${FILE_INPUT}" -threads "${CPUs}" -codec copy -map 0 -f "${DEMUX}" -vbsf h264_mp4toannexb -flags -global_header -hls_time "${HLS_DURATION}" -hls_list_size 0 -hls_flags single_file "${DIR_OUTPUT}"/"${FILENAME}".m3u8)
    fi
fi

echo "Command output: '$COMMAND_OUTPUT'"
echo "Bitrate: ${GET_BITRATE} Resolution: ${GET_RESOLUTION}"

CREATE_BASIC_VARIANT_M3U8_LIST ${FILENAME} ${GET_BITRATE} ${GET_RESOLUTION} "${DIR_OUTPUT}"/"${FILENAME}".m3u8

# ffmpeg parameter explanation
# -i "${FILE_INPUT}": Give the input video stream to segment.
# -codec copy: Copy the stream without reencoding. Documentation https://www.ffmpeg.org/ffmpeg.html#Stream-copy.
# -map 0: Map ALL streams from the video file to output. See documentation of -map https://www.ffmpeg.org/ffmpeg.html#Advanced-options.
# -f "$MUXER": Select muxer segment or hls. For segment Documentation https://www.ffmpeg.org/ffmpeg.html#Main-options and https://ffmpeg.org/ffmpeg-formats.html#segment_002c-stream_005fsegment_002c-ssegment. For hls Documentation https://ffmpeg.org/ffmpeg-formats.html#hls-1.
# -vbsf h264_mp4toannexb: Bitstream filter h264_mp4toannexb. Documentation https://www.ffmpeg.org/ffmpeg-bitstream-filters.html#h264_005fmp4toannexb.
# -flags: Unset flags. Documentation https://www.ffmpeg.org/ffmpeg-all.html#Codec-Options.
# -global_header: Don't place global headers in extradata. Instead place in every keyframe. Documentation under flags https://www.ffmpeg.org/ffmpeg-all.html#Codec-Options.
# See http://www.ffmpeg.org/ffmpeg-formats.html#segment for segmenter settings.
# -segment_format: Override the inner container format, by default it is guessed by the filename extension. Here is mpegts
# -segment_list: Generate also a listfile named name. If not specified no listfile is generated.
# -segment_time: Set segment duration to time, the value must be a duration specification. Default value is "2". Here is 10.
# See https://ffmpeg.org/ffmpeg-formats.html#hls-1 for hls settings
# -hls_time: Set segment duration to time, the value must be a duration specification. Default value is "2". Here is 10.
# -hls_list_size: Set the maximum number of playlist entries. If set to 0 the list file will contain all the segments. Default value is 5.
# -hls_flags single_file: If this flag is set, the muxer will store all segments in a single MPEG-TS file, and will use byte ranges in the playlist. HLS playlists generated with this way will have the version number 4. Documentation https://www.ffmpeg.org/ffmpeg-formats.html#Options-2

Thanks for sharing this! I couldn't find any other sources detailing how to generate iframe playlists. I noticed, however, the length reported by ffprobe (pkt_size) is smaller than the size generated by apple's mediafilesegmenter -iframe-index-file... have you run into this?

BTW, I can suggest some ways to make i-frame generation faster:

• run ffprobe only once and re-use the output to generate each array
• use ffprobe -select_streams v -skip_frame nokey

Hi tmm1, glad you find those scripts useful.

Sorry for the delayed message but I had totally forgotten that I have put those scripts online and I remembered recently.

I was experimenting myself trying to understand how the segmentation is happening and how the playlists are created, so for this reason I created these scripts. Regarding the iframe playlists, I believe the results are not correct (as you probably found out). It was a quick hack to see if I could create iframe playlists using ffprobe. I will have a look at it. Might use another tool, I am not sure yet.

Thanks for the tips regarding the ffprobe, I will use them in my scripts.

Should not this command produce clips with avc1 (annexb) bitstream instead of h.264?

COMMAND_OUTPUT=$(ffmpeg -i "${FILE_INPUT}" -threads "${CPUs}" -codec copy -map 0 -f "${DEMUX}" -vbsf h264_mp4toannexb -flags -global_header -hls_time "${HLS_DURATION}" -hls_list_size 0 "${DIR_OUTPUT}"/"${FILENAME}".m3u8)