avutil/channel_layout: add AV_CHANNEL_LAYOUT_RETYPE_FLAG_CANONICAL

Signed-off-by: Marton Balint <cus@passwd.hu>

doc/APIchanges

@@ -2,6 +2,9 @@ The last version increases of all libraries were on 2024-03-07
 
 API changes, most recent first:
 
+2024-03-xx - xxxxxxxxxx - lavu 59.2.100 - channel_layout.h
+  Add AV_CHANNEL_LAYOUT_RETYPE_FLAG_CANONICAL.
+
 2024-03-08 - xxxxxxxxxx - lavc 61.1.100 - avcodec.h
   Add AVCodecContext.[nb_]side_data_prefer_packet.
 

libavutil/channel_layout.c

@@ -553,6 +553,33 @@ static int ambisonic_order(const AVChannelLayout *channel_layout)
     return order;
 }
 
+static enum AVChannelOrder canonical_order(AVChannelLayout *channel_layout)
+{
+    int has_known_channel = 0;
+    int order;
+
+    if (channel_layout->order != AV_CHANNEL_ORDER_CUSTOM)
+        return channel_layout->order;
+
+    if (has_channel_names(channel_layout))
+        return AV_CHANNEL_ORDER_CUSTOM;
+
+    for (int i = 0; i < channel_layout->nb_channels && !has_known_channel; i++)
+        if (channel_layout->u.map[i].id != AV_CHAN_UNKNOWN)
+            has_known_channel = 1;
+    if (!has_known_channel)
+        return AV_CHANNEL_ORDER_UNSPEC;
+
+    if (masked_description(channel_layout, 0) > 0)
+        return AV_CHANNEL_ORDER_NATIVE;
+
+    order = ambisonic_order(channel_layout);
+    if (order >= 0 && masked_description(channel_layout, (order + 1) * (order + 1)) >= 0)
+        return AV_CHANNEL_ORDER_AMBISONIC;
+
+    return AV_CHANNEL_ORDER_CUSTOM;
+}
+
 /**
  * If the custom layout is n-th order standard-order ambisonic, with optional
  * extra non-diegetic channels at the end, write its string description in bp.
@@ -892,6 +919,9 @@ int av_channel_layout_retype(AVChannelLayout *channel_layout, enum AVChannelOrde
     if (!av_channel_layout_check(channel_layout))
         return AVERROR(EINVAL);
 
+    if (flags & AV_CHANNEL_LAYOUT_RETYPE_FLAG_CANONICAL)
+        order = canonical_order(channel_layout);
+
     if (channel_layout->order == order)
         return 0;
 

libavutil/channel_layout.h

@@ -680,6 +680,13 @@ int av_channel_layout_compare(const AVChannelLayout *chl, const AVChannelLayout
  */
 #define AV_CHANNEL_LAYOUT_RETYPE_FLAG_LOSSLESS (1 << 0)
 
+/**
+ * The specified retype target order is ignored and the simplest possible
+ * (canonical) order is used for which the input layout can be losslessy
+ * represented.
+ */
+#define AV_CHANNEL_LAYOUT_RETYPE_FLAG_CANONICAL (1 << 1)
+
 /**
  * Change the AVChannelOrder of a channel layout.
  *

libavutil/version.h

@@ -79,7 +79,7 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  59
-#define LIBAVUTIL_VERSION_MINOR   1
+#define LIBAVUTIL_VERSION_MINOR   2
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \